From b5e27af4da354ef0128dcb434a18e6b67734524a Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Mon, 14 Jul 2025 15:55:00 -0400 Subject: [PATCH 01/11] release candidate 4.0.0 --- CMakeLists.txt | 6 +- Doxyfile | 2 +- include/simdjson/arm64/implementation.h | 1 - include/simdjson/fallback/implementation.h | 1 - include/simdjson/haswell/implementation.h | 1 - include/simdjson/icelake/implementation.h | 1 - include/simdjson/implementation.h | 17 - include/simdjson/lasx/implementation.h | 1 - include/simdjson/lsx/implementation.h | 1 - include/simdjson/ppc64/implementation.h | 1 - include/simdjson/simdjson_version.h | 6 +- include/simdjson/westmere/implementation.h | 1 - singleheader/simdjson.cpp | 493 +- singleheader/simdjson.h | 11695 ++++++++++++++++++- singleheader/singleheader.zip | Bin 8071720 -> 8041220 bytes src/arm64.cpp | 4 - src/fallback.cpp | 4 - src/generic/stage2/stringparsing.h | 98 - src/haswell.cpp | 4 - src/icelake.cpp | 4 - src/implementation.cpp | 9 - src/lasx.cpp | 4 - src/lsx.cpp | 4 - src/ppc64.cpp | 4 - src/westmere.cpp | 5 - 25 files changed, 12131 insertions(+), 236 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index af1e25d97..e720aa2b3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,7 +5,7 @@ cmake_policy(VERSION 3.5) # For doctest project( simdjson # The version number is modified by tools/release.py - VERSION 3.13.0 + VERSION 4.0.0 DESCRIPTION "Parsing gigabytes of JSON per second" HOMEPAGE_URL "https://simdjson.org/" LANGUAGES CXX C @@ -22,8 +22,8 @@ string( # ---- Options, variables ---- # These version numbers are modified by tools/release.py -set(SIMDJSON_LIB_VERSION "26.0.0" CACHE STRING "simdjson library version") -set(SIMDJSON_LIB_SOVERSION "26" CACHE STRING "simdjson library soversion") +set(SIMDJSON_LIB_VERSION "28.0.0" CACHE STRING "simdjson library version") +set(SIMDJSON_LIB_SOVERSION "28" CACHE STRING "simdjson library soversion") option(SIMDJSON_BUILD_STATIC_LIB "Build simdjson_static library along with simdjson (only makes sense if BUILD_SHARED_LIBS=ON)" OFF) if(SIMDJSON_BUILD_STATIC_LIB AND NOT BUILD_SHARED_LIBS) diff --git a/Doxyfile b/Doxyfile index e76055f67..676ca1015 100644 --- a/Doxyfile +++ b/Doxyfile @@ -38,7 +38,7 @@ PROJECT_NAME = simdjson # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = "3.13.0" +PROJECT_NUMBER = "4.0.0" # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/include/simdjson/arm64/implementation.h b/include/simdjson/arm64/implementation.h index 00c5bf566..c9b7dd753 100644 --- a/include/simdjson/arm64/implementation.h +++ b/include/simdjson/arm64/implementation.h @@ -23,7 +23,6 @@ public: ) const noexcept final; simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final; simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final; - simdjson_warn_unused size_t write_string_escaped(const std::string_view input, char *out) const noexcept final; }; } // namespace arm64 diff --git a/include/simdjson/fallback/implementation.h b/include/simdjson/fallback/implementation.h index e607253a0..523f06d2e 100644 --- a/include/simdjson/fallback/implementation.h +++ b/include/simdjson/fallback/implementation.h @@ -26,7 +26,6 @@ public: ) const noexcept final; simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final; simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final; - simdjson_warn_unused size_t write_string_escaped(const std::string_view input, char *out) const noexcept final; }; } // namespace fallback diff --git a/include/simdjson/haswell/implementation.h b/include/simdjson/haswell/implementation.h index 984d9f7ed..6861e4298 100644 --- a/include/simdjson/haswell/implementation.h +++ b/include/simdjson/haswell/implementation.h @@ -28,7 +28,6 @@ public: ) const noexcept final; simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final; simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final; - simdjson_warn_unused size_t write_string_escaped(const std::string_view input, char *out) const noexcept final; }; } // namespace haswell diff --git a/include/simdjson/icelake/implementation.h b/include/simdjson/icelake/implementation.h index d197b4309..940c5f992 100644 --- a/include/simdjson/icelake/implementation.h +++ b/include/simdjson/icelake/implementation.h @@ -28,7 +28,6 @@ public: ) const noexcept final; simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final; simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final; - simdjson_warn_unused size_t write_string_escaped(const std::string_view input, char *out) const noexcept final; }; } // namespace icelake diff --git a/include/simdjson/implementation.h b/include/simdjson/implementation.h index 366d29119..19cb37162 100644 --- a/include/simdjson/implementation.h +++ b/include/simdjson/implementation.h @@ -26,15 +26,6 @@ simdjson_inline simdjson_warn_unused bool validate_utf8(const std::string_view s return validate_utf8(sv.data(), sv.size()); } -/** - * Write the string to the output buffer while escaping double-quote, backlash and ascii control characters. - * - * @param input the string_view to escape - * @param out output buffer (for escaped string): to be safe, it should have 6 * input.size() allocated bytes. - * @return number of bytes written - */ -simdjson_warn_unused size_t write_string_escaped(const std::string_view input, char *out) noexcept; - /** * Validate the UTF-8 string. * @@ -136,14 +127,6 @@ public: */ simdjson_warn_unused virtual bool validate_utf8(const char *buf, size_t len) const noexcept = 0; - /** - * Write the string to the output buffer while escaping double-quote, backlash and ascii control characters. - * - * @param input the string_view to escape - * @param out output buffer (for escaped string): to be safe, it should have 6 * input.size() allocated bytes. - * @return number of bytes written - */ - simdjson_warn_unused virtual size_t write_string_escaped(const std::string_view input, char *out) const noexcept = 0; protected: /** @private Construct an implementation with the given name and description. For subclasses. */ simdjson_inline implementation( diff --git a/include/simdjson/lasx/implementation.h b/include/simdjson/lasx/implementation.h index d64ec3e49..8aafbb8b8 100644 --- a/include/simdjson/lasx/implementation.h +++ b/include/simdjson/lasx/implementation.h @@ -23,7 +23,6 @@ public: ) const noexcept final; simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final; simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final; - simdjson_warn_unused size_t write_string_escaped(const std::string_view input, char *out) const noexcept final; }; } // namespace lasx diff --git a/include/simdjson/lsx/implementation.h b/include/simdjson/lsx/implementation.h index a17fb5532..14468777d 100644 --- a/include/simdjson/lsx/implementation.h +++ b/include/simdjson/lsx/implementation.h @@ -23,7 +23,6 @@ public: ) const noexcept final; simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final; simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final; - simdjson_warn_unused size_t write_string_escaped(const std::string_view input, char *out) const noexcept final; }; } // namespace lsx diff --git a/include/simdjson/ppc64/implementation.h b/include/simdjson/ppc64/implementation.h index b3445af26..33436f534 100644 --- a/include/simdjson/ppc64/implementation.h +++ b/include/simdjson/ppc64/implementation.h @@ -32,7 +32,6 @@ public: size_t &dst_len) const noexcept final; simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final; - simdjson_warn_unused size_t write_string_escaped(const std::string_view input, char *out) const noexcept final; }; } // namespace ppc64 diff --git a/include/simdjson/simdjson_version.h b/include/simdjson/simdjson_version.h index f06aa81f9..add79802f 100644 --- a/include/simdjson/simdjson_version.h +++ b/include/simdjson/simdjson_version.h @@ -4,18 +4,18 @@ #define SIMDJSON_SIMDJSON_VERSION_H /** The version of simdjson being used (major.minor.revision) */ -#define SIMDJSON_VERSION "3.13.0" +#define SIMDJSON_VERSION "4.0.0" namespace simdjson { enum { /** * The major version (MAJOR.minor.revision) of simdjson being used. */ - SIMDJSON_VERSION_MAJOR = 3, + SIMDJSON_VERSION_MAJOR = 4, /** * The minor version (major.MINOR.revision) of simdjson being used. */ - SIMDJSON_VERSION_MINOR = 13, + SIMDJSON_VERSION_MINOR = 0, /** * The revision (major.minor.REVISION) of simdjson being used. */ diff --git a/include/simdjson/westmere/implementation.h b/include/simdjson/westmere/implementation.h index c7e655b5e..37392be2a 100644 --- a/include/simdjson/westmere/implementation.h +++ b/include/simdjson/westmere/implementation.h @@ -24,7 +24,6 @@ public: ) const noexcept final; simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final; simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final; - simdjson_warn_unused size_t write_string_escaped(const std::string_view input, char *out) const noexcept final; }; } // namespace westmere diff --git a/singleheader/simdjson.cpp b/singleheader/simdjson.cpp index 97ec9ee7a..7724757b3 100644 --- a/singleheader/simdjson.cpp +++ b/singleheader/simdjson.cpp @@ -1,4 +1,4 @@ -/* auto-generated on 2025-06-24 15:13:44 -0400. Do not edit! */ +/* auto-generated on 2025-07-14 15:43:52 -0400. Do not edit! */ /* including simdjson.cpp: */ /* begin file simdjson.cpp */ #define SIMDJSON_SRC_SIMDJSON_CPP @@ -77,12 +77,31 @@ #endif #endif +#ifndef SIMDJSON_CONSTEXPR_LAMBDA +#if SIMDJSON_CPLUSPLUS17 +#define SIMDJSON_CONSTEXPR_LAMBDA constexpr +#else +#define SIMDJSON_CONSTEXPR_LAMBDA +#endif +#endif + + + #ifdef __has_include #if __has_include() #include #endif #endif +// The current specification is unclear on how we detect +// static reflection, both __cpp_lib_reflection and +// __cpp_impl_reflection are proposed in the draft specification. +// For now, we disable static reflect by default. It must be +// specified at compiler time. +#ifndef SIMDJSON_STATIC_REFLECTION +#define SIMDJSON_STATIC_REFLECTION 0 // disabled by default. +#endif + #if defined(__apple_build_version__) #if __apple_build_version__ < 14000000 #define SIMDJSON_CONCEPT_DISABLED 1 // apple-clang/13 doesn't support std::convertible_to @@ -101,6 +120,14 @@ #define SIMDJSON_SUPPORTS_DESERIALIZATION 0 #endif // defined(__cpp_concepts) && !defined(SIMDJSON_CONCEPT_DISABLED) +#if !defined(SIMDJSON_CONSTEVAL) +#if defined(__cpp_consteval) && __cpp_consteval >= 201811L +#define SIMDJSON_CONSTEVAL 1 +#else +#define SIMDJSON_CONSTEVAL 0 +#endif // defined(__cpp_consteval) && __cpp_consteval >= 201811L +#endif // !defined(SIMDJSON_CONSTEVAL) + #endif // SIMDJSON_COMPILER_CHECK_H /* end file simdjson/compiler_check.h */ /* including simdjson/portability.h: #include "simdjson/portability.h" */ @@ -298,6 +325,7 @@ using std::size_t; #if defined(NDEBUG) || defined(__OPTIMIZE__) || (defined(_MSC_VER) && !defined(_DEBUG)) // If NDEBUG is set, or __OPTIMIZE__ is set, or we are under MSVC in release mode, // then do away with asserts and use __assume. +// We still recommend that our users set NDEBUG in release mode. #if SIMDJSON_VISUAL_STUDIO #define SIMDJSON_UNREACHABLE() __assume(0) #define SIMDJSON_ASSUME(COND) __assume(COND) @@ -2342,16 +2370,25 @@ namespace std { // It could also wrongly set SIMDJSON_DEVELOPMENT_CHECKS (e.g., if the programmer // sets _DEBUG in a release build under Visual Studio, or if some compiler fails to // set the __OPTIMIZE__ macro). +// We make it so that if NDEBUG is defined, then SIMDJSON_DEVELOPMENT_CHECKS +// is not defined, irrespective of the compiler. +// We recommend that users set NDEBUG in release builds, so that +// SIMDJSON_DEVELOPMENT_CHECKS is not defined in release builds by default, +// irrespective of the compiler. #ifndef SIMDJSON_DEVELOPMENT_CHECKS #ifdef _MSC_VER // Visual Studio seems to set _DEBUG for debug builds. -#ifdef _DEBUG +// We set SIMDJSON_DEVELOPMENT_CHECKS to 1 if _DEBUG is defined +// and NDEBUG is not defined. +#if defined(_DEBUG) && !defined(NDEBUG) #define SIMDJSON_DEVELOPMENT_CHECKS 1 #endif // _DEBUG #else // _MSC_VER // All other compilers appear to set __OPTIMIZE__ to a positive integer // when the compiler is optimizing. -#ifndef __OPTIMIZE__ +// We only set SIMDJSON_DEVELOPMENT_CHECKS if both __OPTIMIZE__ +// and NDEBUG are not defined. +#if !defined(__OPTIMIZE__) && !defined(NDEBUG) #define SIMDJSON_DEVELOPMENT_CHECKS 1 #endif // __OPTIMIZE__ #endif // _MSC_VER @@ -2463,7 +2500,8 @@ enum error_code { SCALAR_DOCUMENT_AS_VALUE, ///< A scalar document is treated as a value. OUT_OF_BOUNDS, ///< Attempted to access location outside of document. TRAILING_CONTENT, ///< Unexpected trailing content in the JSON input - NUM_ERROR_CODES + OUT_OF_CAPACITY, ///< The capacity was exceeded, we cannot allocate enough memory. + NUM_ERROR_CODES ///< Placeholder for end of error code list. }; /** @@ -8209,6 +8247,12 @@ namespace { tmp = vpaddq_u8(tmp, tmp); return vgetq_lane_u16(vreinterpretq_u16_u8(tmp), 0); } + // Returns 4-bit out of each byte, alternating between the high 4 bits and low + // bits result it is 64 bit. + simdjson_inline uint64_t to_bitmask64() const { + return vget_lane_u64( + vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(*this), 4)), 0); + } simdjson_inline bool any() const { return vmaxvq_u32(vreinterpretq_u32_u8(*this)) != 0; } }; @@ -8285,7 +8329,7 @@ namespace { // Bit-specific operations simdjson_inline simd8 any_bits_set(simd8 bits) const { return vtstq_u8(*this, bits); } - simdjson_inline bool any_bits_set_anywhere() const { return this->max_val() != 0; } + simdjson_inline bool any_bits_set_anywhere() const { return vmaxvq_u32(vreinterpretq_u32_u8(*this)) != 0; } simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return (*this & bits).any_bits_set_anywhere(); } template simdjson_inline simd8 shr() const { return vshrq_n_u8(*this, N); } @@ -8298,7 +8342,12 @@ namespace { return lookup_table.apply_lookup_16_to(*this); } - + // Returns 4-bit out of each byte, alternating between the high 4 bits and low + // bits result it is 64 bit. + simdjson_inline uint64_t to_bitmask64() const { + return vget_lane_u64( + vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(*this), 4)), 0); + } // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). // Passing a 0 value for mask would be equivalent to writing out every byte to output. // Only the first 16 - count_ones(mask) bytes of the result are significant but 16 bytes @@ -8621,6 +8670,32 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin }; } +struct escaping { + static constexpr uint32_t BYTES_PROCESSED = 16; + simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst); + + simdjson_inline bool has_escape() { return escape_bits != 0; } + simdjson_inline int escape_index() { return trailing_zeroes(escape_bits) / 4; } + + uint64_t escape_bits; +}; // struct escaping + + + +simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) { + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v(src); + v.store(dst); + simd8 is_quote = (v == '"'); + simd8 is_backslash = (v == '\\'); + simd8 is_control = (v < 32); + return { + (is_backslash | is_quote | is_control).to_bitmask64() + }; +} + + + } // unnamed namespace } // namespace arm64 } // namespace simdjson @@ -10991,6 +11066,12 @@ namespace { tmp = vpaddq_u8(tmp, tmp); return vgetq_lane_u16(vreinterpretq_u16_u8(tmp), 0); } + // Returns 4-bit out of each byte, alternating between the high 4 bits and low + // bits result it is 64 bit. + simdjson_inline uint64_t to_bitmask64() const { + return vget_lane_u64( + vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(*this), 4)), 0); + } simdjson_inline bool any() const { return vmaxvq_u32(vreinterpretq_u32_u8(*this)) != 0; } }; @@ -11067,7 +11148,7 @@ namespace { // Bit-specific operations simdjson_inline simd8 any_bits_set(simd8 bits) const { return vtstq_u8(*this, bits); } - simdjson_inline bool any_bits_set_anywhere() const { return this->max_val() != 0; } + simdjson_inline bool any_bits_set_anywhere() const { return vmaxvq_u32(vreinterpretq_u32_u8(*this)) != 0; } simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return (*this & bits).any_bits_set_anywhere(); } template simdjson_inline simd8 shr() const { return vshrq_n_u8(*this, N); } @@ -11080,7 +11161,12 @@ namespace { return lookup_table.apply_lookup_16_to(*this); } - + // Returns 4-bit out of each byte, alternating between the high 4 bits and low + // bits result it is 64 bit. + simdjson_inline uint64_t to_bitmask64() const { + return vget_lane_u64( + vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(*this), 4)), 0); + } // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). // Passing a 0 value for mask would be equivalent to writing out every byte to output. // Only the first 16 - count_ones(mask) bytes of the result are significant but 16 bytes @@ -11403,6 +11489,32 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin }; } +struct escaping { + static constexpr uint32_t BYTES_PROCESSED = 16; + simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst); + + simdjson_inline bool has_escape() { return escape_bits != 0; } + simdjson_inline int escape_index() { return trailing_zeroes(escape_bits) / 4; } + + uint64_t escape_bits; +}; // struct escaping + + + +simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) { + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v(src); + v.store(dst); + simd8 is_quote = (v == '"'); + simd8 is_backslash = (v == '\\'); + simd8 is_control = (v < 32); + return { + (is_backslash | is_quote | is_control).to_bitmask64() + }; +} + + + } // unnamed namespace } // namespace arm64 } // namespace simdjson @@ -13508,6 +13620,7 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V /* end file generic/stage2/json_iterator.h for arm64 */ /* including generic/stage2/stringparsing.h for arm64: #include */ /* begin file generic/stage2/stringparsing.h for arm64 */ +#include #ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H /* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ @@ -13747,6 +13860,7 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t } } // namespace stringparsing + } // unnamed namespace } // namespace arm64 } // namespace simdjson @@ -14998,6 +15112,31 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin }; } + +struct escaping { + static constexpr uint32_t BYTES_PROCESSED = 32; + simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst); + + simdjson_inline bool has_escape() { return escape_bits != 0; } + simdjson_inline int escape_index() { return trailing_zeroes(escape_bits); } + + uint64_t escape_bits; +}; // struct escaping + + + +simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) { + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v(src); + v.store(dst); + simd8 is_quote = (v == '"'); + simd8 is_backslash = (v == '\\'); + simd8 is_control = (v < 32); + return { + uint64_t((is_backslash | is_quote | is_control).to_bitmask()) + }; +} + } // unnamed namespace } // namespace haswell } // namespace simdjson @@ -17651,6 +17790,31 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin }; } + +struct escaping { + static constexpr uint32_t BYTES_PROCESSED = 32; + simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst); + + simdjson_inline bool has_escape() { return escape_bits != 0; } + simdjson_inline int escape_index() { return trailing_zeroes(escape_bits); } + + uint64_t escape_bits; +}; // struct escaping + + + +simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) { + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v(src); + v.store(dst); + simd8 is_quote = (v == '"'); + simd8 is_backslash = (v == '\\'); + simd8 is_control = (v < 32); + return { + uint64_t((is_backslash | is_quote | is_control).to_bitmask()) + }; +} + } // unnamed namespace } // namespace haswell } // namespace simdjson @@ -19754,6 +19918,7 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V /* end file generic/stage2/json_iterator.h for haswell */ /* including generic/stage2/stringparsing.h for haswell: #include */ /* begin file generic/stage2/stringparsing.h for haswell */ +#include #ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H /* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ @@ -19993,6 +20158,7 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t } } // namespace stringparsing + } // unnamed namespace } // namespace haswell } // namespace simdjson @@ -20834,7 +21000,6 @@ namespace simd { friend simdjson_really_inline uint64_t operator==(const simd8 lhs, const simd8 rhs) { return _mm512_cmpeq_epi8_mask(lhs, rhs); } - static const int SIZE = sizeof(base::value); template @@ -21177,6 +21342,35 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin }; } + + +struct escaping { + static constexpr uint32_t BYTES_PROCESSED = 64; + simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst); + + simdjson_inline bool has_escape() { return escape_bits != 0; } + simdjson_inline int escape_index() { return trailing_zeroes(uint64_t(escape_bits)); } + + __mmask64 escape_bits; +}; // struct escaping + + + +simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) { + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v(src); + v.store(dst); + __mmask64 is_quote = _mm512_cmpeq_epi8_mask(v, _mm512_set1_epi8('"')); + __mmask64 is_backslash = _mm512_cmpeq_epi8_mask(v, _mm512_set1_epi8('\\')); + __mmask64 is_control = _mm512_cmplt_epi8_mask(v, _mm512_set1_epi8(32)); + return { + (is_backslash | is_quote | is_control) + }; +} + + + + } // unnamed namespace } // namespace icelake } // namespace simdjson @@ -23483,7 +23677,6 @@ namespace simd { friend simdjson_really_inline uint64_t operator==(const simd8 lhs, const simd8 rhs) { return _mm512_cmpeq_epi8_mask(lhs, rhs); } - static const int SIZE = sizeof(base::value); template @@ -23826,6 +24019,35 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin }; } + + +struct escaping { + static constexpr uint32_t BYTES_PROCESSED = 64; + simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst); + + simdjson_inline bool has_escape() { return escape_bits != 0; } + simdjson_inline int escape_index() { return trailing_zeroes(uint64_t(escape_bits)); } + + __mmask64 escape_bits; +}; // struct escaping + + + +simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) { + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v(src); + v.store(dst); + __mmask64 is_quote = _mm512_cmpeq_epi8_mask(v, _mm512_set1_epi8('"')); + __mmask64 is_backslash = _mm512_cmpeq_epi8_mask(v, _mm512_set1_epi8('\\')); + __mmask64 is_control = _mm512_cmplt_epi8_mask(v, _mm512_set1_epi8(32)); + return { + (is_backslash | is_quote | is_control) + }; +} + + + + } // unnamed namespace } // namespace icelake } // namespace simdjson @@ -25989,6 +26211,7 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V /* end file generic/stage2/json_iterator.h for icelake */ /* including generic/stage2/stringparsing.h for icelake: #include */ /* begin file generic/stage2/stringparsing.h for icelake */ +#include #ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H /* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ @@ -26228,6 +26451,7 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t } } // namespace stringparsing + } // unnamed namespace } // namespace icelake } // namespace simdjson @@ -27630,6 +27854,32 @@ backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { }; } + +struct escaping { + static constexpr uint32_t BYTES_PROCESSED = 16; + simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst); + + simdjson_inline bool has_escape() { return escape_bits != 0; } + simdjson_inline int escape_index() { return trailing_zeroes(escape_bits); } + + uint64_t escape_bits; +}; // struct escaping + + + +simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) { + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v(src); + v.store(dst); + simd8 is_quote = (v == '"'); + simd8 is_backslash = (v == '\\'); + simd8 is_control = (v < 32); + return { + // We store it as a 64-bit bitmask even though we only need 16 bits. + uint64_t((is_backslash | is_quote | is_control).to_bitmask()) + }; +} + } // unnamed namespace } // namespace ppc64 } // namespace simdjson @@ -30394,6 +30644,32 @@ backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { }; } + +struct escaping { + static constexpr uint32_t BYTES_PROCESSED = 16; + simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst); + + simdjson_inline bool has_escape() { return escape_bits != 0; } + simdjson_inline int escape_index() { return trailing_zeroes(escape_bits); } + + uint64_t escape_bits; +}; // struct escaping + + + +simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) { + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v(src); + v.store(dst); + simd8 is_quote = (v == '"'); + simd8 is_backslash = (v == '\\'); + simd8 is_control = (v < 32); + return { + // We store it as a 64-bit bitmask even though we only need 16 bits. + uint64_t((is_backslash | is_quote | is_control).to_bitmask()) + }; +} + } // unnamed namespace } // namespace ppc64 } // namespace simdjson @@ -32499,6 +32775,7 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V /* end file generic/stage2/json_iterator.h for ppc64 */ /* including generic/stage2/stringparsing.h for ppc64: #include */ /* begin file generic/stage2/stringparsing.h for ppc64 */ +#include #ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H /* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ @@ -32738,6 +33015,7 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t } } // namespace stringparsing + } // unnamed namespace } // namespace ppc64 } // namespace simdjson @@ -34389,6 +34667,31 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin }; } + +struct escaping { + static constexpr uint32_t BYTES_PROCESSED = 16; + simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst); + + simdjson_inline bool has_escape() { return escape_bits != 0; } + simdjson_inline int escape_index() { return trailing_zeroes(escape_bits); } + + uint64_t escape_bits; +}; // struct escaping + + + +simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) { + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v(src); + v.store(dst); + simd8 is_quote = (v == '"'); + simd8 is_backslash = (v == '\\'); + simd8 is_control = (v < 32); + return { + uint64_t((is_backslash | is_quote | is_control).to_bitmask()) + }; +} + } // unnamed namespace } // namespace westmere } // namespace simdjson @@ -37468,6 +37771,31 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin }; } + +struct escaping { + static constexpr uint32_t BYTES_PROCESSED = 16; + simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst); + + simdjson_inline bool has_escape() { return escape_bits != 0; } + simdjson_inline int escape_index() { return trailing_zeroes(escape_bits); } + + uint64_t escape_bits; +}; // struct escaping + + + +simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) { + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v(src); + v.store(dst); + simd8 is_quote = (v == '"'); + simd8 is_backslash = (v == '\\'); + simd8 is_control = (v < 32); + return { + uint64_t((is_backslash | is_quote | is_control).to_bitmask()) + }; +} + } // unnamed namespace } // namespace westmere } // namespace simdjson @@ -39571,6 +39899,7 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V /* end file generic/stage2/json_iterator.h for westmere */ /* including generic/stage2/stringparsing.h for westmere: #include */ /* begin file generic/stage2/stringparsing.h for westmere */ +#include #ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H /* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ @@ -39810,6 +40139,7 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t } } // namespace stringparsing + } // unnamed namespace } // namespace westmere } // namespace simdjson @@ -40338,7 +40668,6 @@ simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t * if (error) { return error; } return stage2(_doc); } - } // namespace westmere } // namespace simdjson @@ -40968,6 +41297,31 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin }; } + +struct escaping { + static constexpr uint32_t BYTES_PROCESSED = 16; + simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst); + + simdjson_inline bool has_escape() { return escape_bits != 0; } + simdjson_inline int escape_index() { return trailing_zeroes(escape_bits); } + + uint64_t escape_bits; +}; // struct escaping + + + +simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) { + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v(src); + v.store(dst); + simd8 is_quote = (v == '"'); + simd8 is_backslash = (v == '\\'); + simd8 is_control = (v < 32); + return { + (is_backslash | is_quote | is_control).to_bitmask() + }; +} + } // unnamed namespace } // namespace lsx } // namespace simdjson @@ -43517,6 +43871,31 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin }; } + +struct escaping { + static constexpr uint32_t BYTES_PROCESSED = 16; + simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst); + + simdjson_inline bool has_escape() { return escape_bits != 0; } + simdjson_inline int escape_index() { return trailing_zeroes(escape_bits); } + + uint64_t escape_bits; +}; // struct escaping + + + +simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) { + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v(src); + v.store(dst); + simd8 is_quote = (v == '"'); + simd8 is_backslash = (v == '\\'); + simd8 is_control = (v < 32); + return { + (is_backslash | is_quote | is_control).to_bitmask() + }; +} + } // unnamed namespace } // namespace lsx } // namespace simdjson @@ -45622,6 +46001,7 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V /* end file generic/stage2/json_iterator.h for lsx */ /* including generic/stage2/stringparsing.h for lsx: #include */ /* begin file generic/stage2/stringparsing.h for lsx */ +#include #ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H /* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ @@ -45861,6 +46241,7 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t } } // namespace stringparsing + } // unnamed namespace } // namespace lsx } // namespace simdjson @@ -46994,6 +47375,31 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin }; } + +struct escaping { + static constexpr uint32_t BYTES_PROCESSED = 16; + simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst); + + simdjson_inline bool has_escape() { return escape_bits != 0; } + simdjson_inline int escape_index() { return trailing_zeroes(escape_bits); } + + uint64_t escape_bits; +}; // struct escaping + + + +simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) { + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v(src); + v.store(dst); + simd8 is_quote = (v == '"'); + simd8 is_backslash = (v == '\\'); + simd8 is_control = (v < 32); + return { + (is_backslash | is_quote | is_control).to_bitmask() + }; +} + } // unnamed namespace } // namespace lasx } // namespace simdjson @@ -49559,6 +49965,31 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin }; } + +struct escaping { + static constexpr uint32_t BYTES_PROCESSED = 16; + simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst); + + simdjson_inline bool has_escape() { return escape_bits != 0; } + simdjson_inline int escape_index() { return trailing_zeroes(escape_bits); } + + uint64_t escape_bits; +}; // struct escaping + + + +simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) { + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v(src); + v.store(dst); + simd8 is_quote = (v == '"'); + simd8 is_backslash = (v == '\\'); + simd8 is_control = (v < 32); + return { + (is_backslash | is_quote | is_control).to_bitmask() + }; +} + } // unnamed namespace } // namespace lasx } // namespace simdjson @@ -51664,6 +52095,7 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V /* end file generic/stage2/json_iterator.h for lasx */ /* including generic/stage2/stringparsing.h for lasx: #include */ /* begin file generic/stage2/stringparsing.h for lasx */ +#include #ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H /* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ @@ -51903,6 +52335,7 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t } } // namespace stringparsing + } // unnamed namespace } // namespace lasx } // namespace simdjson @@ -52532,6 +52965,24 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin return { src[0] }; } + +struct escaping { + static constexpr uint32_t BYTES_PROCESSED = 1; + simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst); + + simdjson_inline bool has_escape() { return escape_bits; } + simdjson_inline int escape_index() { return 0; } + + bool escape_bits; +}; // struct escaping + + + +simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) { + dst[0] = src[0]; + return { (src[0] == '\\') || (src[0] == '"') || (src[0] < 32) }; +} + } // unnamed namespace } // namespace fallback } // namespace simdjson @@ -54686,6 +55137,24 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin return { src[0] }; } + +struct escaping { + static constexpr uint32_t BYTES_PROCESSED = 1; + simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst); + + simdjson_inline bool has_escape() { return escape_bits; } + simdjson_inline int escape_index() { return 0; } + + bool escape_bits; +}; // struct escaping + + + +simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) { + dst[0] = src[0]; + return { (src[0] == '\\') || (src[0] == '"') || (src[0] < 32) }; +} + } // unnamed namespace } // namespace fallback } // namespace simdjson @@ -54892,6 +55361,7 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par /* end file generic/stage1/find_next_document_index.h for fallback */ /* including generic/stage2/stringparsing.h for fallback: #include */ /* begin file generic/stage2/stringparsing.h for fallback */ +#include #ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H /* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ @@ -55131,6 +55601,7 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t } } // namespace stringparsing + } // unnamed namespace } // namespace fallback } // namespace simdjson diff --git a/singleheader/simdjson.h b/singleheader/simdjson.h index cc3bf0e1f..d1dbaad83 100644 --- a/singleheader/simdjson.h +++ b/singleheader/simdjson.h @@ -1,4 +1,4 @@ -/* auto-generated on 2025-06-24 15:13:44 -0400. Do not edit! */ +/* auto-generated on 2025-07-14 15:43:52 -0400. Do not edit! */ /* including simdjson.h: */ /* begin file simdjson.h */ #ifndef SIMDJSON_H @@ -97,12 +97,31 @@ #endif #endif +#ifndef SIMDJSON_CONSTEXPR_LAMBDA +#if SIMDJSON_CPLUSPLUS17 +#define SIMDJSON_CONSTEXPR_LAMBDA constexpr +#else +#define SIMDJSON_CONSTEXPR_LAMBDA +#endif +#endif + + + #ifdef __has_include #if __has_include() #include #endif #endif +// The current specification is unclear on how we detect +// static reflection, both __cpp_lib_reflection and +// __cpp_impl_reflection are proposed in the draft specification. +// For now, we disable static reflect by default. It must be +// specified at compiler time. +#ifndef SIMDJSON_STATIC_REFLECTION +#define SIMDJSON_STATIC_REFLECTION 0 // disabled by default. +#endif + #if defined(__apple_build_version__) #if __apple_build_version__ < 14000000 #define SIMDJSON_CONCEPT_DISABLED 1 // apple-clang/13 doesn't support std::convertible_to @@ -121,6 +140,14 @@ #define SIMDJSON_SUPPORTS_DESERIALIZATION 0 #endif // defined(__cpp_concepts) && !defined(SIMDJSON_CONCEPT_DISABLED) +#if !defined(SIMDJSON_CONSTEVAL) +#if defined(__cpp_consteval) && __cpp_consteval >= 201811L +#define SIMDJSON_CONSTEVAL 1 +#else +#define SIMDJSON_CONSTEVAL 0 +#endif // defined(__cpp_consteval) && __cpp_consteval >= 201811L +#endif // !defined(SIMDJSON_CONSTEVAL) + #endif // SIMDJSON_COMPILER_CHECK_H /* end file simdjson/compiler_check.h */ /* including simdjson/portability.h: #include "simdjson/portability.h" */ @@ -318,6 +345,7 @@ using std::size_t; #if defined(NDEBUG) || defined(__OPTIMIZE__) || (defined(_MSC_VER) && !defined(_DEBUG)) // If NDEBUG is set, or __OPTIMIZE__ is set, or we are under MSVC in release mode, // then do away with asserts and use __assume. +// We still recommend that our users set NDEBUG in release mode. #if SIMDJSON_VISUAL_STUDIO #define SIMDJSON_UNREACHABLE() __assume(0) #define SIMDJSON_ASSUME(COND) __assume(COND) @@ -2362,16 +2390,25 @@ namespace std { // It could also wrongly set SIMDJSON_DEVELOPMENT_CHECKS (e.g., if the programmer // sets _DEBUG in a release build under Visual Studio, or if some compiler fails to // set the __OPTIMIZE__ macro). +// We make it so that if NDEBUG is defined, then SIMDJSON_DEVELOPMENT_CHECKS +// is not defined, irrespective of the compiler. +// We recommend that users set NDEBUG in release builds, so that +// SIMDJSON_DEVELOPMENT_CHECKS is not defined in release builds by default, +// irrespective of the compiler. #ifndef SIMDJSON_DEVELOPMENT_CHECKS #ifdef _MSC_VER // Visual Studio seems to set _DEBUG for debug builds. -#ifdef _DEBUG +// We set SIMDJSON_DEVELOPMENT_CHECKS to 1 if _DEBUG is defined +// and NDEBUG is not defined. +#if defined(_DEBUG) && !defined(NDEBUG) #define SIMDJSON_DEVELOPMENT_CHECKS 1 #endif // _DEBUG #else // _MSC_VER // All other compilers appear to set __OPTIMIZE__ to a positive integer // when the compiler is optimizing. -#ifndef __OPTIMIZE__ +// We only set SIMDJSON_DEVELOPMENT_CHECKS if both __OPTIMIZE__ +// and NDEBUG are not defined. +#if !defined(__OPTIMIZE__) && !defined(NDEBUG) #define SIMDJSON_DEVELOPMENT_CHECKS 1 #endif // __OPTIMIZE__ #endif // _MSC_VER @@ -2440,18 +2477,18 @@ namespace std { #define SIMDJSON_SIMDJSON_VERSION_H /** The version of simdjson being used (major.minor.revision) */ -#define SIMDJSON_VERSION "3.13.0" +#define SIMDJSON_VERSION "4.0.0" namespace simdjson { enum { /** * The major version (MAJOR.minor.revision) of simdjson being used. */ - SIMDJSON_VERSION_MAJOR = 3, + SIMDJSON_VERSION_MAJOR = 4, /** * The minor version (major.MINOR.revision) of simdjson being used. */ - SIMDJSON_VERSION_MINOR = 13, + SIMDJSON_VERSION_MINOR = 0, /** * The revision (major.minor.REVISION) of simdjson being used. */ @@ -2526,7 +2563,8 @@ enum error_code { SCALAR_DOCUMENT_AS_VALUE, ///< A scalar document is treated as a value. OUT_OF_BOUNDS, ///< Attempted to access location outside of document. TRAILING_CONTENT, ///< Unexpected trailing content in the JSON input - NUM_ERROR_CODES + OUT_OF_CAPACITY, ///< The capacity was exceeded, we cannot allocate enough memory. + NUM_ERROR_CODES ///< Placeholder for end of error code list. }; /** @@ -10623,6 +10661,12 @@ namespace { tmp = vpaddq_u8(tmp, tmp); return vgetq_lane_u16(vreinterpretq_u16_u8(tmp), 0); } + // Returns 4-bit out of each byte, alternating between the high 4 bits and low + // bits result it is 64 bit. + simdjson_inline uint64_t to_bitmask64() const { + return vget_lane_u64( + vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(*this), 4)), 0); + } simdjson_inline bool any() const { return vmaxvq_u32(vreinterpretq_u32_u8(*this)) != 0; } }; @@ -10699,7 +10743,7 @@ namespace { // Bit-specific operations simdjson_inline simd8 any_bits_set(simd8 bits) const { return vtstq_u8(*this, bits); } - simdjson_inline bool any_bits_set_anywhere() const { return this->max_val() != 0; } + simdjson_inline bool any_bits_set_anywhere() const { return vmaxvq_u32(vreinterpretq_u32_u8(*this)) != 0; } simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return (*this & bits).any_bits_set_anywhere(); } template simdjson_inline simd8 shr() const { return vshrq_n_u8(*this, N); } @@ -10712,7 +10756,12 @@ namespace { return lookup_table.apply_lookup_16_to(*this); } - + // Returns 4-bit out of each byte, alternating between the high 4 bits and low + // bits result it is 64 bit. + simdjson_inline uint64_t to_bitmask64() const { + return vget_lane_u64( + vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(*this), 4)), 0); + } // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). // Passing a 0 value for mask would be equivalent to writing out every byte to output. // Only the first 16 - count_ones(mask) bytes of the result are significant but 16 bytes @@ -11035,6 +11084,32 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin }; } +struct escaping { + static constexpr uint32_t BYTES_PROCESSED = 16; + simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst); + + simdjson_inline bool has_escape() { return escape_bits != 0; } + simdjson_inline int escape_index() { return trailing_zeroes(escape_bits) / 4; } + + uint64_t escape_bits; +}; // struct escaping + + + +simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) { + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v(src); + v.store(dst); + simd8 is_quote = (v == '"'); + simd8 is_backslash = (v == '\\'); + simd8 is_control = (v < 32); + return { + (is_backslash | is_quote | is_control).to_bitmask64() + }; +} + + + } // unnamed namespace } // namespace arm64 } // namespace simdjson @@ -13071,6 +13146,24 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin return { src[0] }; } + +struct escaping { + static constexpr uint32_t BYTES_PROCESSED = 1; + simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst); + + simdjson_inline bool has_escape() { return escape_bits; } + simdjson_inline int escape_index() { return 0; } + + bool escape_bits; +}; // struct escaping + + + +simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) { + dst[0] = src[0]; + return { (src[0] == '\\') || (src[0] == '"') || (src[0] < 32) }; +} + } // unnamed namespace } // namespace fallback } // namespace simdjson @@ -15775,6 +15868,31 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin }; } + +struct escaping { + static constexpr uint32_t BYTES_PROCESSED = 32; + simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst); + + simdjson_inline bool has_escape() { return escape_bits != 0; } + simdjson_inline int escape_index() { return trailing_zeroes(escape_bits); } + + uint64_t escape_bits; +}; // struct escaping + + + +simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) { + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v(src); + v.store(dst); + simd8 is_quote = (v == '"'); + simd8 is_backslash = (v == '\\'); + simd8 is_control = (v < 32); + return { + uint64_t((is_backslash | is_quote | is_control).to_bitmask()) + }; +} + } // unnamed namespace } // namespace haswell } // namespace simdjson @@ -17984,7 +18102,6 @@ namespace simd { friend simdjson_really_inline uint64_t operator==(const simd8 lhs, const simd8 rhs) { return _mm512_cmpeq_epi8_mask(lhs, rhs); } - static const int SIZE = sizeof(base::value); template @@ -18327,6 +18444,35 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin }; } + + +struct escaping { + static constexpr uint32_t BYTES_PROCESSED = 64; + simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst); + + simdjson_inline bool has_escape() { return escape_bits != 0; } + simdjson_inline int escape_index() { return trailing_zeroes(uint64_t(escape_bits)); } + + __mmask64 escape_bits; +}; // struct escaping + + + +simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) { + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v(src); + v.store(dst); + __mmask64 is_quote = _mm512_cmpeq_epi8_mask(v, _mm512_set1_epi8('"')); + __mmask64 is_backslash = _mm512_cmpeq_epi8_mask(v, _mm512_set1_epi8('\\')); + __mmask64 is_control = _mm512_cmplt_epi8_mask(v, _mm512_set1_epi8(32)); + return { + (is_backslash | is_quote | is_control) + }; +} + + + + } // unnamed namespace } // namespace icelake } // namespace simdjson @@ -21114,6 +21260,32 @@ backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { }; } + +struct escaping { + static constexpr uint32_t BYTES_PROCESSED = 16; + simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst); + + simdjson_inline bool has_escape() { return escape_bits != 0; } + simdjson_inline int escape_index() { return trailing_zeroes(escape_bits); } + + uint64_t escape_bits; +}; // struct escaping + + + +simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) { + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v(src); + v.store(dst); + simd8 is_quote = (v == '"'); + simd8 is_backslash = (v == '\\'); + simd8 is_control = (v < 32); + return { + // We store it as a 64-bit bitmask even though we only need 16 bits. + uint64_t((is_backslash | is_quote | is_control).to_bitmask()) + }; +} + } // unnamed namespace } // namespace ppc64 } // namespace simdjson @@ -24162,6 +24334,31 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin }; } + +struct escaping { + static constexpr uint32_t BYTES_PROCESSED = 16; + simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst); + + simdjson_inline bool has_escape() { return escape_bits != 0; } + simdjson_inline int escape_index() { return trailing_zeroes(escape_bits); } + + uint64_t escape_bits; +}; // struct escaping + + + +simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) { + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v(src); + v.store(dst); + simd8 is_quote = (v == '"'); + simd8 is_backslash = (v == '\\'); + simd8 is_control = (v < 32); + return { + uint64_t((is_backslash | is_quote | is_control).to_bitmask()) + }; +} + } // unnamed namespace } // namespace westmere } // namespace simdjson @@ -26683,6 +26880,31 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin }; } + +struct escaping { + static constexpr uint32_t BYTES_PROCESSED = 16; + simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst); + + simdjson_inline bool has_escape() { return escape_bits != 0; } + simdjson_inline int escape_index() { return trailing_zeroes(escape_bits); } + + uint64_t escape_bits; +}; // struct escaping + + + +simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) { + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v(src); + v.store(dst); + simd8 is_quote = (v == '"'); + simd8 is_backslash = (v == '\\'); + simd8 is_control = (v < 32); + return { + (is_backslash | is_quote | is_control).to_bitmask() + }; +} + } // unnamed namespace } // namespace lsx } // namespace simdjson @@ -29219,6 +29441,31 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin }; } + +struct escaping { + static constexpr uint32_t BYTES_PROCESSED = 16; + simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst); + + simdjson_inline bool has_escape() { return escape_bits != 0; } + simdjson_inline int escape_index() { return trailing_zeroes(escape_bits); } + + uint64_t escape_bits; +}; // struct escaping + + + +simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) { + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v(src); + v.store(dst); + simd8 is_quote = (v == '"'); + simd8 is_backslash = (v == '\\'); + simd8 is_control = (v < 32); + return { + (is_backslash | is_quote | is_control).to_bitmask() + }; +} + } // unnamed namespace } // namespace lasx } // namespace simdjson @@ -31162,6 +31409,7 @@ simdjson_inline implementation_simdjson_result_base::implementation_simdjson_ // Internal headers needed for ondemand generics. // All includes not under simdjson/generic/ondemand must be here! // Otherwise, amalgamation will fail. +/* skipped duplicate #include "simdjson/concepts.h" */ /* skipped duplicate #include "simdjson/dom/base.h" // for MINIMAL_DOCUMENT_CAPACITY */ /* skipped duplicate #include "simdjson/implementation.h" */ /* skipped duplicate #include "simdjson/padded_string.h" */ @@ -31596,6 +31844,12 @@ namespace { tmp = vpaddq_u8(tmp, tmp); return vgetq_lane_u16(vreinterpretq_u16_u8(tmp), 0); } + // Returns 4-bit out of each byte, alternating between the high 4 bits and low + // bits result it is 64 bit. + simdjson_inline uint64_t to_bitmask64() const { + return vget_lane_u64( + vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(*this), 4)), 0); + } simdjson_inline bool any() const { return vmaxvq_u32(vreinterpretq_u32_u8(*this)) != 0; } }; @@ -31672,7 +31926,7 @@ namespace { // Bit-specific operations simdjson_inline simd8 any_bits_set(simd8 bits) const { return vtstq_u8(*this, bits); } - simdjson_inline bool any_bits_set_anywhere() const { return this->max_val() != 0; } + simdjson_inline bool any_bits_set_anywhere() const { return vmaxvq_u32(vreinterpretq_u32_u8(*this)) != 0; } simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return (*this & bits).any_bits_set_anywhere(); } template simdjson_inline simd8 shr() const { return vshrq_n_u8(*this, N); } @@ -31685,7 +31939,12 @@ namespace { return lookup_table.apply_lookup_16_to(*this); } - + // Returns 4-bit out of each byte, alternating between the high 4 bits and low + // bits result it is 64 bit. + simdjson_inline uint64_t to_bitmask64() const { + return vget_lane_u64( + vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(*this), 4)), 0); + } // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). // Passing a 0 value for mask would be equivalent to writing out every byte to output. // Only the first 16 - count_ones(mask) bytes of the result are significant but 16 bytes @@ -32008,6 +32267,32 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin }; } +struct escaping { + static constexpr uint32_t BYTES_PROCESSED = 16; + simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst); + + simdjson_inline bool has_escape() { return escape_bits != 0; } + simdjson_inline int escape_index() { return trailing_zeroes(escape_bits) / 4; } + + uint64_t escape_bits; +}; // struct escaping + + + +simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) { + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v(src); + v.store(dst); + simd8 is_quote = (v == '"'); + simd8 is_backslash = (v == '\\'); + simd8 is_control = (v < 32); + return { + (is_backslash | is_quote | is_control).to_bitmask64() + }; +} + + + } // unnamed namespace } // namespace arm64 } // namespace simdjson @@ -32166,10 +32451,26 @@ concept nothrow_deserializable = nothrow_custom_deserializable || is_bu /// Deserialize Tag inline constexpr struct deserialize_tag { + using array_type = arm64::ondemand::array; + using object_type = arm64::ondemand::object; using value_type = arm64::ondemand::value; using document_type = arm64::ondemand::document; using document_reference_type = arm64::ondemand::document_reference; + // Customization Point for array + template + requires custom_deserializable + [[nodiscard]] constexpr /* error_code */ auto operator()(array_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + return tag_invoke(*this, object, output); + } + + // Customization Point for object + template + requires custom_deserializable + [[nodiscard]] constexpr /* error_code */ auto operator()(object_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + return tag_invoke(*this, object, output); + } + // Customization Point for value template requires custom_deserializable @@ -32737,6 +33038,7 @@ public: * * You may use get_double(), get_bool(), get_uint64(), get_int64(), * get_object(), get_array(), get_raw_json_string(), or get_string() instead. + * When SIMDJSON_SUPPORTS_DESERIALIZATION is set, custom types are also supported. * * @returns A value of the given type, parsed from the JSON. * @returns INCORRECT_TYPE If the JSON value is not the given type. @@ -32760,6 +33062,7 @@ public: * Get this value as the given type. * * Supported types: object, array, raw_json_string, string_view, uint64_t, int64_t, double, bool + * If the macro SIMDJSON_SUPPORTS_DESERIALIZATION is set, then custom types are also supported. * * @param out This is set to a value of the given type, parsed from the JSON. If there is an error, this may not be initialized. * @returns INCORRECT_TYPE If the JSON value is not an object. @@ -35002,6 +35305,37 @@ public: * - INDEX_OUT_OF_BOUNDS if the array index is larger than an array length */ simdjson_inline simdjson_result at(size_t index) noexcept; + +#if SIMDJSON_SUPPORTS_DESERIALIZATION + /** + * Get this array as the given type. + * + * @param out This is set to a value of the given type, parsed from the JSON. If there is an error, this may not be initialized. + * @returns INCORRECT_TYPE If the JSON array is not of the given type. + * @returns SUCCESS If the parse succeeded and the out parameter was set to the value. + */ + template + simdjson_inline error_code get(T &out) + noexcept(custom_deserializable ? nothrow_custom_deserializable : true) { + static_assert(custom_deserializable); + return deserialize(*this, out); + } + /** + * Get this array as the given type. + * + * @returns A value of the given type, parsed from the JSON. + * @returns INCORRECT_TYPE If the JSON value is not the given type. + */ + template + simdjson_inline simdjson_result get() + noexcept(custom_deserializable ? nothrow_custom_deserializable : true) + { + static_assert(std::is_default_constructible::value, "The specified type is not default constructible."); + T out{}; + SIMDJSON_TRY(get(out)); + return out; + } +#endif // SIMDJSON_SUPPORTS_DESERIALIZATION protected: /** * Go to the end of the array, no matter where you are right now. @@ -35080,7 +35414,28 @@ public: simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; simdjson_inline simdjson_result raw_json() noexcept; +#if SIMDJSON_SUPPORTS_DESERIALIZATION + // TODO: move this code into object-inl.h + template + simdjson_inline simdjson_result get() noexcept { + if (error()) { return error(); } + if constexpr (std::is_same_v) { + return first; + } + return first.get(); + } + template + simdjson_inline error_code get(T& out) noexcept { + if (error()) { return error(); } + if constexpr (std::is_same_v) { + out = first; + } else { + SIMDJSON_TRY( first.get(out) ); + } + return SUCCESS; + } +#endif // SIMDJSON_SUPPORTS_DESERIALIZATION }; } // namespace simdjson @@ -35414,7 +35769,7 @@ public: * Be mindful that the document instance must remain in scope while you are accessing object, array and value instances. * * @param out This is set to a value of the given type, parsed from the JSON. If there is an error, this may not be initialized. - * @returns INCORRECT_TYPE If the JSON value is not an object. + * @returns INCORRECT_TYPE If the JSON value is of the given type. * @returns SUCCESS If the parse succeeded and the out parameter was set to the value. */ template @@ -36901,6 +37256,36 @@ public: */ simdjson_inline simdjson_result raw_json() noexcept; +#if SIMDJSON_SUPPORTS_DESERIALIZATION + /** + * Get this object as the given type. + * + * @param out This is set to a value of the given type, parsed from the JSON. If there is an error, this may not be initialized. + * @returns INCORRECT_TYPE If the JSON object is not of the given type. + * @returns SUCCESS If the parse succeeded and the out parameter was set to the value. + */ + template + simdjson_inline error_code get(T &out) + noexcept(custom_deserializable ? nothrow_custom_deserializable : true) { + static_assert(custom_deserializable); + return deserialize(*this, out); + } + /** + * Get this array as the given type. + * + * @returns A value of the given type, parsed from the JSON. + * @returns INCORRECT_TYPE If the JSON value is not the given type. + */ + template + simdjson_inline simdjson_result get() + noexcept(custom_deserializable ? nothrow_custom_deserializable : true) + { + static_assert(std::is_default_constructible::value, "The specified type is not default constructible."); + T out{}; + SIMDJSON_TRY(get(out)); + return out; + } +#endif // SIMDJSON_SUPPORTS_DESERIALIZATION protected: /** * Go to the end of the object, no matter where you are right now. @@ -36944,12 +37329,32 @@ public: simdjson_inline simdjson_result operator[](std::string_view key) && noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - inline simdjson_result reset() noexcept; inline simdjson_result is_empty() noexcept; inline simdjson_result count_fields() & noexcept; inline simdjson_result raw_json() noexcept; + #if SIMDJSON_SUPPORTS_DESERIALIZATION + // TODO: move this code into object-inl.h + template + simdjson_inline simdjson_result get() noexcept { + if (error()) { return error(); } + if constexpr (std::is_same_v) { + return first; + } + return first.get(); + } + template + simdjson_inline error_code get(T& out) noexcept { + if (error()) { return error(); } + if constexpr (std::is_same_v) { + out = first; + } else { + SIMDJSON_TRY( first.get(out) ); + } + return SUCCESS; + } +#endif // SIMDJSON_SUPPORTS_DESERIALIZATION }; } // namespace simdjson @@ -37154,12 +37559,17 @@ inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result #include +#if SIMDJSON_STATIC_REFLECTION +#include +// #include // for std::define_static_string - header not available yet +#endif namespace simdjson { template @@ -37206,6 +37616,22 @@ error_code tag_invoke(deserialize_tag, auto &val, T &out) noexcept { return SUCCESS; } +////////////////////////////// +// String deserialization +////////////////////////////// + +// just a character! +error_code tag_invoke(deserialize_tag, auto &val, char &out) noexcept { + std::string_view x; + SIMDJSON_TRY(val.get_string().get(x)); + if(x.size() != 1) { + return INCORRECT_TYPE; + } + out = x[0]; + return SUCCESS; +} + +// any string-like type (can be constructed from std::string_view) template requires(!require_custom_serialization) error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept(std::is_nothrow_constructible_v) { @@ -37227,15 +37653,20 @@ template requires(!require_custom_serialization) error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept(false) { using value_type = typename std::remove_cvref_t::value_type; - static_assert( + /*static_assert( deserializable, - "The specified type inside the container must itself be deserializable"); + "The specified type inside the container must itself be deserializable");*/ static_assert( std::is_default_constructible_v, "The specified type inside the container must default constructible."); arm64::ondemand::array arr; - SIMDJSON_TRY(val.get_array().get(arr)); + if constexpr (std::is_same_v, arm64::ondemand::array>) { + arr = val; + } else { + SIMDJSON_TRY(val.get_array().get(arr)); + } + for (auto v : arr) { if constexpr (concepts::returns_reference) { if (auto const err = v.get().get(concepts::emplace_one(out)); @@ -37292,7 +37723,45 @@ error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept(false) { return SUCCESS; } +template +error_code tag_invoke(deserialize_tag, arm64::ondemand::object &obj, T &out) noexcept { + using value_type = typename std::remove_cvref_t::mapped_type; + out.clear(); + for (auto field : obj) { + std::string_view key; + SIMDJSON_TRY(field.unescaped_key().get(key)); + + arm64::ondemand::value value_obj; + SIMDJSON_TRY(field.value().get(value_obj)); + + value_type this_value; + SIMDJSON_TRY(value_obj.get(this_value)); + out.emplace(typename T::key_type(key), std::move(this_value)); + } + return SUCCESS; +} + +template +error_code tag_invoke(deserialize_tag, arm64::ondemand::value &val, T &out) noexcept { + arm64::ondemand::object obj; + SIMDJSON_TRY(val.get_object().get(obj)); + return simdjson::deserialize(obj, out); +} + +template +error_code tag_invoke(deserialize_tag, arm64::ondemand::document &doc, T &out) noexcept { + arm64::ondemand::object obj; + SIMDJSON_TRY(doc.get_object().get(obj)); + return simdjson::deserialize(obj, out); +} + +template +error_code tag_invoke(deserialize_tag, arm64::ondemand::document_reference &doc, T &out) noexcept { + arm64::ondemand::object obj; + SIMDJSON_TRY(doc.get_object().get(obj)); + return simdjson::deserialize(obj, out); +} /** @@ -37354,6 +37823,199 @@ error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept(nothrow_deser return SUCCESS; } + +#if SIMDJSON_STATIC_REFLECTION + + +template +constexpr bool user_defined_type = (std::is_class_v +&& !std::is_same_v && !std::is_same_v && !concepts::optional_type && +!concepts::appendable_containers && !require_custom_serialization); + + +// workaround from +// https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2024/p2996r10.html#back-and-forth +// for missing expansion statements +namespace __impl { + template + struct replicator_type { + template + constexpr void operator>>(F body) const { + (body.template operator()(), ...); + } + }; + + template + replicator_type replicator = {}; +} + +template +consteval auto expand(R range) { + std::vector args; + for (auto r : range) { + args.push_back(reflect_value(r)); + } + return substitute(^^__impl::replicator, args); +} +// end of workaround + +template + requires(user_defined_type && std::is_class_v) +error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept { + arm64::ondemand::object obj; + if constexpr (std::is_same_v, arm64::ondemand::object>) { + obj = val; + } else { + SIMDJSON_TRY(val.get_object().get(obj)); + } + error_code e = simdjson::SUCCESS; + + [:expand(std::meta::nonstatic_data_members_of(^^T, std::meta::access_context::unchecked())):] >> [&]() { + if constexpr (!std::meta::is_const(mem) && std::meta::is_public(mem)) { + constexpr std::string_view key = std::define_static_string(std::meta::identifier_of(mem)); + static_assert( + deserializable, + "The specified type inside the class must itself be deserializable"); + // as long we are succesful or the field is not found, we continue + if(e == simdjson::SUCCESS || e == simdjson::NO_SUCH_FIELD) { + obj[key].get(out.[:mem:]); + } + } + }; + return e; +} +template + requires(user_defined_type>) +error_code tag_invoke(deserialize_tag, simdjson_value &val, std::unique_ptr &out) noexcept { + if (!out) { + out = std::make_unique(); + if (!out) { + return MEMALLOC; + } + } + if (auto err = val.get(*out)) { + out.reset(); + return err; + } + return SUCCESS; +} + +template + requires(user_defined_type>) +error_code tag_invoke(deserialize_tag, simdjson_value &val, std::shared_ptr &out) noexcept { + if (!out) { + out = std::make_shared(); + if (!out) { + return MEMALLOC; + } + } + if (auto err = val.get(*out)) { + out.reset(); + return err; + } + return SUCCESS; +} + +#endif // SIMDJSON_STATIC_REFLECTION + +//////////////////////////////////////// +// Unique pointers +//////////////////////////////////////// +error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) noexcept { + if (!out) { + out = std::make_unique(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_bool().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) noexcept { + if (!out) { + out = std::make_unique(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_int64().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) noexcept { + if (!out) { + out = std::make_unique(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_uint64().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) noexcept { + if (!out) { + out = std::make_unique(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_double().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) noexcept { + if (!out) { + out = std::make_unique(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_string().get(*out)); + return SUCCESS; +} + + +//////////////////////////////////////// +// Shared pointers +//////////////////////////////////////// +error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) noexcept { + if (!out) { + out = std::make_shared(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_bool().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) noexcept { + if (!out) { + out = std::make_shared(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_int64().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) noexcept { + if (!out) { + out = std::make_shared(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_uint64().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) noexcept { + if (!out) { + out = std::make_shared(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_double().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) noexcept { + if (!out) { + out = std::make_shared(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_string().get(*out)); + return SUCCESS; +} + + } // namespace simdjson #endif // SIMDJSON_ONDEMAND_DESERIALIZE_H @@ -40841,6 +41503,7 @@ simdjson_inline simdjson_result simdjson_result::simdjson_resul #endif // SIMDJSON_GENERIC_ONDEMAND_VALUE_ITERATOR_INL_H /* end file simdjson/generic/ondemand/value_iterator-inl.h for arm64 */ +// JSON builder, ideally they should not be part of the ondemand directory +// but it is convenient for now to have them here. +/* including simdjson/generic/ondemand/json_string_builder.h for arm64: #include "simdjson/generic/ondemand/json_string_builder.h" */ +/* begin file simdjson/generic/ondemand/json_string_builder.h for arm64 */ +/** + * This file is part of the builder API. It is temporarily in the ondemand directory + * but we will move it to a builder directory later. + */ +#ifndef SIMDJSON_GENERIC_STRING_BUILDER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { +namespace arm64 { +namespace builder { + +/** + * A builder for JSON strings representing documents. This is a low-level + * builder that is not meant to be used directly by end-users. Though it + * supports atomic types (Booleans, strings), it does not support composed + * types (arrays and objects). + * + * Ultimately, this class should support kernel-specific optimizations. E.g., + * it may make use of SIMD instructions to escape strings faster. + */ +class string_builder { +public: + simdjson_inline string_builder(size_t initial_capacity = 1024); + + /** + * Append number (includes Booleans). Booleans are mapped to the strings + * false and true. Numbers are converted to strings abiding by the JSON standard. + * Floating-point numbers are converted to the shortest string that 'correctly' + * represents the number. + */ + template::value>::type> + simdjson_inline void append(number_type v) noexcept; + + /** + * Append character c. + */ + simdjson_inline void append(char c) noexcept; + + /** + * Append the string 'null'. + */ + simdjson_inline void append_null() noexcept; + + /** + * Clear the content. + */ + simdjson_inline void clear() noexcept; + + /** + * Append the std::string_view, after escaping it. + * There is no UTF-8 validation. + */ + simdjson_inline void escape_and_append(std::string_view input) noexcept; + + /** + * Append the std::string_view surrounded by double quotes, after escaping it. + * There is no UTF-8 validation. + */ + simdjson_inline void escape_and_append_with_quotes(std::string_view input) noexcept; + + /** + * Append the character surrounded by double quotes, after escaping it. + * There is no UTF-8 validation. + */ + simdjson_inline void escape_and_append_with_quotes(char input) noexcept; + + /** + * Append the character surrounded by double quotes, after escaping it. + * There is no UTF-8 validation. + */ + simdjson_inline void escape_and_append_with_quotes(const char* input) noexcept; + + /** + * Append the C string directly, without escaping. + * There is no UTF-8 validation. + */ + simdjson_inline void append_raw(const char *c) noexcept; + + /** + * Append "{" to the buffer. + */ + simdjson_inline void start_object() noexcept; + + /** + * Append "}" to the buffer. + */ + simdjson_inline void end_object() noexcept; + + /** + * Append "[" to the buffer. + */ + simdjson_inline void start_array() noexcept; + + /** + * Append "]" to the buffer. + */ + simdjson_inline void end_array() noexcept; + + /** + * Append "," to the buffer. + */ + simdjson_inline void append_comma() noexcept; + + /** + * Append ":" to the buffer. + */ + simdjson_inline void append_colon() noexcept; + + /** + * Append a key-value pair to the buffer. + * The key is escaped and surrounded by double quotes. + * The value is escaped if it is a string. + */ + template + simdjson_inline void append_key_value(key_type key, value_type value) noexcept; + /** + * Append the std::string_view directly, without escaping. + * There is no UTF-8 validation. + */ + simdjson_inline void append_raw(std::string_view input) noexcept; + + /** + * Append len characters from str. + * There is no UTF-8 validation. + */ + simdjson_inline void append_raw(const char *str, size_t len) noexcept; +#if SIMDJSON_EXCEPTIONS + /** + * Creates an std::string from the written JSON buffer. + * Throws if memory allocation failed + * + * The result may not be valid UTF-8 if some of your content was not valid UTF-8. + * Use validate_unicode() to check the content if needed. + */ + simdjson_inline operator std::string() const noexcept(false); + + /** + * Creates an std::string_view from the written JSON buffer. + * Throws if memory allocation failed. + * + * The result may not be valid UTF-8 if some of your content was not valid UTF-8. + * Use validate_unicode() to check the content if needed. + */ + simdjson_inline operator std::string_view() const noexcept(false); +#endif + + /** + * Returns a view on the written JSON buffer. Returns an error + * if memory allocation failed. + * + * The result may not be valid UTF-8 if some of your content was not valid UTF-8. + * Use validate_unicode() to check the content. + */ + simdjson_inline simdjson_result view() const noexcept; + + /** + * Appends the null character to the buffer and returns + * a pointer to the beginning of the written JSON buffer. + * Returns an error if memory allocation failed. + * The result is null-terminated. + * + * The result may not be valid UTF-8 if some of your content was not valid UTF-8. + * Use validate_unicode() to check the content. + */ + simdjson_inline simdjson_result c_str() noexcept; + + /** + * Return true if the content is valid UTF-8. + */ + simdjson_inline bool validate_unicode() const noexcept; + + /** + * Returns the current size of the written JSON buffer. + * If an error occurred, returns 0. + */ + simdjson_inline size_t size() const noexcept; + +private: + /** + * Returns true if we can write at least upcoming_bytes bytes. + * The underlying buffer is reallocated if needed. It is designed + * to be called before writing to the buffer. It should be fast. + */ + simdjson_inline bool capacity_check(size_t upcoming_bytes); + + /** + * Grow the buffer to at least desired_capacity bytes. + * If the allocation fails, is_valid is set to false. We expect + * that this function would not be repeatedly called. + */ + simdjson_inline void grow_buffer(size_t desired_capacity); + + /** + * We use this helper function to make sure that is_valid is kept consistent. + */ + simdjson_inline void set_valid(bool valid) noexcept; + + std::unique_ptr buffer{}; + size_t position{0}; + size_t capacity{0}; + bool is_valid{true}; +}; + + + +} +} +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_STRING_BUILDER_H +/* end file simdjson/generic/ondemand/json_string_builder.h for arm64 */ +/* including simdjson/generic/ondemand/json_string_builder-inl.h for arm64: #include "simdjson/generic/ondemand/json_string_builder-inl.h" */ +/* begin file simdjson/generic/ondemand/json_string_builder-inl.h for arm64 */ +/** + * This file is part of the builder API. It is temporarily in the ondemand + * directory but we will move it to a builder directory later. + */ +#include +#include +#include +#ifndef SIMDJSON_GENERIC_STRING_BUILDER_INL_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_INL_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +/* + * Empirically, we have found that an inlined optimization is important for + * performance. The following macros are not ideal. We should find a better + * way to inline the code. + */ + +#if defined(__SSE2__) || defined(__x86_64__) || defined(__x86_64) || \ + (defined(_M_AMD64) || defined(_M_X64) || \ + (defined(_M_IX86_FP) && _M_IX86_FP == 2)) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_SSE2 +#define SIMDJSON_EXPERIMENTAL_HAS_SSE2 1 +#endif +#endif + +#if defined(__aarch64__) || defined(_M_ARM64) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_NEON +#define SIMDJSON_EXPERIMENTAL_HAS_NEON 1 +#endif +#endif +#if SIMDJSON_EXPERIMENTAL_HAS_NEON +#include +#endif +#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 +#include +#endif + +namespace simdjson { +namespace arm64 { +namespace builder { + +static SIMDJSON_CONSTEXPR_LAMBDA std::array json_quotable_character = { + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + +/** + +A possible SWAR implementation of has_json_escapable_byte. It is not used because +it is slower than the current implementation. It is kept here for reference (to show +that we tried it). + +inline bool has_json_escapable_byte(uint64_t x) { + uint64_t is_ascii = 0x8080808080808080ULL & ~x; + uint64_t xor2 = x ^ 0x0202020202020202ULL; + uint64_t lt32_or_eq34 = xor2 - 0x2121212121212121ULL; + uint64_t sub92 = x ^ 0x5C5C5C5C5C5C5C5CULL; + uint64_t eq92 = (sub92 - 0x0101010101010101ULL); + return ((lt32_or_eq34 | eq92) & is_ascii) != 0; +} + +**/ + +SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline bool +simple_needs_escaping(std::string_view v) { + for (char c : v) { + // a table lookup is faster than a series of comparisons + if(json_quotable_character[static_cast(c)]) { + return true; + } + } + return false; +} + +#if SIMDJSON_EXPERIMENTAL_HAS_NEON +simdjson_inline bool fast_needs_escaping(std::string_view view) { + if (view.size() < 16) { + return simple_needs_escaping(view); + } + size_t i = 0; + uint8x16_t running = vdupq_n_u8(0); + uint8x16_t v34 = vdupq_n_u8(34); + uint8x16_t v92 = vdupq_n_u8(92); + + for (; i + 15 < view.size(); i += 16) { + uint8x16_t word = vld1q_u8((const uint8_t *)view.data() + i); + running = vorrq_u8(running, vceqq_u8(word, v34)); + running = vorrq_u8(running, vceqq_u8(word, v92)); + running = vorrq_u8(running, vcltq_u8(word, vdupq_n_u8(32))); + } + if (i < view.size()) { + uint8x16_t word = + vld1q_u8((const uint8_t *)view.data() + view.length() - 16); + running = vorrq_u8(running, vceqq_u8(word, v34)); + running = vorrq_u8(running, vceqq_u8(word, v92)); + running = vorrq_u8(running, vcltq_u8(word, vdupq_n_u8(32))); + } + return vmaxvq_u32(vreinterpretq_u32_u8(running)) != 0; +} +#elif SIMDJSON_EXPERIMENTAL_HAS_SSE2 +simdjson_inline bool fast_needs_escaping(std::string_view view) { + if (view.size() < 16) { + return simple_needs_escaping(view); + } + size_t i = 0; + __m128i running = _mm_setzero_si128(); + for (; i + 15 < view.size(); i += 16) { + + __m128i word = _mm_loadu_si128(reinterpret_cast(view.data() + i)); + running = _mm_or_si128(running, _mm_cmpeq_epi8(word, _mm_set1_epi8(34))); + running = _mm_or_si128(running, _mm_cmpeq_epi8(word, _mm_set1_epi8(92))); + running = _mm_or_si128( + running, _mm_cmpeq_epi8(_mm_subs_epu8(word, _mm_set1_epi8(31)), + _mm_setzero_si128())); + } + if (i < view.size()) { + __m128i word = + _mm_loadu_si128(reinterpret_cast(view.data() + view.length() - 16)); + running = _mm_or_si128(running, _mm_cmpeq_epi8(word, _mm_set1_epi8(34))); + running = _mm_or_si128(running, _mm_cmpeq_epi8(word, _mm_set1_epi8(92))); + running = _mm_or_si128( + running, _mm_cmpeq_epi8(_mm_subs_epu8(word, _mm_set1_epi8(31)), + _mm_setzero_si128())); + } + return _mm_movemask_epi8(running) != 0; +} +#else +simdjson_inline bool fast_needs_escaping(std::string_view view) { + return simple_needs_escaping(view); +} +#endif + + +SIMDJSON_CONSTEXPR_LAMBDA inline size_t +find_next_json_quotable_character(const std::string_view view, + size_t location) noexcept { + + for (auto pos = view.begin() + location; pos != view.end(); ++pos) { + if (json_quotable_character[static_cast(*pos)]) { + return pos - view.begin(); + } + } + return size_t(view.size()); +} + +SIMDJSON_CONSTEXPR_LAMBDA static std::string_view control_chars[] = { + "\\x0000", "\\x0001", "\\x0002", "\\x0003", "\\x0004", "\\x0005", "\\x0006", + "\\x0007", "\\x0008", "\\t", "\\n", "\\x000b", "\\f", "\\r", + "\\x000e", "\\x000f", "\\x0010", "\\x0011", "\\x0012", "\\x0013", "\\x0014", + "\\x0015", "\\x0016", "\\x0017", "\\x0018", "\\x0019", "\\x001a", "\\x001b", + "\\x001c", "\\x001d", "\\x001e", "\\x001f"}; + +SIMDJSON_CONSTEXPR_LAMBDA void escape_json_char(char c, char *&out) { + if (c == '"') { + memcpy(out, "\\\"", 2); + out += 2; + } else if (c == '\\') { + memcpy(out, "\\\\", 2); + out += 2; + } else { + std::string_view v = control_chars[uint8_t(c)]; + memcpy(out, v.data(), v.size()); + out += v.size(); + } +} + +inline size_t write_string_escaped(const std::string_view input, char *out) { + size_t mysize = input.size(); + if (!fast_needs_escaping(input)) { // fast path! + memcpy(out, input.data(), input.size()); + return input.size(); + } + const char *const initout = out; + size_t location = find_next_json_quotable_character(input, 0); + memcpy(out, input.data(), location); + out += location; + escape_json_char(input[location], out); + location += 1; + while (location < mysize) { + size_t newlocation = find_next_json_quotable_character(input, location); + memcpy(out, input.data() + location, newlocation - location); + out += newlocation - location; + location = newlocation; + if (location == mysize) { + break; + } + escape_json_char(input[location], out); + location += 1; + } + return out - initout; +} + +#if SIMDJSON_CONSTEVAL +// unoptimized, meant for compile-time execution +consteval std::string consteval_to_quoted_escaped(std::string_view input) { + std::string out = "\""; + for (char c : input) { + if (json_quotable_character[uint8_t(c)]) { + if (c == '"') { + out.append("\\\""); + } else if (c == '\\') { + out.append("\\\\"); + } else { + std::string_view v = control_chars[uint8_t(c)]; + out.append(v); + } + } else { + out.push_back(c); + } + } + out.push_back('"'); + return out; +} +#endif // SIMDJSON_CONSTEVAL + +simdjson_inline string_builder::string_builder(size_t initial_capacity) + : buffer(new(std::nothrow) char[initial_capacity]), position(0), + capacity(buffer.get() != nullptr ? initial_capacity : 0), + is_valid(buffer.get() != nullptr) {} + +simdjson_inline bool string_builder::capacity_check(size_t upcoming_bytes) { + // We use the convention that when is_valid is false, then the capacity and + // the position are 0. + // Most of the time, this function will return true. + if (simdjson_likely(upcoming_bytes <= capacity - position)) { + return true; + } + // check for overflow, most of the time there is no overflow + if (simdjson_likely(position + upcoming_bytes < position)) { + return false; + } + // We will rarely get here. + grow_buffer((std::max)(capacity * 2, position + upcoming_bytes)); + // If the buffer allocation failed, we set is_valid to false. + return is_valid; +} + +simdjson_inline void string_builder::grow_buffer(size_t desired_capacity) { + if (!is_valid) { + return; + } + std::unique_ptr new_buffer(new (std::nothrow) char[desired_capacity]); + if (new_buffer.get() == nullptr) { + set_valid(false); + return; + } + std::memcpy(new_buffer.get(), buffer.get(), position); + buffer.swap(new_buffer); + capacity = desired_capacity; +} + +simdjson_inline void string_builder::set_valid(bool valid) noexcept { + if (!valid) { + is_valid = false; + capacity = 0; + position = 0; + buffer.reset(); + } else { + is_valid = true; + } +} + +simdjson_inline size_t string_builder::size() const noexcept { + return position; +} + +simdjson_inline void string_builder::append(char c) noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = c; + } +} + +simdjson_inline void string_builder::append_null() noexcept { + constexpr char null_literal[] = "null"; + constexpr size_t null_len = sizeof(null_literal) - 1; + if (capacity_check(null_len)) { + std::memcpy(buffer.get() + position, null_literal, null_len); + position += null_len; + } +} + +simdjson_inline void string_builder::clear() noexcept { + position = 0; + // if it was invalid, we should try to repair it + if (!is_valid) { + capacity = 0; + buffer.reset(); + is_valid = true; + } +} + +namespace internal { + +// We could specialize further for 32-bit integers. +int int_log2(uint32_t x) { return (63 - leading_zeroes(x | 1)); } + +int fast_digit_count(uint32_t x) { + static uint64_t table[] = { + 4294967296, 8589934582, 8589934582, 8589934582, 12884901788, + 12884901788, 12884901788, 17179868184, 17179868184, 17179868184, + 21474826480, 21474826480, 21474826480, 21474826480, 25769703776, + 25769703776, 25769703776, 30063771072, 30063771072, 30063771072, + 34349738368, 34349738368, 34349738368, 34349738368, 38554705664, + 38554705664, 38554705664, 41949672960, 41949672960, 41949672960, + 42949672960, 42949672960}; + return uint32_t((x + table[int_log2(x)]) >> 32); +} + +int int_log2(uint64_t x) { return 63 - leading_zeroes(x | 1); } + +int fast_digit_count(uint64_t x) { + static uint64_t table[] = {9, + 99, + 999, + 9999, + 99999, + 999999, + 9999999, + 99999999, + 999999999, + 9999999999, + 99999999999, + 999999999999, + 9999999999999, + 99999999999999, + 999999999999999ULL, + 9999999999999999ULL, + 99999999999999999ULL, + 999999999999999999ULL, + 9999999999999999999ULL}; + int y = (19 * int_log2(x) >> 6); + y += x > table[y]; + return y + 1; +} + +template ::value>::type> +simdjson_inline size_t digit_count(number_type v) noexcept { + static_assert(sizeof(number_type) == 8 || sizeof(number_type) == 4 || + sizeof(number_type) == 2 || sizeof(number_type) == 1, + "We only support 8-bit, 16-bit, 32-bit and 64-bit numbers"); + return fast_digit_count(v); +} +static const char decimal_table[200] = { + 0x30, 0x30, 0x30, 0x31, 0x30, 0x32, 0x30, 0x33, 0x30, 0x34, 0x30, 0x35, + 0x30, 0x36, 0x30, 0x37, 0x30, 0x38, 0x30, 0x39, 0x31, 0x30, 0x31, 0x31, + 0x31, 0x32, 0x31, 0x33, 0x31, 0x34, 0x31, 0x35, 0x31, 0x36, 0x31, 0x37, + 0x31, 0x38, 0x31, 0x39, 0x32, 0x30, 0x32, 0x31, 0x32, 0x32, 0x32, 0x33, + 0x32, 0x34, 0x32, 0x35, 0x32, 0x36, 0x32, 0x37, 0x32, 0x38, 0x32, 0x39, + 0x33, 0x30, 0x33, 0x31, 0x33, 0x32, 0x33, 0x33, 0x33, 0x34, 0x33, 0x35, + 0x33, 0x36, 0x33, 0x37, 0x33, 0x38, 0x33, 0x39, 0x34, 0x30, 0x34, 0x31, + 0x34, 0x32, 0x34, 0x33, 0x34, 0x34, 0x34, 0x35, 0x34, 0x36, 0x34, 0x37, + 0x34, 0x38, 0x34, 0x39, 0x35, 0x30, 0x35, 0x31, 0x35, 0x32, 0x35, 0x33, + 0x35, 0x34, 0x35, 0x35, 0x35, 0x36, 0x35, 0x37, 0x35, 0x38, 0x35, 0x39, + 0x36, 0x30, 0x36, 0x31, 0x36, 0x32, 0x36, 0x33, 0x36, 0x34, 0x36, 0x35, + 0x36, 0x36, 0x36, 0x37, 0x36, 0x38, 0x36, 0x39, 0x37, 0x30, 0x37, 0x31, + 0x37, 0x32, 0x37, 0x33, 0x37, 0x34, 0x37, 0x35, 0x37, 0x36, 0x37, 0x37, + 0x37, 0x38, 0x37, 0x39, 0x38, 0x30, 0x38, 0x31, 0x38, 0x32, 0x38, 0x33, + 0x38, 0x34, 0x38, 0x35, 0x38, 0x36, 0x38, 0x37, 0x38, 0x38, 0x38, 0x39, + 0x39, 0x30, 0x39, 0x31, 0x39, 0x32, 0x39, 0x33, 0x39, 0x34, 0x39, 0x35, + 0x39, 0x36, 0x39, 0x37, 0x39, 0x38, 0x39, 0x39, +}; +} // namespace internal + +template +simdjson_inline void string_builder::append(number_type v) noexcept { + static_assert(std::is_same::value || + std::is_integral::value || + std::is_floating_point::value, + "Unsupported number type"); + // If C++17 is available, we can 'if constexpr' here. + SIMDJSON_IF_CONSTEXPR(std::is_same::value) { + if (v) { + constexpr char true_literal[] = "true"; + constexpr size_t true_len = sizeof(true_literal) - 1; + if (capacity_check(true_len)) { + std::memcpy(buffer.get() + position, true_literal, true_len); + position += true_len; + } + } else { + constexpr char false_literal[] = "false"; + constexpr size_t false_len = sizeof(false_literal) - 1; + if (capacity_check(false_len)) { + std::memcpy(buffer.get() + position, false_literal, false_len); + position += false_len; + } + } + } + else SIMDJSON_IF_CONSTEXPR(std::is_unsigned::value) { + constexpr size_t max_number_size = 20; + if (capacity_check(max_number_size)) { + using unsigned_type = typename std::make_unsigned::type; + unsigned_type pv = static_cast(v); + size_t dc = internal::digit_count(pv); + char *write_pointer = buffer.get() + position + dc - 1; + while (pv >= 100) { + memcpy(write_pointer - 1, &internal::decimal_table[(pv % 100)*2], 2); + write_pointer -= 2; + pv /= 100; + } + if (pv >= 10) { + *write_pointer-- = char('0' + (pv % 10)); + pv /= 10; + } + *write_pointer = char('0' + pv); + position += dc; + } + } + else SIMDJSON_IF_CONSTEXPR(std::is_integral::value) { + constexpr size_t max_number_size = 20; + if (capacity_check(max_number_size)) { + using unsigned_type = typename std::make_unsigned::type; + bool negative = v < 0; + unsigned_type pv = static_cast(v); + if (negative) { + pv = 0 - pv; // the 0 is for Microsoft + } + size_t dc = internal::digit_count(pv); + if (negative) { + buffer.get()[position++] = '-'; + } + char *write_pointer = buffer.get() + position + dc - 1; + while (pv >= 100) { + memcpy(write_pointer - 1, &internal::decimal_table[(pv % 100)*2], 2); + write_pointer -= 2; + pv /= 100; + } + if (pv >= 10) { + *write_pointer-- = char('0' + (pv % 10)); + pv /= 10; + } + *write_pointer = char('0' + pv); + position += dc; + } + } + else SIMDJSON_IF_CONSTEXPR(std::is_floating_point::value) { + constexpr size_t max_number_size = 24; + if (capacity_check(max_number_size)) { + // We could specialize for float. + char *end = simdjson::internal::to_chars(buffer.get() + position, nullptr, + double(v)); + position = end - buffer.get(); + } + } +} + +simdjson_inline void +string_builder::escape_and_append(std::string_view input) noexcept { + // escaping might turn a control character into \x00xx so 6 characters. + if (capacity_check(6 * input.size())) { + position += write_string_escaped(input, buffer.get() + position); + } +} + +simdjson_inline void +string_builder::escape_and_append_with_quotes(std::string_view input) noexcept { + // escaping might turn a control character into \x00xx so 6 characters. + if (capacity_check(2 + 6 * input.size())) { + buffer.get()[position++] = '"'; + position += write_string_escaped(input, buffer.get() + position); + buffer.get()[position++] = '"'; + } +} + +simdjson_inline void +string_builder::escape_and_append_with_quotes(char input) noexcept { + // escaping might turn a control character into \x00xx so 6 characters. + if (capacity_check(2 + 6 * 1)) { + buffer.get()[position++] = '"'; + std::string_view cinput(&input, 1); + position += write_string_escaped(cinput, buffer.get() + position); + buffer.get()[position++] = '"'; + } +} + +simdjson_inline void string_builder::escape_and_append_with_quotes(const char* input) noexcept { + std::string_view cinput(input); + escape_and_append_with_quotes(cinput); +} + +simdjson_inline void string_builder::append_raw(const char *c) noexcept { + size_t len = std::strlen(c); + append_raw(c, len); +} + +simdjson_inline void +string_builder::append_raw(std::string_view input) noexcept { + if (capacity_check(input.size())) { + std::memcpy(buffer.get() + position, input.data(), input.size()); + position += input.size(); + } +} + +simdjson_inline void string_builder::append_raw(const char *str, + size_t len) noexcept { + if (capacity_check(len)) { + std::memcpy(buffer.get() + position, str, len); + position += len; + } +} + +#if SIMDJSON_EXCEPTIONS +simdjson_inline string_builder::operator std::string() const noexcept(false) { + return std::string(std::string_view()); +} + +simdjson_inline string_builder::operator std::string_view() const + noexcept(false) { + return view(); +} +#endif + +simdjson_inline simdjson_result +string_builder::view() const noexcept { + if (!is_valid) { + return simdjson::OUT_OF_CAPACITY; + } + return std::string_view(buffer.get(), position); +} + +simdjson_inline simdjson_result string_builder::c_str() noexcept { + if (capacity_check(1)) { + buffer.get()[position] = '\0'; + return buffer.get(); + } + return simdjson::OUT_OF_CAPACITY; +} + +simdjson_inline bool string_builder::validate_unicode() const noexcept { + return simdjson::validate_utf8(buffer.get(), position); +} + +simdjson_inline void string_builder::start_object() noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = '{'; + } +} + + +simdjson_inline void string_builder::end_object() noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = '}'; + } +} + + +simdjson_inline void string_builder::start_array() noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = '['; + } +} + + +simdjson_inline void string_builder::end_array() noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = ']'; + } +} + + +simdjson_inline void string_builder::append_comma() noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = ','; + } +} + + +simdjson_inline void string_builder::append_colon() noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = ':'; + } +} + +template +simdjson_inline void string_builder::append_key_value(key_type key, value_type value) noexcept { + static_assert( + std::is_arithmetic::value || + std::is_same::value || + std::is_same::value || + std::is_convertible::value || + std::is_same::value, + "Unsupported value type"); + static_assert( + std::is_same::value || + std::is_convertible::value, + "Unsupported key type"); + escape_and_append_with_quotes(key); + append_colon(); + SIMDJSON_IF_CONSTEXPR(std::is_same::value) { + append_null(); + } else SIMDJSON_IF_CONSTEXPR(std::is_same::value) { + escape_and_append_with_quotes(value); + } else SIMDJSON_IF_CONSTEXPR(std::is_convertible::value) { + escape_and_append_with_quotes(value); + } else SIMDJSON_IF_CONSTEXPR(std::is_same::value) { + escape_and_append_with_quotes(value); + } else { + append(value); + } +} + +} // namespace builder +} // namespace arm64 +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_STRING_BUILDER_INL_H +/* end file simdjson/generic/ondemand/json_string_builder-inl.h for arm64 */ +/* including simdjson/generic/ondemand/json_builder.h for arm64: #include "simdjson/generic/ondemand/json_builder.h" */ +/* begin file simdjson/generic/ondemand/json_builder.h for arm64 */ +/** + * This file is part of the builder API. It is temporarily in the ondemand directory + * but we will move it to a builder directory later. + */ +#ifndef SIMDJSON_GENERIC_BUILDER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/concepts.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ +#if SIMDJSON_STATIC_REFLECTION + +#include +#include +#include +#include +#include +#include +// #include // for std::define_static_string - header not available yet + +namespace simdjson { +namespace arm64 { +namespace builder { + +// Concept that checks if a type is a container but not a string (because +// strings handling must be handled differently) +template +concept container_but_not_string = + requires(T a) { + { a.size() } -> std::convertible_to; + { + a[std::declval()] + }; // check if elements are accessible for the subscript operator + } && !std::is_same_v && + !std::is_same_v && !std::is_same_v; + +template + requires(container_but_not_string) +constexpr void atom(string_builder &b, const T &t) { + if (t.size() == 0) { + b.append_raw("[]"); + return; + } + b.append('['); + atom(b, t[0]); + for (size_t i = 1; i < t.size(); ++i) { + b.append(','); + atom(b, t[i]); + } + b.append(']'); +} + +template + requires(std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v) +constexpr void atom(string_builder &b, const T &t) { + b.escape_and_append_with_quotes(t); +} + +template +constexpr void atom(string_builder &b, const T &m) { + if (m.empty()) { + b.append_raw("{}"); + return; + } + b.append('{'); + bool first = true; + for (const auto& [key, value] : m) { + if (!first) { + b.append(','); + } + first = false; + // Keys must be convertible to string_view per the concept + b.escape_and_append_with_quotes(key); + b.append(':'); + atom(b, value); + } + b.append('}'); +} + + +template::value && !std::is_same_v>::type> +constexpr void atom(string_builder &b, const number_type t) { + b.append(t); +} +#if SIMDJSON_CONSTEVAL +consteval std::string consteval_to_quoted_escaped(std::string_view input); +#endif + +template + requires(std::is_class_v && !container_but_not_string && + !concepts::string_view_keyed_map && + !std::is_same_v && + !std::is_same_v) +constexpr void atom(string_builder &b, const T &t) { + int i = 0; + b.append('{'); + [:expand(std::meta::nonstatic_data_members_of(^^T, std::meta::access_context::unchecked())):] >> [&]() { + if (i != 0) + b.append(','); + constexpr auto key = std::define_static_string(consteval_to_quoted_escaped(std::meta::identifier_of(dm))); + b.append_raw(key); + b.append(':'); + atom(b, t.[:dm:]); + i++; + }; + b.append('}'); +} + +// works for struct +template void append(string_builder &b, const Z &z) { + int i = 0; + b.append('{'); + [:expand(std::meta::nonstatic_data_members_of(^^Z, std::meta::access_context::unchecked())):] >> [&]() { + if (i != 0) + b.append(','); + constexpr auto key = std::define_static_string(consteval_to_quoted_escaped(std::meta::identifier_of(dm))); + b.append_raw(key); + b.append(':'); + atom(b, z.[:dm:]); + i++; + }; + b.append('}'); +} + +// works for container +template + requires(container_but_not_string) +void append(string_builder &b, const Z &z) { + if (z.size() == 0) { + b.append_raw("[]"); + return; + } + b.append('['); + atom(b, z[0]); + for (size_t i = 1; i < z.size(); ++i) { + b.append(','); + atom(b, z[i]); + } + b.append(']'); +} + +template +simdjson_result to_json_string(const Z &z) { + string_builder b; + append(b, z); + std::string_view s; + if(auto e = b.view().get(s); e) { return e; } + return std::string(s); +} + +template +simdjson_error to_json(const Z &z, std::string &s) { + string_builder b; + append(b, z); + std::string_view view; + if(auto e = b.view().get(view); e) { return e; } + s.assign(view); + return SUCCESS; +} +} // namespace json_builder +} // namespace arm64 +} // namespace simdjson +#endif // SIMDJSON_STATIC_REFLECTION + +#endif +/* end file simdjson/generic/ondemand/json_builder.h for arm64 */ /* end file simdjson/generic/ondemand/amalgamated.h for arm64 */ /* including simdjson/arm64/end.h: #include "simdjson/arm64/end.h" */ @@ -42968,6 +44644,24 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin return { src[0] }; } + +struct escaping { + static constexpr uint32_t BYTES_PROCESSED = 1; + simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst); + + simdjson_inline bool has_escape() { return escape_bits; } + simdjson_inline int escape_index() { return 0; } + + bool escape_bits; +}; // struct escaping + + + +simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) { + dst[0] = src[0]; + return { (src[0] == '\\') || (src[0] == '"') || (src[0] < 32) }; +} + } // unnamed namespace } // namespace fallback } // namespace simdjson @@ -43213,10 +44907,26 @@ concept nothrow_deserializable = nothrow_custom_deserializable || is_bu /// Deserialize Tag inline constexpr struct deserialize_tag { + using array_type = fallback::ondemand::array; + using object_type = fallback::ondemand::object; using value_type = fallback::ondemand::value; using document_type = fallback::ondemand::document; using document_reference_type = fallback::ondemand::document_reference; + // Customization Point for array + template + requires custom_deserializable + [[nodiscard]] constexpr /* error_code */ auto operator()(array_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + return tag_invoke(*this, object, output); + } + + // Customization Point for object + template + requires custom_deserializable + [[nodiscard]] constexpr /* error_code */ auto operator()(object_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + return tag_invoke(*this, object, output); + } + // Customization Point for value template requires custom_deserializable @@ -43784,6 +45494,7 @@ public: * * You may use get_double(), get_bool(), get_uint64(), get_int64(), * get_object(), get_array(), get_raw_json_string(), or get_string() instead. + * When SIMDJSON_SUPPORTS_DESERIALIZATION is set, custom types are also supported. * * @returns A value of the given type, parsed from the JSON. * @returns INCORRECT_TYPE If the JSON value is not the given type. @@ -43807,6 +45518,7 @@ public: * Get this value as the given type. * * Supported types: object, array, raw_json_string, string_view, uint64_t, int64_t, double, bool + * If the macro SIMDJSON_SUPPORTS_DESERIALIZATION is set, then custom types are also supported. * * @param out This is set to a value of the given type, parsed from the JSON. If there is an error, this may not be initialized. * @returns INCORRECT_TYPE If the JSON value is not an object. @@ -46049,6 +47761,37 @@ public: * - INDEX_OUT_OF_BOUNDS if the array index is larger than an array length */ simdjson_inline simdjson_result at(size_t index) noexcept; + +#if SIMDJSON_SUPPORTS_DESERIALIZATION + /** + * Get this array as the given type. + * + * @param out This is set to a value of the given type, parsed from the JSON. If there is an error, this may not be initialized. + * @returns INCORRECT_TYPE If the JSON array is not of the given type. + * @returns SUCCESS If the parse succeeded and the out parameter was set to the value. + */ + template + simdjson_inline error_code get(T &out) + noexcept(custom_deserializable ? nothrow_custom_deserializable : true) { + static_assert(custom_deserializable); + return deserialize(*this, out); + } + /** + * Get this array as the given type. + * + * @returns A value of the given type, parsed from the JSON. + * @returns INCORRECT_TYPE If the JSON value is not the given type. + */ + template + simdjson_inline simdjson_result get() + noexcept(custom_deserializable ? nothrow_custom_deserializable : true) + { + static_assert(std::is_default_constructible::value, "The specified type is not default constructible."); + T out{}; + SIMDJSON_TRY(get(out)); + return out; + } +#endif // SIMDJSON_SUPPORTS_DESERIALIZATION protected: /** * Go to the end of the array, no matter where you are right now. @@ -46127,7 +47870,28 @@ public: simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; simdjson_inline simdjson_result raw_json() noexcept; +#if SIMDJSON_SUPPORTS_DESERIALIZATION + // TODO: move this code into object-inl.h + template + simdjson_inline simdjson_result get() noexcept { + if (error()) { return error(); } + if constexpr (std::is_same_v) { + return first; + } + return first.get(); + } + template + simdjson_inline error_code get(T& out) noexcept { + if (error()) { return error(); } + if constexpr (std::is_same_v) { + out = first; + } else { + SIMDJSON_TRY( first.get(out) ); + } + return SUCCESS; + } +#endif // SIMDJSON_SUPPORTS_DESERIALIZATION }; } // namespace simdjson @@ -46461,7 +48225,7 @@ public: * Be mindful that the document instance must remain in scope while you are accessing object, array and value instances. * * @param out This is set to a value of the given type, parsed from the JSON. If there is an error, this may not be initialized. - * @returns INCORRECT_TYPE If the JSON value is not an object. + * @returns INCORRECT_TYPE If the JSON value is of the given type. * @returns SUCCESS If the parse succeeded and the out parameter was set to the value. */ template @@ -47948,6 +49712,36 @@ public: */ simdjson_inline simdjson_result raw_json() noexcept; +#if SIMDJSON_SUPPORTS_DESERIALIZATION + /** + * Get this object as the given type. + * + * @param out This is set to a value of the given type, parsed from the JSON. If there is an error, this may not be initialized. + * @returns INCORRECT_TYPE If the JSON object is not of the given type. + * @returns SUCCESS If the parse succeeded and the out parameter was set to the value. + */ + template + simdjson_inline error_code get(T &out) + noexcept(custom_deserializable ? nothrow_custom_deserializable : true) { + static_assert(custom_deserializable); + return deserialize(*this, out); + } + /** + * Get this array as the given type. + * + * @returns A value of the given type, parsed from the JSON. + * @returns INCORRECT_TYPE If the JSON value is not the given type. + */ + template + simdjson_inline simdjson_result get() + noexcept(custom_deserializable ? nothrow_custom_deserializable : true) + { + static_assert(std::is_default_constructible::value, "The specified type is not default constructible."); + T out{}; + SIMDJSON_TRY(get(out)); + return out; + } +#endif // SIMDJSON_SUPPORTS_DESERIALIZATION protected: /** * Go to the end of the object, no matter where you are right now. @@ -47991,12 +49785,32 @@ public: simdjson_inline simdjson_result operator[](std::string_view key) && noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - inline simdjson_result reset() noexcept; inline simdjson_result is_empty() noexcept; inline simdjson_result count_fields() & noexcept; inline simdjson_result raw_json() noexcept; + #if SIMDJSON_SUPPORTS_DESERIALIZATION + // TODO: move this code into object-inl.h + template + simdjson_inline simdjson_result get() noexcept { + if (error()) { return error(); } + if constexpr (std::is_same_v) { + return first; + } + return first.get(); + } + template + simdjson_inline error_code get(T& out) noexcept { + if (error()) { return error(); } + if constexpr (std::is_same_v) { + out = first; + } else { + SIMDJSON_TRY( first.get(out) ); + } + return SUCCESS; + } +#endif // SIMDJSON_SUPPORTS_DESERIALIZATION }; } // namespace simdjson @@ -48201,12 +50015,17 @@ inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result #include +#if SIMDJSON_STATIC_REFLECTION +#include +// #include // for std::define_static_string - header not available yet +#endif namespace simdjson { template @@ -48253,6 +50072,22 @@ error_code tag_invoke(deserialize_tag, auto &val, T &out) noexcept { return SUCCESS; } +////////////////////////////// +// String deserialization +////////////////////////////// + +// just a character! +error_code tag_invoke(deserialize_tag, auto &val, char &out) noexcept { + std::string_view x; + SIMDJSON_TRY(val.get_string().get(x)); + if(x.size() != 1) { + return INCORRECT_TYPE; + } + out = x[0]; + return SUCCESS; +} + +// any string-like type (can be constructed from std::string_view) template requires(!require_custom_serialization) error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept(std::is_nothrow_constructible_v) { @@ -48274,15 +50109,20 @@ template requires(!require_custom_serialization) error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept(false) { using value_type = typename std::remove_cvref_t::value_type; - static_assert( + /*static_assert( deserializable, - "The specified type inside the container must itself be deserializable"); + "The specified type inside the container must itself be deserializable");*/ static_assert( std::is_default_constructible_v, "The specified type inside the container must default constructible."); fallback::ondemand::array arr; - SIMDJSON_TRY(val.get_array().get(arr)); + if constexpr (std::is_same_v, fallback::ondemand::array>) { + arr = val; + } else { + SIMDJSON_TRY(val.get_array().get(arr)); + } + for (auto v : arr) { if constexpr (concepts::returns_reference) { if (auto const err = v.get().get(concepts::emplace_one(out)); @@ -48339,7 +50179,45 @@ error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept(false) { return SUCCESS; } +template +error_code tag_invoke(deserialize_tag, fallback::ondemand::object &obj, T &out) noexcept { + using value_type = typename std::remove_cvref_t::mapped_type; + out.clear(); + for (auto field : obj) { + std::string_view key; + SIMDJSON_TRY(field.unescaped_key().get(key)); + + fallback::ondemand::value value_obj; + SIMDJSON_TRY(field.value().get(value_obj)); + + value_type this_value; + SIMDJSON_TRY(value_obj.get(this_value)); + out.emplace(typename T::key_type(key), std::move(this_value)); + } + return SUCCESS; +} + +template +error_code tag_invoke(deserialize_tag, fallback::ondemand::value &val, T &out) noexcept { + fallback::ondemand::object obj; + SIMDJSON_TRY(val.get_object().get(obj)); + return simdjson::deserialize(obj, out); +} + +template +error_code tag_invoke(deserialize_tag, fallback::ondemand::document &doc, T &out) noexcept { + fallback::ondemand::object obj; + SIMDJSON_TRY(doc.get_object().get(obj)); + return simdjson::deserialize(obj, out); +} + +template +error_code tag_invoke(deserialize_tag, fallback::ondemand::document_reference &doc, T &out) noexcept { + fallback::ondemand::object obj; + SIMDJSON_TRY(doc.get_object().get(obj)); + return simdjson::deserialize(obj, out); +} /** @@ -48401,6 +50279,199 @@ error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept(nothrow_deser return SUCCESS; } + +#if SIMDJSON_STATIC_REFLECTION + + +template +constexpr bool user_defined_type = (std::is_class_v +&& !std::is_same_v && !std::is_same_v && !concepts::optional_type && +!concepts::appendable_containers && !require_custom_serialization); + + +// workaround from +// https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2024/p2996r10.html#back-and-forth +// for missing expansion statements +namespace __impl { + template + struct replicator_type { + template + constexpr void operator>>(F body) const { + (body.template operator()(), ...); + } + }; + + template + replicator_type replicator = {}; +} + +template +consteval auto expand(R range) { + std::vector args; + for (auto r : range) { + args.push_back(reflect_value(r)); + } + return substitute(^^__impl::replicator, args); +} +// end of workaround + +template + requires(user_defined_type && std::is_class_v) +error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept { + fallback::ondemand::object obj; + if constexpr (std::is_same_v, fallback::ondemand::object>) { + obj = val; + } else { + SIMDJSON_TRY(val.get_object().get(obj)); + } + error_code e = simdjson::SUCCESS; + + [:expand(std::meta::nonstatic_data_members_of(^^T, std::meta::access_context::unchecked())):] >> [&]() { + if constexpr (!std::meta::is_const(mem) && std::meta::is_public(mem)) { + constexpr std::string_view key = std::define_static_string(std::meta::identifier_of(mem)); + static_assert( + deserializable, + "The specified type inside the class must itself be deserializable"); + // as long we are succesful or the field is not found, we continue + if(e == simdjson::SUCCESS || e == simdjson::NO_SUCH_FIELD) { + obj[key].get(out.[:mem:]); + } + } + }; + return e; +} +template + requires(user_defined_type>) +error_code tag_invoke(deserialize_tag, simdjson_value &val, std::unique_ptr &out) noexcept { + if (!out) { + out = std::make_unique(); + if (!out) { + return MEMALLOC; + } + } + if (auto err = val.get(*out)) { + out.reset(); + return err; + } + return SUCCESS; +} + +template + requires(user_defined_type>) +error_code tag_invoke(deserialize_tag, simdjson_value &val, std::shared_ptr &out) noexcept { + if (!out) { + out = std::make_shared(); + if (!out) { + return MEMALLOC; + } + } + if (auto err = val.get(*out)) { + out.reset(); + return err; + } + return SUCCESS; +} + +#endif // SIMDJSON_STATIC_REFLECTION + +//////////////////////////////////////// +// Unique pointers +//////////////////////////////////////// +error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) noexcept { + if (!out) { + out = std::make_unique(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_bool().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) noexcept { + if (!out) { + out = std::make_unique(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_int64().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) noexcept { + if (!out) { + out = std::make_unique(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_uint64().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) noexcept { + if (!out) { + out = std::make_unique(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_double().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) noexcept { + if (!out) { + out = std::make_unique(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_string().get(*out)); + return SUCCESS; +} + + +//////////////////////////////////////// +// Shared pointers +//////////////////////////////////////// +error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) noexcept { + if (!out) { + out = std::make_shared(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_bool().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) noexcept { + if (!out) { + out = std::make_shared(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_int64().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) noexcept { + if (!out) { + out = std::make_shared(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_uint64().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) noexcept { + if (!out) { + out = std::make_shared(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_double().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) noexcept { + if (!out) { + out = std::make_shared(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_string().get(*out)); + return SUCCESS; +} + + } // namespace simdjson #endif // SIMDJSON_ONDEMAND_DESERIALIZE_H @@ -51888,6 +53959,7 @@ simdjson_inline simdjson_result simdjson_result::simdjson_re #endif // SIMDJSON_GENERIC_ONDEMAND_VALUE_ITERATOR_INL_H /* end file simdjson/generic/ondemand/value_iterator-inl.h for fallback */ +// JSON builder, ideally they should not be part of the ondemand directory +// but it is convenient for now to have them here. +/* including simdjson/generic/ondemand/json_string_builder.h for fallback: #include "simdjson/generic/ondemand/json_string_builder.h" */ +/* begin file simdjson/generic/ondemand/json_string_builder.h for fallback */ +/** + * This file is part of the builder API. It is temporarily in the ondemand directory + * but we will move it to a builder directory later. + */ +#ifndef SIMDJSON_GENERIC_STRING_BUILDER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { +namespace fallback { +namespace builder { + +/** + * A builder for JSON strings representing documents. This is a low-level + * builder that is not meant to be used directly by end-users. Though it + * supports atomic types (Booleans, strings), it does not support composed + * types (arrays and objects). + * + * Ultimately, this class should support kernel-specific optimizations. E.g., + * it may make use of SIMD instructions to escape strings faster. + */ +class string_builder { +public: + simdjson_inline string_builder(size_t initial_capacity = 1024); + + /** + * Append number (includes Booleans). Booleans are mapped to the strings + * false and true. Numbers are converted to strings abiding by the JSON standard. + * Floating-point numbers are converted to the shortest string that 'correctly' + * represents the number. + */ + template::value>::type> + simdjson_inline void append(number_type v) noexcept; + + /** + * Append character c. + */ + simdjson_inline void append(char c) noexcept; + + /** + * Append the string 'null'. + */ + simdjson_inline void append_null() noexcept; + + /** + * Clear the content. + */ + simdjson_inline void clear() noexcept; + + /** + * Append the std::string_view, after escaping it. + * There is no UTF-8 validation. + */ + simdjson_inline void escape_and_append(std::string_view input) noexcept; + + /** + * Append the std::string_view surrounded by double quotes, after escaping it. + * There is no UTF-8 validation. + */ + simdjson_inline void escape_and_append_with_quotes(std::string_view input) noexcept; + + /** + * Append the character surrounded by double quotes, after escaping it. + * There is no UTF-8 validation. + */ + simdjson_inline void escape_and_append_with_quotes(char input) noexcept; + + /** + * Append the character surrounded by double quotes, after escaping it. + * There is no UTF-8 validation. + */ + simdjson_inline void escape_and_append_with_quotes(const char* input) noexcept; + + /** + * Append the C string directly, without escaping. + * There is no UTF-8 validation. + */ + simdjson_inline void append_raw(const char *c) noexcept; + + /** + * Append "{" to the buffer. + */ + simdjson_inline void start_object() noexcept; + + /** + * Append "}" to the buffer. + */ + simdjson_inline void end_object() noexcept; + + /** + * Append "[" to the buffer. + */ + simdjson_inline void start_array() noexcept; + + /** + * Append "]" to the buffer. + */ + simdjson_inline void end_array() noexcept; + + /** + * Append "," to the buffer. + */ + simdjson_inline void append_comma() noexcept; + + /** + * Append ":" to the buffer. + */ + simdjson_inline void append_colon() noexcept; + + /** + * Append a key-value pair to the buffer. + * The key is escaped and surrounded by double quotes. + * The value is escaped if it is a string. + */ + template + simdjson_inline void append_key_value(key_type key, value_type value) noexcept; + /** + * Append the std::string_view directly, without escaping. + * There is no UTF-8 validation. + */ + simdjson_inline void append_raw(std::string_view input) noexcept; + + /** + * Append len characters from str. + * There is no UTF-8 validation. + */ + simdjson_inline void append_raw(const char *str, size_t len) noexcept; +#if SIMDJSON_EXCEPTIONS + /** + * Creates an std::string from the written JSON buffer. + * Throws if memory allocation failed + * + * The result may not be valid UTF-8 if some of your content was not valid UTF-8. + * Use validate_unicode() to check the content if needed. + */ + simdjson_inline operator std::string() const noexcept(false); + + /** + * Creates an std::string_view from the written JSON buffer. + * Throws if memory allocation failed. + * + * The result may not be valid UTF-8 if some of your content was not valid UTF-8. + * Use validate_unicode() to check the content if needed. + */ + simdjson_inline operator std::string_view() const noexcept(false); +#endif + + /** + * Returns a view on the written JSON buffer. Returns an error + * if memory allocation failed. + * + * The result may not be valid UTF-8 if some of your content was not valid UTF-8. + * Use validate_unicode() to check the content. + */ + simdjson_inline simdjson_result view() const noexcept; + + /** + * Appends the null character to the buffer and returns + * a pointer to the beginning of the written JSON buffer. + * Returns an error if memory allocation failed. + * The result is null-terminated. + * + * The result may not be valid UTF-8 if some of your content was not valid UTF-8. + * Use validate_unicode() to check the content. + */ + simdjson_inline simdjson_result c_str() noexcept; + + /** + * Return true if the content is valid UTF-8. + */ + simdjson_inline bool validate_unicode() const noexcept; + + /** + * Returns the current size of the written JSON buffer. + * If an error occurred, returns 0. + */ + simdjson_inline size_t size() const noexcept; + +private: + /** + * Returns true if we can write at least upcoming_bytes bytes. + * The underlying buffer is reallocated if needed. It is designed + * to be called before writing to the buffer. It should be fast. + */ + simdjson_inline bool capacity_check(size_t upcoming_bytes); + + /** + * Grow the buffer to at least desired_capacity bytes. + * If the allocation fails, is_valid is set to false. We expect + * that this function would not be repeatedly called. + */ + simdjson_inline void grow_buffer(size_t desired_capacity); + + /** + * We use this helper function to make sure that is_valid is kept consistent. + */ + simdjson_inline void set_valid(bool valid) noexcept; + + std::unique_ptr buffer{}; + size_t position{0}; + size_t capacity{0}; + bool is_valid{true}; +}; + + + +} +} +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_STRING_BUILDER_H +/* end file simdjson/generic/ondemand/json_string_builder.h for fallback */ +/* including simdjson/generic/ondemand/json_string_builder-inl.h for fallback: #include "simdjson/generic/ondemand/json_string_builder-inl.h" */ +/* begin file simdjson/generic/ondemand/json_string_builder-inl.h for fallback */ +/** + * This file is part of the builder API. It is temporarily in the ondemand + * directory but we will move it to a builder directory later. + */ +#include +#include +#include +#ifndef SIMDJSON_GENERIC_STRING_BUILDER_INL_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_INL_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +/* + * Empirically, we have found that an inlined optimization is important for + * performance. The following macros are not ideal. We should find a better + * way to inline the code. + */ + +#if defined(__SSE2__) || defined(__x86_64__) || defined(__x86_64) || \ + (defined(_M_AMD64) || defined(_M_X64) || \ + (defined(_M_IX86_FP) && _M_IX86_FP == 2)) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_SSE2 +#define SIMDJSON_EXPERIMENTAL_HAS_SSE2 1 +#endif +#endif + +#if defined(__aarch64__) || defined(_M_ARM64) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_NEON +#define SIMDJSON_EXPERIMENTAL_HAS_NEON 1 +#endif +#endif +#if SIMDJSON_EXPERIMENTAL_HAS_NEON +#include +#endif +#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 +#include +#endif + +namespace simdjson { +namespace fallback { +namespace builder { + +static SIMDJSON_CONSTEXPR_LAMBDA std::array json_quotable_character = { + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + +/** + +A possible SWAR implementation of has_json_escapable_byte. It is not used because +it is slower than the current implementation. It is kept here for reference (to show +that we tried it). + +inline bool has_json_escapable_byte(uint64_t x) { + uint64_t is_ascii = 0x8080808080808080ULL & ~x; + uint64_t xor2 = x ^ 0x0202020202020202ULL; + uint64_t lt32_or_eq34 = xor2 - 0x2121212121212121ULL; + uint64_t sub92 = x ^ 0x5C5C5C5C5C5C5C5CULL; + uint64_t eq92 = (sub92 - 0x0101010101010101ULL); + return ((lt32_or_eq34 | eq92) & is_ascii) != 0; +} + +**/ + +SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline bool +simple_needs_escaping(std::string_view v) { + for (char c : v) { + // a table lookup is faster than a series of comparisons + if(json_quotable_character[static_cast(c)]) { + return true; + } + } + return false; +} + +#if SIMDJSON_EXPERIMENTAL_HAS_NEON +simdjson_inline bool fast_needs_escaping(std::string_view view) { + if (view.size() < 16) { + return simple_needs_escaping(view); + } + size_t i = 0; + uint8x16_t running = vdupq_n_u8(0); + uint8x16_t v34 = vdupq_n_u8(34); + uint8x16_t v92 = vdupq_n_u8(92); + + for (; i + 15 < view.size(); i += 16) { + uint8x16_t word = vld1q_u8((const uint8_t *)view.data() + i); + running = vorrq_u8(running, vceqq_u8(word, v34)); + running = vorrq_u8(running, vceqq_u8(word, v92)); + running = vorrq_u8(running, vcltq_u8(word, vdupq_n_u8(32))); + } + if (i < view.size()) { + uint8x16_t word = + vld1q_u8((const uint8_t *)view.data() + view.length() - 16); + running = vorrq_u8(running, vceqq_u8(word, v34)); + running = vorrq_u8(running, vceqq_u8(word, v92)); + running = vorrq_u8(running, vcltq_u8(word, vdupq_n_u8(32))); + } + return vmaxvq_u32(vreinterpretq_u32_u8(running)) != 0; +} +#elif SIMDJSON_EXPERIMENTAL_HAS_SSE2 +simdjson_inline bool fast_needs_escaping(std::string_view view) { + if (view.size() < 16) { + return simple_needs_escaping(view); + } + size_t i = 0; + __m128i running = _mm_setzero_si128(); + for (; i + 15 < view.size(); i += 16) { + + __m128i word = _mm_loadu_si128(reinterpret_cast(view.data() + i)); + running = _mm_or_si128(running, _mm_cmpeq_epi8(word, _mm_set1_epi8(34))); + running = _mm_or_si128(running, _mm_cmpeq_epi8(word, _mm_set1_epi8(92))); + running = _mm_or_si128( + running, _mm_cmpeq_epi8(_mm_subs_epu8(word, _mm_set1_epi8(31)), + _mm_setzero_si128())); + } + if (i < view.size()) { + __m128i word = + _mm_loadu_si128(reinterpret_cast(view.data() + view.length() - 16)); + running = _mm_or_si128(running, _mm_cmpeq_epi8(word, _mm_set1_epi8(34))); + running = _mm_or_si128(running, _mm_cmpeq_epi8(word, _mm_set1_epi8(92))); + running = _mm_or_si128( + running, _mm_cmpeq_epi8(_mm_subs_epu8(word, _mm_set1_epi8(31)), + _mm_setzero_si128())); + } + return _mm_movemask_epi8(running) != 0; +} +#else +simdjson_inline bool fast_needs_escaping(std::string_view view) { + return simple_needs_escaping(view); +} +#endif + + +SIMDJSON_CONSTEXPR_LAMBDA inline size_t +find_next_json_quotable_character(const std::string_view view, + size_t location) noexcept { + + for (auto pos = view.begin() + location; pos != view.end(); ++pos) { + if (json_quotable_character[static_cast(*pos)]) { + return pos - view.begin(); + } + } + return size_t(view.size()); +} + +SIMDJSON_CONSTEXPR_LAMBDA static std::string_view control_chars[] = { + "\\x0000", "\\x0001", "\\x0002", "\\x0003", "\\x0004", "\\x0005", "\\x0006", + "\\x0007", "\\x0008", "\\t", "\\n", "\\x000b", "\\f", "\\r", + "\\x000e", "\\x000f", "\\x0010", "\\x0011", "\\x0012", "\\x0013", "\\x0014", + "\\x0015", "\\x0016", "\\x0017", "\\x0018", "\\x0019", "\\x001a", "\\x001b", + "\\x001c", "\\x001d", "\\x001e", "\\x001f"}; + +SIMDJSON_CONSTEXPR_LAMBDA void escape_json_char(char c, char *&out) { + if (c == '"') { + memcpy(out, "\\\"", 2); + out += 2; + } else if (c == '\\') { + memcpy(out, "\\\\", 2); + out += 2; + } else { + std::string_view v = control_chars[uint8_t(c)]; + memcpy(out, v.data(), v.size()); + out += v.size(); + } +} + +inline size_t write_string_escaped(const std::string_view input, char *out) { + size_t mysize = input.size(); + if (!fast_needs_escaping(input)) { // fast path! + memcpy(out, input.data(), input.size()); + return input.size(); + } + const char *const initout = out; + size_t location = find_next_json_quotable_character(input, 0); + memcpy(out, input.data(), location); + out += location; + escape_json_char(input[location], out); + location += 1; + while (location < mysize) { + size_t newlocation = find_next_json_quotable_character(input, location); + memcpy(out, input.data() + location, newlocation - location); + out += newlocation - location; + location = newlocation; + if (location == mysize) { + break; + } + escape_json_char(input[location], out); + location += 1; + } + return out - initout; +} + +#if SIMDJSON_CONSTEVAL +// unoptimized, meant for compile-time execution +consteval std::string consteval_to_quoted_escaped(std::string_view input) { + std::string out = "\""; + for (char c : input) { + if (json_quotable_character[uint8_t(c)]) { + if (c == '"') { + out.append("\\\""); + } else if (c == '\\') { + out.append("\\\\"); + } else { + std::string_view v = control_chars[uint8_t(c)]; + out.append(v); + } + } else { + out.push_back(c); + } + } + out.push_back('"'); + return out; +} +#endif // SIMDJSON_CONSTEVAL + +simdjson_inline string_builder::string_builder(size_t initial_capacity) + : buffer(new(std::nothrow) char[initial_capacity]), position(0), + capacity(buffer.get() != nullptr ? initial_capacity : 0), + is_valid(buffer.get() != nullptr) {} + +simdjson_inline bool string_builder::capacity_check(size_t upcoming_bytes) { + // We use the convention that when is_valid is false, then the capacity and + // the position are 0. + // Most of the time, this function will return true. + if (simdjson_likely(upcoming_bytes <= capacity - position)) { + return true; + } + // check for overflow, most of the time there is no overflow + if (simdjson_likely(position + upcoming_bytes < position)) { + return false; + } + // We will rarely get here. + grow_buffer((std::max)(capacity * 2, position + upcoming_bytes)); + // If the buffer allocation failed, we set is_valid to false. + return is_valid; +} + +simdjson_inline void string_builder::grow_buffer(size_t desired_capacity) { + if (!is_valid) { + return; + } + std::unique_ptr new_buffer(new (std::nothrow) char[desired_capacity]); + if (new_buffer.get() == nullptr) { + set_valid(false); + return; + } + std::memcpy(new_buffer.get(), buffer.get(), position); + buffer.swap(new_buffer); + capacity = desired_capacity; +} + +simdjson_inline void string_builder::set_valid(bool valid) noexcept { + if (!valid) { + is_valid = false; + capacity = 0; + position = 0; + buffer.reset(); + } else { + is_valid = true; + } +} + +simdjson_inline size_t string_builder::size() const noexcept { + return position; +} + +simdjson_inline void string_builder::append(char c) noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = c; + } +} + +simdjson_inline void string_builder::append_null() noexcept { + constexpr char null_literal[] = "null"; + constexpr size_t null_len = sizeof(null_literal) - 1; + if (capacity_check(null_len)) { + std::memcpy(buffer.get() + position, null_literal, null_len); + position += null_len; + } +} + +simdjson_inline void string_builder::clear() noexcept { + position = 0; + // if it was invalid, we should try to repair it + if (!is_valid) { + capacity = 0; + buffer.reset(); + is_valid = true; + } +} + +namespace internal { + +// We could specialize further for 32-bit integers. +int int_log2(uint32_t x) { return (63 - leading_zeroes(x | 1)); } + +int fast_digit_count(uint32_t x) { + static uint64_t table[] = { + 4294967296, 8589934582, 8589934582, 8589934582, 12884901788, + 12884901788, 12884901788, 17179868184, 17179868184, 17179868184, + 21474826480, 21474826480, 21474826480, 21474826480, 25769703776, + 25769703776, 25769703776, 30063771072, 30063771072, 30063771072, + 34349738368, 34349738368, 34349738368, 34349738368, 38554705664, + 38554705664, 38554705664, 41949672960, 41949672960, 41949672960, + 42949672960, 42949672960}; + return uint32_t((x + table[int_log2(x)]) >> 32); +} + +int int_log2(uint64_t x) { return 63 - leading_zeroes(x | 1); } + +int fast_digit_count(uint64_t x) { + static uint64_t table[] = {9, + 99, + 999, + 9999, + 99999, + 999999, + 9999999, + 99999999, + 999999999, + 9999999999, + 99999999999, + 999999999999, + 9999999999999, + 99999999999999, + 999999999999999ULL, + 9999999999999999ULL, + 99999999999999999ULL, + 999999999999999999ULL, + 9999999999999999999ULL}; + int y = (19 * int_log2(x) >> 6); + y += x > table[y]; + return y + 1; +} + +template ::value>::type> +simdjson_inline size_t digit_count(number_type v) noexcept { + static_assert(sizeof(number_type) == 8 || sizeof(number_type) == 4 || + sizeof(number_type) == 2 || sizeof(number_type) == 1, + "We only support 8-bit, 16-bit, 32-bit and 64-bit numbers"); + return fast_digit_count(v); +} +static const char decimal_table[200] = { + 0x30, 0x30, 0x30, 0x31, 0x30, 0x32, 0x30, 0x33, 0x30, 0x34, 0x30, 0x35, + 0x30, 0x36, 0x30, 0x37, 0x30, 0x38, 0x30, 0x39, 0x31, 0x30, 0x31, 0x31, + 0x31, 0x32, 0x31, 0x33, 0x31, 0x34, 0x31, 0x35, 0x31, 0x36, 0x31, 0x37, + 0x31, 0x38, 0x31, 0x39, 0x32, 0x30, 0x32, 0x31, 0x32, 0x32, 0x32, 0x33, + 0x32, 0x34, 0x32, 0x35, 0x32, 0x36, 0x32, 0x37, 0x32, 0x38, 0x32, 0x39, + 0x33, 0x30, 0x33, 0x31, 0x33, 0x32, 0x33, 0x33, 0x33, 0x34, 0x33, 0x35, + 0x33, 0x36, 0x33, 0x37, 0x33, 0x38, 0x33, 0x39, 0x34, 0x30, 0x34, 0x31, + 0x34, 0x32, 0x34, 0x33, 0x34, 0x34, 0x34, 0x35, 0x34, 0x36, 0x34, 0x37, + 0x34, 0x38, 0x34, 0x39, 0x35, 0x30, 0x35, 0x31, 0x35, 0x32, 0x35, 0x33, + 0x35, 0x34, 0x35, 0x35, 0x35, 0x36, 0x35, 0x37, 0x35, 0x38, 0x35, 0x39, + 0x36, 0x30, 0x36, 0x31, 0x36, 0x32, 0x36, 0x33, 0x36, 0x34, 0x36, 0x35, + 0x36, 0x36, 0x36, 0x37, 0x36, 0x38, 0x36, 0x39, 0x37, 0x30, 0x37, 0x31, + 0x37, 0x32, 0x37, 0x33, 0x37, 0x34, 0x37, 0x35, 0x37, 0x36, 0x37, 0x37, + 0x37, 0x38, 0x37, 0x39, 0x38, 0x30, 0x38, 0x31, 0x38, 0x32, 0x38, 0x33, + 0x38, 0x34, 0x38, 0x35, 0x38, 0x36, 0x38, 0x37, 0x38, 0x38, 0x38, 0x39, + 0x39, 0x30, 0x39, 0x31, 0x39, 0x32, 0x39, 0x33, 0x39, 0x34, 0x39, 0x35, + 0x39, 0x36, 0x39, 0x37, 0x39, 0x38, 0x39, 0x39, +}; +} // namespace internal + +template +simdjson_inline void string_builder::append(number_type v) noexcept { + static_assert(std::is_same::value || + std::is_integral::value || + std::is_floating_point::value, + "Unsupported number type"); + // If C++17 is available, we can 'if constexpr' here. + SIMDJSON_IF_CONSTEXPR(std::is_same::value) { + if (v) { + constexpr char true_literal[] = "true"; + constexpr size_t true_len = sizeof(true_literal) - 1; + if (capacity_check(true_len)) { + std::memcpy(buffer.get() + position, true_literal, true_len); + position += true_len; + } + } else { + constexpr char false_literal[] = "false"; + constexpr size_t false_len = sizeof(false_literal) - 1; + if (capacity_check(false_len)) { + std::memcpy(buffer.get() + position, false_literal, false_len); + position += false_len; + } + } + } + else SIMDJSON_IF_CONSTEXPR(std::is_unsigned::value) { + constexpr size_t max_number_size = 20; + if (capacity_check(max_number_size)) { + using unsigned_type = typename std::make_unsigned::type; + unsigned_type pv = static_cast(v); + size_t dc = internal::digit_count(pv); + char *write_pointer = buffer.get() + position + dc - 1; + while (pv >= 100) { + memcpy(write_pointer - 1, &internal::decimal_table[(pv % 100)*2], 2); + write_pointer -= 2; + pv /= 100; + } + if (pv >= 10) { + *write_pointer-- = char('0' + (pv % 10)); + pv /= 10; + } + *write_pointer = char('0' + pv); + position += dc; + } + } + else SIMDJSON_IF_CONSTEXPR(std::is_integral::value) { + constexpr size_t max_number_size = 20; + if (capacity_check(max_number_size)) { + using unsigned_type = typename std::make_unsigned::type; + bool negative = v < 0; + unsigned_type pv = static_cast(v); + if (negative) { + pv = 0 - pv; // the 0 is for Microsoft + } + size_t dc = internal::digit_count(pv); + if (negative) { + buffer.get()[position++] = '-'; + } + char *write_pointer = buffer.get() + position + dc - 1; + while (pv >= 100) { + memcpy(write_pointer - 1, &internal::decimal_table[(pv % 100)*2], 2); + write_pointer -= 2; + pv /= 100; + } + if (pv >= 10) { + *write_pointer-- = char('0' + (pv % 10)); + pv /= 10; + } + *write_pointer = char('0' + pv); + position += dc; + } + } + else SIMDJSON_IF_CONSTEXPR(std::is_floating_point::value) { + constexpr size_t max_number_size = 24; + if (capacity_check(max_number_size)) { + // We could specialize for float. + char *end = simdjson::internal::to_chars(buffer.get() + position, nullptr, + double(v)); + position = end - buffer.get(); + } + } +} + +simdjson_inline void +string_builder::escape_and_append(std::string_view input) noexcept { + // escaping might turn a control character into \x00xx so 6 characters. + if (capacity_check(6 * input.size())) { + position += write_string_escaped(input, buffer.get() + position); + } +} + +simdjson_inline void +string_builder::escape_and_append_with_quotes(std::string_view input) noexcept { + // escaping might turn a control character into \x00xx so 6 characters. + if (capacity_check(2 + 6 * input.size())) { + buffer.get()[position++] = '"'; + position += write_string_escaped(input, buffer.get() + position); + buffer.get()[position++] = '"'; + } +} + +simdjson_inline void +string_builder::escape_and_append_with_quotes(char input) noexcept { + // escaping might turn a control character into \x00xx so 6 characters. + if (capacity_check(2 + 6 * 1)) { + buffer.get()[position++] = '"'; + std::string_view cinput(&input, 1); + position += write_string_escaped(cinput, buffer.get() + position); + buffer.get()[position++] = '"'; + } +} + +simdjson_inline void string_builder::escape_and_append_with_quotes(const char* input) noexcept { + std::string_view cinput(input); + escape_and_append_with_quotes(cinput); +} + +simdjson_inline void string_builder::append_raw(const char *c) noexcept { + size_t len = std::strlen(c); + append_raw(c, len); +} + +simdjson_inline void +string_builder::append_raw(std::string_view input) noexcept { + if (capacity_check(input.size())) { + std::memcpy(buffer.get() + position, input.data(), input.size()); + position += input.size(); + } +} + +simdjson_inline void string_builder::append_raw(const char *str, + size_t len) noexcept { + if (capacity_check(len)) { + std::memcpy(buffer.get() + position, str, len); + position += len; + } +} + +#if SIMDJSON_EXCEPTIONS +simdjson_inline string_builder::operator std::string() const noexcept(false) { + return std::string(std::string_view()); +} + +simdjson_inline string_builder::operator std::string_view() const + noexcept(false) { + return view(); +} +#endif + +simdjson_inline simdjson_result +string_builder::view() const noexcept { + if (!is_valid) { + return simdjson::OUT_OF_CAPACITY; + } + return std::string_view(buffer.get(), position); +} + +simdjson_inline simdjson_result string_builder::c_str() noexcept { + if (capacity_check(1)) { + buffer.get()[position] = '\0'; + return buffer.get(); + } + return simdjson::OUT_OF_CAPACITY; +} + +simdjson_inline bool string_builder::validate_unicode() const noexcept { + return simdjson::validate_utf8(buffer.get(), position); +} + +simdjson_inline void string_builder::start_object() noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = '{'; + } +} + + +simdjson_inline void string_builder::end_object() noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = '}'; + } +} + + +simdjson_inline void string_builder::start_array() noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = '['; + } +} + + +simdjson_inline void string_builder::end_array() noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = ']'; + } +} + + +simdjson_inline void string_builder::append_comma() noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = ','; + } +} + + +simdjson_inline void string_builder::append_colon() noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = ':'; + } +} + +template +simdjson_inline void string_builder::append_key_value(key_type key, value_type value) noexcept { + static_assert( + std::is_arithmetic::value || + std::is_same::value || + std::is_same::value || + std::is_convertible::value || + std::is_same::value, + "Unsupported value type"); + static_assert( + std::is_same::value || + std::is_convertible::value, + "Unsupported key type"); + escape_and_append_with_quotes(key); + append_colon(); + SIMDJSON_IF_CONSTEXPR(std::is_same::value) { + append_null(); + } else SIMDJSON_IF_CONSTEXPR(std::is_same::value) { + escape_and_append_with_quotes(value); + } else SIMDJSON_IF_CONSTEXPR(std::is_convertible::value) { + escape_and_append_with_quotes(value); + } else SIMDJSON_IF_CONSTEXPR(std::is_same::value) { + escape_and_append_with_quotes(value); + } else { + append(value); + } +} + +} // namespace builder +} // namespace fallback +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_STRING_BUILDER_INL_H +/* end file simdjson/generic/ondemand/json_string_builder-inl.h for fallback */ +/* including simdjson/generic/ondemand/json_builder.h for fallback: #include "simdjson/generic/ondemand/json_builder.h" */ +/* begin file simdjson/generic/ondemand/json_builder.h for fallback */ +/** + * This file is part of the builder API. It is temporarily in the ondemand directory + * but we will move it to a builder directory later. + */ +#ifndef SIMDJSON_GENERIC_BUILDER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/concepts.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ +#if SIMDJSON_STATIC_REFLECTION + +#include +#include +#include +#include +#include +#include +// #include // for std::define_static_string - header not available yet + +namespace simdjson { +namespace fallback { +namespace builder { + +// Concept that checks if a type is a container but not a string (because +// strings handling must be handled differently) +template +concept container_but_not_string = + requires(T a) { + { a.size() } -> std::convertible_to; + { + a[std::declval()] + }; // check if elements are accessible for the subscript operator + } && !std::is_same_v && + !std::is_same_v && !std::is_same_v; + +template + requires(container_but_not_string) +constexpr void atom(string_builder &b, const T &t) { + if (t.size() == 0) { + b.append_raw("[]"); + return; + } + b.append('['); + atom(b, t[0]); + for (size_t i = 1; i < t.size(); ++i) { + b.append(','); + atom(b, t[i]); + } + b.append(']'); +} + +template + requires(std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v) +constexpr void atom(string_builder &b, const T &t) { + b.escape_and_append_with_quotes(t); +} + +template +constexpr void atom(string_builder &b, const T &m) { + if (m.empty()) { + b.append_raw("{}"); + return; + } + b.append('{'); + bool first = true; + for (const auto& [key, value] : m) { + if (!first) { + b.append(','); + } + first = false; + // Keys must be convertible to string_view per the concept + b.escape_and_append_with_quotes(key); + b.append(':'); + atom(b, value); + } + b.append('}'); +} + + +template::value && !std::is_same_v>::type> +constexpr void atom(string_builder &b, const number_type t) { + b.append(t); +} +#if SIMDJSON_CONSTEVAL +consteval std::string consteval_to_quoted_escaped(std::string_view input); +#endif + +template + requires(std::is_class_v && !container_but_not_string && + !concepts::string_view_keyed_map && + !std::is_same_v && + !std::is_same_v) +constexpr void atom(string_builder &b, const T &t) { + int i = 0; + b.append('{'); + [:expand(std::meta::nonstatic_data_members_of(^^T, std::meta::access_context::unchecked())):] >> [&]() { + if (i != 0) + b.append(','); + constexpr auto key = std::define_static_string(consteval_to_quoted_escaped(std::meta::identifier_of(dm))); + b.append_raw(key); + b.append(':'); + atom(b, t.[:dm:]); + i++; + }; + b.append('}'); +} + +// works for struct +template void append(string_builder &b, const Z &z) { + int i = 0; + b.append('{'); + [:expand(std::meta::nonstatic_data_members_of(^^Z, std::meta::access_context::unchecked())):] >> [&]() { + if (i != 0) + b.append(','); + constexpr auto key = std::define_static_string(consteval_to_quoted_escaped(std::meta::identifier_of(dm))); + b.append_raw(key); + b.append(':'); + atom(b, z.[:dm:]); + i++; + }; + b.append('}'); +} + +// works for container +template + requires(container_but_not_string) +void append(string_builder &b, const Z &z) { + if (z.size() == 0) { + b.append_raw("[]"); + return; + } + b.append('['); + atom(b, z[0]); + for (size_t i = 1; i < z.size(); ++i) { + b.append(','); + atom(b, z[i]); + } + b.append(']'); +} + +template +simdjson_result to_json_string(const Z &z) { + string_builder b; + append(b, z); + std::string_view s; + if(auto e = b.view().get(s); e) { return e; } + return std::string(s); +} + +template +simdjson_error to_json(const Z &z, std::string &s) { + string_builder b; + append(b, z); + std::string_view view; + if(auto e = b.view().get(view); e) { return e; } + s.assign(view); + return SUCCESS; +} +} // namespace json_builder +} // namespace fallback +} // namespace simdjson +#endif // SIMDJSON_STATIC_REFLECTION + +#endif +/* end file simdjson/generic/ondemand/json_builder.h for fallback */ /* end file simdjson/generic/ondemand/amalgamated.h for fallback */ /* including simdjson/fallback/end.h: #include "simdjson/fallback/end.h" */ @@ -54596,6 +57681,31 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin }; } + +struct escaping { + static constexpr uint32_t BYTES_PROCESSED = 32; + simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst); + + simdjson_inline bool has_escape() { return escape_bits != 0; } + simdjson_inline int escape_index() { return trailing_zeroes(escape_bits); } + + uint64_t escape_bits; +}; // struct escaping + + + +simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) { + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v(src); + v.store(dst); + simd8 is_quote = (v == '"'); + simd8 is_backslash = (v == '\\'); + simd8 is_control = (v < 32); + return { + uint64_t((is_backslash | is_quote | is_control).to_bitmask()) + }; +} + } // unnamed namespace } // namespace haswell } // namespace simdjson @@ -54752,10 +57862,26 @@ concept nothrow_deserializable = nothrow_custom_deserializable || is_bu /// Deserialize Tag inline constexpr struct deserialize_tag { + using array_type = haswell::ondemand::array; + using object_type = haswell::ondemand::object; using value_type = haswell::ondemand::value; using document_type = haswell::ondemand::document; using document_reference_type = haswell::ondemand::document_reference; + // Customization Point for array + template + requires custom_deserializable + [[nodiscard]] constexpr /* error_code */ auto operator()(array_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + return tag_invoke(*this, object, output); + } + + // Customization Point for object + template + requires custom_deserializable + [[nodiscard]] constexpr /* error_code */ auto operator()(object_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + return tag_invoke(*this, object, output); + } + // Customization Point for value template requires custom_deserializable @@ -55323,6 +58449,7 @@ public: * * You may use get_double(), get_bool(), get_uint64(), get_int64(), * get_object(), get_array(), get_raw_json_string(), or get_string() instead. + * When SIMDJSON_SUPPORTS_DESERIALIZATION is set, custom types are also supported. * * @returns A value of the given type, parsed from the JSON. * @returns INCORRECT_TYPE If the JSON value is not the given type. @@ -55346,6 +58473,7 @@ public: * Get this value as the given type. * * Supported types: object, array, raw_json_string, string_view, uint64_t, int64_t, double, bool + * If the macro SIMDJSON_SUPPORTS_DESERIALIZATION is set, then custom types are also supported. * * @param out This is set to a value of the given type, parsed from the JSON. If there is an error, this may not be initialized. * @returns INCORRECT_TYPE If the JSON value is not an object. @@ -57588,6 +60716,37 @@ public: * - INDEX_OUT_OF_BOUNDS if the array index is larger than an array length */ simdjson_inline simdjson_result at(size_t index) noexcept; + +#if SIMDJSON_SUPPORTS_DESERIALIZATION + /** + * Get this array as the given type. + * + * @param out This is set to a value of the given type, parsed from the JSON. If there is an error, this may not be initialized. + * @returns INCORRECT_TYPE If the JSON array is not of the given type. + * @returns SUCCESS If the parse succeeded and the out parameter was set to the value. + */ + template + simdjson_inline error_code get(T &out) + noexcept(custom_deserializable ? nothrow_custom_deserializable : true) { + static_assert(custom_deserializable); + return deserialize(*this, out); + } + /** + * Get this array as the given type. + * + * @returns A value of the given type, parsed from the JSON. + * @returns INCORRECT_TYPE If the JSON value is not the given type. + */ + template + simdjson_inline simdjson_result get() + noexcept(custom_deserializable ? nothrow_custom_deserializable : true) + { + static_assert(std::is_default_constructible::value, "The specified type is not default constructible."); + T out{}; + SIMDJSON_TRY(get(out)); + return out; + } +#endif // SIMDJSON_SUPPORTS_DESERIALIZATION protected: /** * Go to the end of the array, no matter where you are right now. @@ -57666,7 +60825,28 @@ public: simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; simdjson_inline simdjson_result raw_json() noexcept; +#if SIMDJSON_SUPPORTS_DESERIALIZATION + // TODO: move this code into object-inl.h + template + simdjson_inline simdjson_result get() noexcept { + if (error()) { return error(); } + if constexpr (std::is_same_v) { + return first; + } + return first.get(); + } + template + simdjson_inline error_code get(T& out) noexcept { + if (error()) { return error(); } + if constexpr (std::is_same_v) { + out = first; + } else { + SIMDJSON_TRY( first.get(out) ); + } + return SUCCESS; + } +#endif // SIMDJSON_SUPPORTS_DESERIALIZATION }; } // namespace simdjson @@ -58000,7 +61180,7 @@ public: * Be mindful that the document instance must remain in scope while you are accessing object, array and value instances. * * @param out This is set to a value of the given type, parsed from the JSON. If there is an error, this may not be initialized. - * @returns INCORRECT_TYPE If the JSON value is not an object. + * @returns INCORRECT_TYPE If the JSON value is of the given type. * @returns SUCCESS If the parse succeeded and the out parameter was set to the value. */ template @@ -59487,6 +62667,36 @@ public: */ simdjson_inline simdjson_result raw_json() noexcept; +#if SIMDJSON_SUPPORTS_DESERIALIZATION + /** + * Get this object as the given type. + * + * @param out This is set to a value of the given type, parsed from the JSON. If there is an error, this may not be initialized. + * @returns INCORRECT_TYPE If the JSON object is not of the given type. + * @returns SUCCESS If the parse succeeded and the out parameter was set to the value. + */ + template + simdjson_inline error_code get(T &out) + noexcept(custom_deserializable ? nothrow_custom_deserializable : true) { + static_assert(custom_deserializable); + return deserialize(*this, out); + } + /** + * Get this array as the given type. + * + * @returns A value of the given type, parsed from the JSON. + * @returns INCORRECT_TYPE If the JSON value is not the given type. + */ + template + simdjson_inline simdjson_result get() + noexcept(custom_deserializable ? nothrow_custom_deserializable : true) + { + static_assert(std::is_default_constructible::value, "The specified type is not default constructible."); + T out{}; + SIMDJSON_TRY(get(out)); + return out; + } +#endif // SIMDJSON_SUPPORTS_DESERIALIZATION protected: /** * Go to the end of the object, no matter where you are right now. @@ -59530,12 +62740,32 @@ public: simdjson_inline simdjson_result operator[](std::string_view key) && noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - inline simdjson_result reset() noexcept; inline simdjson_result is_empty() noexcept; inline simdjson_result count_fields() & noexcept; inline simdjson_result raw_json() noexcept; + #if SIMDJSON_SUPPORTS_DESERIALIZATION + // TODO: move this code into object-inl.h + template + simdjson_inline simdjson_result get() noexcept { + if (error()) { return error(); } + if constexpr (std::is_same_v) { + return first; + } + return first.get(); + } + template + simdjson_inline error_code get(T& out) noexcept { + if (error()) { return error(); } + if constexpr (std::is_same_v) { + out = first; + } else { + SIMDJSON_TRY( first.get(out) ); + } + return SUCCESS; + } +#endif // SIMDJSON_SUPPORTS_DESERIALIZATION }; } // namespace simdjson @@ -59740,12 +62970,17 @@ inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result #include +#if SIMDJSON_STATIC_REFLECTION +#include +// #include // for std::define_static_string - header not available yet +#endif namespace simdjson { template @@ -59792,6 +63027,22 @@ error_code tag_invoke(deserialize_tag, auto &val, T &out) noexcept { return SUCCESS; } +////////////////////////////// +// String deserialization +////////////////////////////// + +// just a character! +error_code tag_invoke(deserialize_tag, auto &val, char &out) noexcept { + std::string_view x; + SIMDJSON_TRY(val.get_string().get(x)); + if(x.size() != 1) { + return INCORRECT_TYPE; + } + out = x[0]; + return SUCCESS; +} + +// any string-like type (can be constructed from std::string_view) template requires(!require_custom_serialization) error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept(std::is_nothrow_constructible_v) { @@ -59813,15 +63064,20 @@ template requires(!require_custom_serialization) error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept(false) { using value_type = typename std::remove_cvref_t::value_type; - static_assert( + /*static_assert( deserializable, - "The specified type inside the container must itself be deserializable"); + "The specified type inside the container must itself be deserializable");*/ static_assert( std::is_default_constructible_v, "The specified type inside the container must default constructible."); haswell::ondemand::array arr; - SIMDJSON_TRY(val.get_array().get(arr)); + if constexpr (std::is_same_v, haswell::ondemand::array>) { + arr = val; + } else { + SIMDJSON_TRY(val.get_array().get(arr)); + } + for (auto v : arr) { if constexpr (concepts::returns_reference) { if (auto const err = v.get().get(concepts::emplace_one(out)); @@ -59878,7 +63134,45 @@ error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept(false) { return SUCCESS; } +template +error_code tag_invoke(deserialize_tag, haswell::ondemand::object &obj, T &out) noexcept { + using value_type = typename std::remove_cvref_t::mapped_type; + out.clear(); + for (auto field : obj) { + std::string_view key; + SIMDJSON_TRY(field.unescaped_key().get(key)); + + haswell::ondemand::value value_obj; + SIMDJSON_TRY(field.value().get(value_obj)); + + value_type this_value; + SIMDJSON_TRY(value_obj.get(this_value)); + out.emplace(typename T::key_type(key), std::move(this_value)); + } + return SUCCESS; +} + +template +error_code tag_invoke(deserialize_tag, haswell::ondemand::value &val, T &out) noexcept { + haswell::ondemand::object obj; + SIMDJSON_TRY(val.get_object().get(obj)); + return simdjson::deserialize(obj, out); +} + +template +error_code tag_invoke(deserialize_tag, haswell::ondemand::document &doc, T &out) noexcept { + haswell::ondemand::object obj; + SIMDJSON_TRY(doc.get_object().get(obj)); + return simdjson::deserialize(obj, out); +} + +template +error_code tag_invoke(deserialize_tag, haswell::ondemand::document_reference &doc, T &out) noexcept { + haswell::ondemand::object obj; + SIMDJSON_TRY(doc.get_object().get(obj)); + return simdjson::deserialize(obj, out); +} /** @@ -59940,6 +63234,199 @@ error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept(nothrow_deser return SUCCESS; } + +#if SIMDJSON_STATIC_REFLECTION + + +template +constexpr bool user_defined_type = (std::is_class_v +&& !std::is_same_v && !std::is_same_v && !concepts::optional_type && +!concepts::appendable_containers && !require_custom_serialization); + + +// workaround from +// https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2024/p2996r10.html#back-and-forth +// for missing expansion statements +namespace __impl { + template + struct replicator_type { + template + constexpr void operator>>(F body) const { + (body.template operator()(), ...); + } + }; + + template + replicator_type replicator = {}; +} + +template +consteval auto expand(R range) { + std::vector args; + for (auto r : range) { + args.push_back(reflect_value(r)); + } + return substitute(^^__impl::replicator, args); +} +// end of workaround + +template + requires(user_defined_type && std::is_class_v) +error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept { + haswell::ondemand::object obj; + if constexpr (std::is_same_v, haswell::ondemand::object>) { + obj = val; + } else { + SIMDJSON_TRY(val.get_object().get(obj)); + } + error_code e = simdjson::SUCCESS; + + [:expand(std::meta::nonstatic_data_members_of(^^T, std::meta::access_context::unchecked())):] >> [&]() { + if constexpr (!std::meta::is_const(mem) && std::meta::is_public(mem)) { + constexpr std::string_view key = std::define_static_string(std::meta::identifier_of(mem)); + static_assert( + deserializable, + "The specified type inside the class must itself be deserializable"); + // as long we are succesful or the field is not found, we continue + if(e == simdjson::SUCCESS || e == simdjson::NO_SUCH_FIELD) { + obj[key].get(out.[:mem:]); + } + } + }; + return e; +} +template + requires(user_defined_type>) +error_code tag_invoke(deserialize_tag, simdjson_value &val, std::unique_ptr &out) noexcept { + if (!out) { + out = std::make_unique(); + if (!out) { + return MEMALLOC; + } + } + if (auto err = val.get(*out)) { + out.reset(); + return err; + } + return SUCCESS; +} + +template + requires(user_defined_type>) +error_code tag_invoke(deserialize_tag, simdjson_value &val, std::shared_ptr &out) noexcept { + if (!out) { + out = std::make_shared(); + if (!out) { + return MEMALLOC; + } + } + if (auto err = val.get(*out)) { + out.reset(); + return err; + } + return SUCCESS; +} + +#endif // SIMDJSON_STATIC_REFLECTION + +//////////////////////////////////////// +// Unique pointers +//////////////////////////////////////// +error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) noexcept { + if (!out) { + out = std::make_unique(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_bool().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) noexcept { + if (!out) { + out = std::make_unique(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_int64().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) noexcept { + if (!out) { + out = std::make_unique(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_uint64().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) noexcept { + if (!out) { + out = std::make_unique(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_double().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) noexcept { + if (!out) { + out = std::make_unique(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_string().get(*out)); + return SUCCESS; +} + + +//////////////////////////////////////// +// Shared pointers +//////////////////////////////////////// +error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) noexcept { + if (!out) { + out = std::make_shared(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_bool().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) noexcept { + if (!out) { + out = std::make_shared(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_int64().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) noexcept { + if (!out) { + out = std::make_shared(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_uint64().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) noexcept { + if (!out) { + out = std::make_shared(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_double().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) noexcept { + if (!out) { + out = std::make_shared(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_string().get(*out)); + return SUCCESS; +} + + } // namespace simdjson #endif // SIMDJSON_ONDEMAND_DESERIALIZE_H @@ -63427,6 +66914,7 @@ simdjson_inline simdjson_result simdjson_result::simdjson_res #endif // SIMDJSON_GENERIC_ONDEMAND_VALUE_ITERATOR_INL_H /* end file simdjson/generic/ondemand/value_iterator-inl.h for haswell */ +// JSON builder, ideally they should not be part of the ondemand directory +// but it is convenient for now to have them here. +/* including simdjson/generic/ondemand/json_string_builder.h for haswell: #include "simdjson/generic/ondemand/json_string_builder.h" */ +/* begin file simdjson/generic/ondemand/json_string_builder.h for haswell */ +/** + * This file is part of the builder API. It is temporarily in the ondemand directory + * but we will move it to a builder directory later. + */ +#ifndef SIMDJSON_GENERIC_STRING_BUILDER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { +namespace haswell { +namespace builder { + +/** + * A builder for JSON strings representing documents. This is a low-level + * builder that is not meant to be used directly by end-users. Though it + * supports atomic types (Booleans, strings), it does not support composed + * types (arrays and objects). + * + * Ultimately, this class should support kernel-specific optimizations. E.g., + * it may make use of SIMD instructions to escape strings faster. + */ +class string_builder { +public: + simdjson_inline string_builder(size_t initial_capacity = 1024); + + /** + * Append number (includes Booleans). Booleans are mapped to the strings + * false and true. Numbers are converted to strings abiding by the JSON standard. + * Floating-point numbers are converted to the shortest string that 'correctly' + * represents the number. + */ + template::value>::type> + simdjson_inline void append(number_type v) noexcept; + + /** + * Append character c. + */ + simdjson_inline void append(char c) noexcept; + + /** + * Append the string 'null'. + */ + simdjson_inline void append_null() noexcept; + + /** + * Clear the content. + */ + simdjson_inline void clear() noexcept; + + /** + * Append the std::string_view, after escaping it. + * There is no UTF-8 validation. + */ + simdjson_inline void escape_and_append(std::string_view input) noexcept; + + /** + * Append the std::string_view surrounded by double quotes, after escaping it. + * There is no UTF-8 validation. + */ + simdjson_inline void escape_and_append_with_quotes(std::string_view input) noexcept; + + /** + * Append the character surrounded by double quotes, after escaping it. + * There is no UTF-8 validation. + */ + simdjson_inline void escape_and_append_with_quotes(char input) noexcept; + + /** + * Append the character surrounded by double quotes, after escaping it. + * There is no UTF-8 validation. + */ + simdjson_inline void escape_and_append_with_quotes(const char* input) noexcept; + + /** + * Append the C string directly, without escaping. + * There is no UTF-8 validation. + */ + simdjson_inline void append_raw(const char *c) noexcept; + + /** + * Append "{" to the buffer. + */ + simdjson_inline void start_object() noexcept; + + /** + * Append "}" to the buffer. + */ + simdjson_inline void end_object() noexcept; + + /** + * Append "[" to the buffer. + */ + simdjson_inline void start_array() noexcept; + + /** + * Append "]" to the buffer. + */ + simdjson_inline void end_array() noexcept; + + /** + * Append "," to the buffer. + */ + simdjson_inline void append_comma() noexcept; + + /** + * Append ":" to the buffer. + */ + simdjson_inline void append_colon() noexcept; + + /** + * Append a key-value pair to the buffer. + * The key is escaped and surrounded by double quotes. + * The value is escaped if it is a string. + */ + template + simdjson_inline void append_key_value(key_type key, value_type value) noexcept; + /** + * Append the std::string_view directly, without escaping. + * There is no UTF-8 validation. + */ + simdjson_inline void append_raw(std::string_view input) noexcept; + + /** + * Append len characters from str. + * There is no UTF-8 validation. + */ + simdjson_inline void append_raw(const char *str, size_t len) noexcept; +#if SIMDJSON_EXCEPTIONS + /** + * Creates an std::string from the written JSON buffer. + * Throws if memory allocation failed + * + * The result may not be valid UTF-8 if some of your content was not valid UTF-8. + * Use validate_unicode() to check the content if needed. + */ + simdjson_inline operator std::string() const noexcept(false); + + /** + * Creates an std::string_view from the written JSON buffer. + * Throws if memory allocation failed. + * + * The result may not be valid UTF-8 if some of your content was not valid UTF-8. + * Use validate_unicode() to check the content if needed. + */ + simdjson_inline operator std::string_view() const noexcept(false); +#endif + + /** + * Returns a view on the written JSON buffer. Returns an error + * if memory allocation failed. + * + * The result may not be valid UTF-8 if some of your content was not valid UTF-8. + * Use validate_unicode() to check the content. + */ + simdjson_inline simdjson_result view() const noexcept; + + /** + * Appends the null character to the buffer and returns + * a pointer to the beginning of the written JSON buffer. + * Returns an error if memory allocation failed. + * The result is null-terminated. + * + * The result may not be valid UTF-8 if some of your content was not valid UTF-8. + * Use validate_unicode() to check the content. + */ + simdjson_inline simdjson_result c_str() noexcept; + + /** + * Return true if the content is valid UTF-8. + */ + simdjson_inline bool validate_unicode() const noexcept; + + /** + * Returns the current size of the written JSON buffer. + * If an error occurred, returns 0. + */ + simdjson_inline size_t size() const noexcept; + +private: + /** + * Returns true if we can write at least upcoming_bytes bytes. + * The underlying buffer is reallocated if needed. It is designed + * to be called before writing to the buffer. It should be fast. + */ + simdjson_inline bool capacity_check(size_t upcoming_bytes); + + /** + * Grow the buffer to at least desired_capacity bytes. + * If the allocation fails, is_valid is set to false. We expect + * that this function would not be repeatedly called. + */ + simdjson_inline void grow_buffer(size_t desired_capacity); + + /** + * We use this helper function to make sure that is_valid is kept consistent. + */ + simdjson_inline void set_valid(bool valid) noexcept; + + std::unique_ptr buffer{}; + size_t position{0}; + size_t capacity{0}; + bool is_valid{true}; +}; + + + +} +} +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_STRING_BUILDER_H +/* end file simdjson/generic/ondemand/json_string_builder.h for haswell */ +/* including simdjson/generic/ondemand/json_string_builder-inl.h for haswell: #include "simdjson/generic/ondemand/json_string_builder-inl.h" */ +/* begin file simdjson/generic/ondemand/json_string_builder-inl.h for haswell */ +/** + * This file is part of the builder API. It is temporarily in the ondemand + * directory but we will move it to a builder directory later. + */ +#include +#include +#include +#ifndef SIMDJSON_GENERIC_STRING_BUILDER_INL_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_INL_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +/* + * Empirically, we have found that an inlined optimization is important for + * performance. The following macros are not ideal. We should find a better + * way to inline the code. + */ + +#if defined(__SSE2__) || defined(__x86_64__) || defined(__x86_64) || \ + (defined(_M_AMD64) || defined(_M_X64) || \ + (defined(_M_IX86_FP) && _M_IX86_FP == 2)) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_SSE2 +#define SIMDJSON_EXPERIMENTAL_HAS_SSE2 1 +#endif +#endif + +#if defined(__aarch64__) || defined(_M_ARM64) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_NEON +#define SIMDJSON_EXPERIMENTAL_HAS_NEON 1 +#endif +#endif +#if SIMDJSON_EXPERIMENTAL_HAS_NEON +#include +#endif +#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 +#include +#endif + +namespace simdjson { +namespace haswell { +namespace builder { + +static SIMDJSON_CONSTEXPR_LAMBDA std::array json_quotable_character = { + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + +/** + +A possible SWAR implementation of has_json_escapable_byte. It is not used because +it is slower than the current implementation. It is kept here for reference (to show +that we tried it). + +inline bool has_json_escapable_byte(uint64_t x) { + uint64_t is_ascii = 0x8080808080808080ULL & ~x; + uint64_t xor2 = x ^ 0x0202020202020202ULL; + uint64_t lt32_or_eq34 = xor2 - 0x2121212121212121ULL; + uint64_t sub92 = x ^ 0x5C5C5C5C5C5C5C5CULL; + uint64_t eq92 = (sub92 - 0x0101010101010101ULL); + return ((lt32_or_eq34 | eq92) & is_ascii) != 0; +} + +**/ + +SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline bool +simple_needs_escaping(std::string_view v) { + for (char c : v) { + // a table lookup is faster than a series of comparisons + if(json_quotable_character[static_cast(c)]) { + return true; + } + } + return false; +} + +#if SIMDJSON_EXPERIMENTAL_HAS_NEON +simdjson_inline bool fast_needs_escaping(std::string_view view) { + if (view.size() < 16) { + return simple_needs_escaping(view); + } + size_t i = 0; + uint8x16_t running = vdupq_n_u8(0); + uint8x16_t v34 = vdupq_n_u8(34); + uint8x16_t v92 = vdupq_n_u8(92); + + for (; i + 15 < view.size(); i += 16) { + uint8x16_t word = vld1q_u8((const uint8_t *)view.data() + i); + running = vorrq_u8(running, vceqq_u8(word, v34)); + running = vorrq_u8(running, vceqq_u8(word, v92)); + running = vorrq_u8(running, vcltq_u8(word, vdupq_n_u8(32))); + } + if (i < view.size()) { + uint8x16_t word = + vld1q_u8((const uint8_t *)view.data() + view.length() - 16); + running = vorrq_u8(running, vceqq_u8(word, v34)); + running = vorrq_u8(running, vceqq_u8(word, v92)); + running = vorrq_u8(running, vcltq_u8(word, vdupq_n_u8(32))); + } + return vmaxvq_u32(vreinterpretq_u32_u8(running)) != 0; +} +#elif SIMDJSON_EXPERIMENTAL_HAS_SSE2 +simdjson_inline bool fast_needs_escaping(std::string_view view) { + if (view.size() < 16) { + return simple_needs_escaping(view); + } + size_t i = 0; + __m128i running = _mm_setzero_si128(); + for (; i + 15 < view.size(); i += 16) { + + __m128i word = _mm_loadu_si128(reinterpret_cast(view.data() + i)); + running = _mm_or_si128(running, _mm_cmpeq_epi8(word, _mm_set1_epi8(34))); + running = _mm_or_si128(running, _mm_cmpeq_epi8(word, _mm_set1_epi8(92))); + running = _mm_or_si128( + running, _mm_cmpeq_epi8(_mm_subs_epu8(word, _mm_set1_epi8(31)), + _mm_setzero_si128())); + } + if (i < view.size()) { + __m128i word = + _mm_loadu_si128(reinterpret_cast(view.data() + view.length() - 16)); + running = _mm_or_si128(running, _mm_cmpeq_epi8(word, _mm_set1_epi8(34))); + running = _mm_or_si128(running, _mm_cmpeq_epi8(word, _mm_set1_epi8(92))); + running = _mm_or_si128( + running, _mm_cmpeq_epi8(_mm_subs_epu8(word, _mm_set1_epi8(31)), + _mm_setzero_si128())); + } + return _mm_movemask_epi8(running) != 0; +} +#else +simdjson_inline bool fast_needs_escaping(std::string_view view) { + return simple_needs_escaping(view); +} +#endif + + +SIMDJSON_CONSTEXPR_LAMBDA inline size_t +find_next_json_quotable_character(const std::string_view view, + size_t location) noexcept { + + for (auto pos = view.begin() + location; pos != view.end(); ++pos) { + if (json_quotable_character[static_cast(*pos)]) { + return pos - view.begin(); + } + } + return size_t(view.size()); +} + +SIMDJSON_CONSTEXPR_LAMBDA static std::string_view control_chars[] = { + "\\x0000", "\\x0001", "\\x0002", "\\x0003", "\\x0004", "\\x0005", "\\x0006", + "\\x0007", "\\x0008", "\\t", "\\n", "\\x000b", "\\f", "\\r", + "\\x000e", "\\x000f", "\\x0010", "\\x0011", "\\x0012", "\\x0013", "\\x0014", + "\\x0015", "\\x0016", "\\x0017", "\\x0018", "\\x0019", "\\x001a", "\\x001b", + "\\x001c", "\\x001d", "\\x001e", "\\x001f"}; + +SIMDJSON_CONSTEXPR_LAMBDA void escape_json_char(char c, char *&out) { + if (c == '"') { + memcpy(out, "\\\"", 2); + out += 2; + } else if (c == '\\') { + memcpy(out, "\\\\", 2); + out += 2; + } else { + std::string_view v = control_chars[uint8_t(c)]; + memcpy(out, v.data(), v.size()); + out += v.size(); + } +} + +inline size_t write_string_escaped(const std::string_view input, char *out) { + size_t mysize = input.size(); + if (!fast_needs_escaping(input)) { // fast path! + memcpy(out, input.data(), input.size()); + return input.size(); + } + const char *const initout = out; + size_t location = find_next_json_quotable_character(input, 0); + memcpy(out, input.data(), location); + out += location; + escape_json_char(input[location], out); + location += 1; + while (location < mysize) { + size_t newlocation = find_next_json_quotable_character(input, location); + memcpy(out, input.data() + location, newlocation - location); + out += newlocation - location; + location = newlocation; + if (location == mysize) { + break; + } + escape_json_char(input[location], out); + location += 1; + } + return out - initout; +} + +#if SIMDJSON_CONSTEVAL +// unoptimized, meant for compile-time execution +consteval std::string consteval_to_quoted_escaped(std::string_view input) { + std::string out = "\""; + for (char c : input) { + if (json_quotable_character[uint8_t(c)]) { + if (c == '"') { + out.append("\\\""); + } else if (c == '\\') { + out.append("\\\\"); + } else { + std::string_view v = control_chars[uint8_t(c)]; + out.append(v); + } + } else { + out.push_back(c); + } + } + out.push_back('"'); + return out; +} +#endif // SIMDJSON_CONSTEVAL + +simdjson_inline string_builder::string_builder(size_t initial_capacity) + : buffer(new(std::nothrow) char[initial_capacity]), position(0), + capacity(buffer.get() != nullptr ? initial_capacity : 0), + is_valid(buffer.get() != nullptr) {} + +simdjson_inline bool string_builder::capacity_check(size_t upcoming_bytes) { + // We use the convention that when is_valid is false, then the capacity and + // the position are 0. + // Most of the time, this function will return true. + if (simdjson_likely(upcoming_bytes <= capacity - position)) { + return true; + } + // check for overflow, most of the time there is no overflow + if (simdjson_likely(position + upcoming_bytes < position)) { + return false; + } + // We will rarely get here. + grow_buffer((std::max)(capacity * 2, position + upcoming_bytes)); + // If the buffer allocation failed, we set is_valid to false. + return is_valid; +} + +simdjson_inline void string_builder::grow_buffer(size_t desired_capacity) { + if (!is_valid) { + return; + } + std::unique_ptr new_buffer(new (std::nothrow) char[desired_capacity]); + if (new_buffer.get() == nullptr) { + set_valid(false); + return; + } + std::memcpy(new_buffer.get(), buffer.get(), position); + buffer.swap(new_buffer); + capacity = desired_capacity; +} + +simdjson_inline void string_builder::set_valid(bool valid) noexcept { + if (!valid) { + is_valid = false; + capacity = 0; + position = 0; + buffer.reset(); + } else { + is_valid = true; + } +} + +simdjson_inline size_t string_builder::size() const noexcept { + return position; +} + +simdjson_inline void string_builder::append(char c) noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = c; + } +} + +simdjson_inline void string_builder::append_null() noexcept { + constexpr char null_literal[] = "null"; + constexpr size_t null_len = sizeof(null_literal) - 1; + if (capacity_check(null_len)) { + std::memcpy(buffer.get() + position, null_literal, null_len); + position += null_len; + } +} + +simdjson_inline void string_builder::clear() noexcept { + position = 0; + // if it was invalid, we should try to repair it + if (!is_valid) { + capacity = 0; + buffer.reset(); + is_valid = true; + } +} + +namespace internal { + +// We could specialize further for 32-bit integers. +int int_log2(uint32_t x) { return (63 - leading_zeroes(x | 1)); } + +int fast_digit_count(uint32_t x) { + static uint64_t table[] = { + 4294967296, 8589934582, 8589934582, 8589934582, 12884901788, + 12884901788, 12884901788, 17179868184, 17179868184, 17179868184, + 21474826480, 21474826480, 21474826480, 21474826480, 25769703776, + 25769703776, 25769703776, 30063771072, 30063771072, 30063771072, + 34349738368, 34349738368, 34349738368, 34349738368, 38554705664, + 38554705664, 38554705664, 41949672960, 41949672960, 41949672960, + 42949672960, 42949672960}; + return uint32_t((x + table[int_log2(x)]) >> 32); +} + +int int_log2(uint64_t x) { return 63 - leading_zeroes(x | 1); } + +int fast_digit_count(uint64_t x) { + static uint64_t table[] = {9, + 99, + 999, + 9999, + 99999, + 999999, + 9999999, + 99999999, + 999999999, + 9999999999, + 99999999999, + 999999999999, + 9999999999999, + 99999999999999, + 999999999999999ULL, + 9999999999999999ULL, + 99999999999999999ULL, + 999999999999999999ULL, + 9999999999999999999ULL}; + int y = (19 * int_log2(x) >> 6); + y += x > table[y]; + return y + 1; +} + +template ::value>::type> +simdjson_inline size_t digit_count(number_type v) noexcept { + static_assert(sizeof(number_type) == 8 || sizeof(number_type) == 4 || + sizeof(number_type) == 2 || sizeof(number_type) == 1, + "We only support 8-bit, 16-bit, 32-bit and 64-bit numbers"); + return fast_digit_count(v); +} +static const char decimal_table[200] = { + 0x30, 0x30, 0x30, 0x31, 0x30, 0x32, 0x30, 0x33, 0x30, 0x34, 0x30, 0x35, + 0x30, 0x36, 0x30, 0x37, 0x30, 0x38, 0x30, 0x39, 0x31, 0x30, 0x31, 0x31, + 0x31, 0x32, 0x31, 0x33, 0x31, 0x34, 0x31, 0x35, 0x31, 0x36, 0x31, 0x37, + 0x31, 0x38, 0x31, 0x39, 0x32, 0x30, 0x32, 0x31, 0x32, 0x32, 0x32, 0x33, + 0x32, 0x34, 0x32, 0x35, 0x32, 0x36, 0x32, 0x37, 0x32, 0x38, 0x32, 0x39, + 0x33, 0x30, 0x33, 0x31, 0x33, 0x32, 0x33, 0x33, 0x33, 0x34, 0x33, 0x35, + 0x33, 0x36, 0x33, 0x37, 0x33, 0x38, 0x33, 0x39, 0x34, 0x30, 0x34, 0x31, + 0x34, 0x32, 0x34, 0x33, 0x34, 0x34, 0x34, 0x35, 0x34, 0x36, 0x34, 0x37, + 0x34, 0x38, 0x34, 0x39, 0x35, 0x30, 0x35, 0x31, 0x35, 0x32, 0x35, 0x33, + 0x35, 0x34, 0x35, 0x35, 0x35, 0x36, 0x35, 0x37, 0x35, 0x38, 0x35, 0x39, + 0x36, 0x30, 0x36, 0x31, 0x36, 0x32, 0x36, 0x33, 0x36, 0x34, 0x36, 0x35, + 0x36, 0x36, 0x36, 0x37, 0x36, 0x38, 0x36, 0x39, 0x37, 0x30, 0x37, 0x31, + 0x37, 0x32, 0x37, 0x33, 0x37, 0x34, 0x37, 0x35, 0x37, 0x36, 0x37, 0x37, + 0x37, 0x38, 0x37, 0x39, 0x38, 0x30, 0x38, 0x31, 0x38, 0x32, 0x38, 0x33, + 0x38, 0x34, 0x38, 0x35, 0x38, 0x36, 0x38, 0x37, 0x38, 0x38, 0x38, 0x39, + 0x39, 0x30, 0x39, 0x31, 0x39, 0x32, 0x39, 0x33, 0x39, 0x34, 0x39, 0x35, + 0x39, 0x36, 0x39, 0x37, 0x39, 0x38, 0x39, 0x39, +}; +} // namespace internal + +template +simdjson_inline void string_builder::append(number_type v) noexcept { + static_assert(std::is_same::value || + std::is_integral::value || + std::is_floating_point::value, + "Unsupported number type"); + // If C++17 is available, we can 'if constexpr' here. + SIMDJSON_IF_CONSTEXPR(std::is_same::value) { + if (v) { + constexpr char true_literal[] = "true"; + constexpr size_t true_len = sizeof(true_literal) - 1; + if (capacity_check(true_len)) { + std::memcpy(buffer.get() + position, true_literal, true_len); + position += true_len; + } + } else { + constexpr char false_literal[] = "false"; + constexpr size_t false_len = sizeof(false_literal) - 1; + if (capacity_check(false_len)) { + std::memcpy(buffer.get() + position, false_literal, false_len); + position += false_len; + } + } + } + else SIMDJSON_IF_CONSTEXPR(std::is_unsigned::value) { + constexpr size_t max_number_size = 20; + if (capacity_check(max_number_size)) { + using unsigned_type = typename std::make_unsigned::type; + unsigned_type pv = static_cast(v); + size_t dc = internal::digit_count(pv); + char *write_pointer = buffer.get() + position + dc - 1; + while (pv >= 100) { + memcpy(write_pointer - 1, &internal::decimal_table[(pv % 100)*2], 2); + write_pointer -= 2; + pv /= 100; + } + if (pv >= 10) { + *write_pointer-- = char('0' + (pv % 10)); + pv /= 10; + } + *write_pointer = char('0' + pv); + position += dc; + } + } + else SIMDJSON_IF_CONSTEXPR(std::is_integral::value) { + constexpr size_t max_number_size = 20; + if (capacity_check(max_number_size)) { + using unsigned_type = typename std::make_unsigned::type; + bool negative = v < 0; + unsigned_type pv = static_cast(v); + if (negative) { + pv = 0 - pv; // the 0 is for Microsoft + } + size_t dc = internal::digit_count(pv); + if (negative) { + buffer.get()[position++] = '-'; + } + char *write_pointer = buffer.get() + position + dc - 1; + while (pv >= 100) { + memcpy(write_pointer - 1, &internal::decimal_table[(pv % 100)*2], 2); + write_pointer -= 2; + pv /= 100; + } + if (pv >= 10) { + *write_pointer-- = char('0' + (pv % 10)); + pv /= 10; + } + *write_pointer = char('0' + pv); + position += dc; + } + } + else SIMDJSON_IF_CONSTEXPR(std::is_floating_point::value) { + constexpr size_t max_number_size = 24; + if (capacity_check(max_number_size)) { + // We could specialize for float. + char *end = simdjson::internal::to_chars(buffer.get() + position, nullptr, + double(v)); + position = end - buffer.get(); + } + } +} + +simdjson_inline void +string_builder::escape_and_append(std::string_view input) noexcept { + // escaping might turn a control character into \x00xx so 6 characters. + if (capacity_check(6 * input.size())) { + position += write_string_escaped(input, buffer.get() + position); + } +} + +simdjson_inline void +string_builder::escape_and_append_with_quotes(std::string_view input) noexcept { + // escaping might turn a control character into \x00xx so 6 characters. + if (capacity_check(2 + 6 * input.size())) { + buffer.get()[position++] = '"'; + position += write_string_escaped(input, buffer.get() + position); + buffer.get()[position++] = '"'; + } +} + +simdjson_inline void +string_builder::escape_and_append_with_quotes(char input) noexcept { + // escaping might turn a control character into \x00xx so 6 characters. + if (capacity_check(2 + 6 * 1)) { + buffer.get()[position++] = '"'; + std::string_view cinput(&input, 1); + position += write_string_escaped(cinput, buffer.get() + position); + buffer.get()[position++] = '"'; + } +} + +simdjson_inline void string_builder::escape_and_append_with_quotes(const char* input) noexcept { + std::string_view cinput(input); + escape_and_append_with_quotes(cinput); +} + +simdjson_inline void string_builder::append_raw(const char *c) noexcept { + size_t len = std::strlen(c); + append_raw(c, len); +} + +simdjson_inline void +string_builder::append_raw(std::string_view input) noexcept { + if (capacity_check(input.size())) { + std::memcpy(buffer.get() + position, input.data(), input.size()); + position += input.size(); + } +} + +simdjson_inline void string_builder::append_raw(const char *str, + size_t len) noexcept { + if (capacity_check(len)) { + std::memcpy(buffer.get() + position, str, len); + position += len; + } +} + +#if SIMDJSON_EXCEPTIONS +simdjson_inline string_builder::operator std::string() const noexcept(false) { + return std::string(std::string_view()); +} + +simdjson_inline string_builder::operator std::string_view() const + noexcept(false) { + return view(); +} +#endif + +simdjson_inline simdjson_result +string_builder::view() const noexcept { + if (!is_valid) { + return simdjson::OUT_OF_CAPACITY; + } + return std::string_view(buffer.get(), position); +} + +simdjson_inline simdjson_result string_builder::c_str() noexcept { + if (capacity_check(1)) { + buffer.get()[position] = '\0'; + return buffer.get(); + } + return simdjson::OUT_OF_CAPACITY; +} + +simdjson_inline bool string_builder::validate_unicode() const noexcept { + return simdjson::validate_utf8(buffer.get(), position); +} + +simdjson_inline void string_builder::start_object() noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = '{'; + } +} + + +simdjson_inline void string_builder::end_object() noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = '}'; + } +} + + +simdjson_inline void string_builder::start_array() noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = '['; + } +} + + +simdjson_inline void string_builder::end_array() noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = ']'; + } +} + + +simdjson_inline void string_builder::append_comma() noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = ','; + } +} + + +simdjson_inline void string_builder::append_colon() noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = ':'; + } +} + +template +simdjson_inline void string_builder::append_key_value(key_type key, value_type value) noexcept { + static_assert( + std::is_arithmetic::value || + std::is_same::value || + std::is_same::value || + std::is_convertible::value || + std::is_same::value, + "Unsupported value type"); + static_assert( + std::is_same::value || + std::is_convertible::value, + "Unsupported key type"); + escape_and_append_with_quotes(key); + append_colon(); + SIMDJSON_IF_CONSTEXPR(std::is_same::value) { + append_null(); + } else SIMDJSON_IF_CONSTEXPR(std::is_same::value) { + escape_and_append_with_quotes(value); + } else SIMDJSON_IF_CONSTEXPR(std::is_convertible::value) { + escape_and_append_with_quotes(value); + } else SIMDJSON_IF_CONSTEXPR(std::is_same::value) { + escape_and_append_with_quotes(value); + } else { + append(value); + } +} + +} // namespace builder +} // namespace haswell +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_STRING_BUILDER_INL_H +/* end file simdjson/generic/ondemand/json_string_builder-inl.h for haswell */ +/* including simdjson/generic/ondemand/json_builder.h for haswell: #include "simdjson/generic/ondemand/json_builder.h" */ +/* begin file simdjson/generic/ondemand/json_builder.h for haswell */ +/** + * This file is part of the builder API. It is temporarily in the ondemand directory + * but we will move it to a builder directory later. + */ +#ifndef SIMDJSON_GENERIC_BUILDER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/concepts.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ +#if SIMDJSON_STATIC_REFLECTION + +#include +#include +#include +#include +#include +#include +// #include // for std::define_static_string - header not available yet + +namespace simdjson { +namespace haswell { +namespace builder { + +// Concept that checks if a type is a container but not a string (because +// strings handling must be handled differently) +template +concept container_but_not_string = + requires(T a) { + { a.size() } -> std::convertible_to; + { + a[std::declval()] + }; // check if elements are accessible for the subscript operator + } && !std::is_same_v && + !std::is_same_v && !std::is_same_v; + +template + requires(container_but_not_string) +constexpr void atom(string_builder &b, const T &t) { + if (t.size() == 0) { + b.append_raw("[]"); + return; + } + b.append('['); + atom(b, t[0]); + for (size_t i = 1; i < t.size(); ++i) { + b.append(','); + atom(b, t[i]); + } + b.append(']'); +} + +template + requires(std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v) +constexpr void atom(string_builder &b, const T &t) { + b.escape_and_append_with_quotes(t); +} + +template +constexpr void atom(string_builder &b, const T &m) { + if (m.empty()) { + b.append_raw("{}"); + return; + } + b.append('{'); + bool first = true; + for (const auto& [key, value] : m) { + if (!first) { + b.append(','); + } + first = false; + // Keys must be convertible to string_view per the concept + b.escape_and_append_with_quotes(key); + b.append(':'); + atom(b, value); + } + b.append('}'); +} + + +template::value && !std::is_same_v>::type> +constexpr void atom(string_builder &b, const number_type t) { + b.append(t); +} +#if SIMDJSON_CONSTEVAL +consteval std::string consteval_to_quoted_escaped(std::string_view input); +#endif + +template + requires(std::is_class_v && !container_but_not_string && + !concepts::string_view_keyed_map && + !std::is_same_v && + !std::is_same_v) +constexpr void atom(string_builder &b, const T &t) { + int i = 0; + b.append('{'); + [:expand(std::meta::nonstatic_data_members_of(^^T, std::meta::access_context::unchecked())):] >> [&]() { + if (i != 0) + b.append(','); + constexpr auto key = std::define_static_string(consteval_to_quoted_escaped(std::meta::identifier_of(dm))); + b.append_raw(key); + b.append(':'); + atom(b, t.[:dm:]); + i++; + }; + b.append('}'); +} + +// works for struct +template void append(string_builder &b, const Z &z) { + int i = 0; + b.append('{'); + [:expand(std::meta::nonstatic_data_members_of(^^Z, std::meta::access_context::unchecked())):] >> [&]() { + if (i != 0) + b.append(','); + constexpr auto key = std::define_static_string(consteval_to_quoted_escaped(std::meta::identifier_of(dm))); + b.append_raw(key); + b.append(':'); + atom(b, z.[:dm:]); + i++; + }; + b.append('}'); +} + +// works for container +template + requires(container_but_not_string) +void append(string_builder &b, const Z &z) { + if (z.size() == 0) { + b.append_raw("[]"); + return; + } + b.append('['); + atom(b, z[0]); + for (size_t i = 1; i < z.size(); ++i) { + b.append(','); + atom(b, z[i]); + } + b.append(']'); +} + +template +simdjson_result to_json_string(const Z &z) { + string_builder b; + append(b, z); + std::string_view s; + if(auto e = b.view().get(s); e) { return e; } + return std::string(s); +} + +template +simdjson_error to_json(const Z &z, std::string &s) { + string_builder b; + append(b, z); + std::string_view view; + if(auto e = b.view().get(view); e) { return e; } + s.assign(view); + return SUCCESS; +} +} // namespace json_builder +} // namespace haswell +} // namespace simdjson +#endif // SIMDJSON_STATIC_REFLECTION + +#endif +/* end file simdjson/generic/ondemand/json_builder.h for haswell */ /* end file simdjson/generic/ondemand/amalgamated.h for haswell */ /* including simdjson/haswell/end.h: #include "simdjson/haswell/end.h" */ @@ -65729,7 +70230,6 @@ namespace simd { friend simdjson_really_inline uint64_t operator==(const simd8 lhs, const simd8 rhs) { return _mm512_cmpeq_epi8_mask(lhs, rhs); } - static const int SIZE = sizeof(base::value); template @@ -66072,6 +70572,35 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin }; } + + +struct escaping { + static constexpr uint32_t BYTES_PROCESSED = 64; + simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst); + + simdjson_inline bool has_escape() { return escape_bits != 0; } + simdjson_inline int escape_index() { return trailing_zeroes(uint64_t(escape_bits)); } + + __mmask64 escape_bits; +}; // struct escaping + + + +simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) { + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v(src); + v.store(dst); + __mmask64 is_quote = _mm512_cmpeq_epi8_mask(v, _mm512_set1_epi8('"')); + __mmask64 is_backslash = _mm512_cmpeq_epi8_mask(v, _mm512_set1_epi8('\\')); + __mmask64 is_control = _mm512_cmplt_epi8_mask(v, _mm512_set1_epi8(32)); + return { + (is_backslash | is_quote | is_control) + }; +} + + + + } // unnamed namespace } // namespace icelake } // namespace simdjson @@ -66288,10 +70817,26 @@ concept nothrow_deserializable = nothrow_custom_deserializable || is_bu /// Deserialize Tag inline constexpr struct deserialize_tag { + using array_type = icelake::ondemand::array; + using object_type = icelake::ondemand::object; using value_type = icelake::ondemand::value; using document_type = icelake::ondemand::document; using document_reference_type = icelake::ondemand::document_reference; + // Customization Point for array + template + requires custom_deserializable + [[nodiscard]] constexpr /* error_code */ auto operator()(array_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + return tag_invoke(*this, object, output); + } + + // Customization Point for object + template + requires custom_deserializable + [[nodiscard]] constexpr /* error_code */ auto operator()(object_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + return tag_invoke(*this, object, output); + } + // Customization Point for value template requires custom_deserializable @@ -66859,6 +71404,7 @@ public: * * You may use get_double(), get_bool(), get_uint64(), get_int64(), * get_object(), get_array(), get_raw_json_string(), or get_string() instead. + * When SIMDJSON_SUPPORTS_DESERIALIZATION is set, custom types are also supported. * * @returns A value of the given type, parsed from the JSON. * @returns INCORRECT_TYPE If the JSON value is not the given type. @@ -66882,6 +71428,7 @@ public: * Get this value as the given type. * * Supported types: object, array, raw_json_string, string_view, uint64_t, int64_t, double, bool + * If the macro SIMDJSON_SUPPORTS_DESERIALIZATION is set, then custom types are also supported. * * @param out This is set to a value of the given type, parsed from the JSON. If there is an error, this may not be initialized. * @returns INCORRECT_TYPE If the JSON value is not an object. @@ -69124,6 +73671,37 @@ public: * - INDEX_OUT_OF_BOUNDS if the array index is larger than an array length */ simdjson_inline simdjson_result at(size_t index) noexcept; + +#if SIMDJSON_SUPPORTS_DESERIALIZATION + /** + * Get this array as the given type. + * + * @param out This is set to a value of the given type, parsed from the JSON. If there is an error, this may not be initialized. + * @returns INCORRECT_TYPE If the JSON array is not of the given type. + * @returns SUCCESS If the parse succeeded and the out parameter was set to the value. + */ + template + simdjson_inline error_code get(T &out) + noexcept(custom_deserializable ? nothrow_custom_deserializable : true) { + static_assert(custom_deserializable); + return deserialize(*this, out); + } + /** + * Get this array as the given type. + * + * @returns A value of the given type, parsed from the JSON. + * @returns INCORRECT_TYPE If the JSON value is not the given type. + */ + template + simdjson_inline simdjson_result get() + noexcept(custom_deserializable ? nothrow_custom_deserializable : true) + { + static_assert(std::is_default_constructible::value, "The specified type is not default constructible."); + T out{}; + SIMDJSON_TRY(get(out)); + return out; + } +#endif // SIMDJSON_SUPPORTS_DESERIALIZATION protected: /** * Go to the end of the array, no matter where you are right now. @@ -69202,7 +73780,28 @@ public: simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; simdjson_inline simdjson_result raw_json() noexcept; +#if SIMDJSON_SUPPORTS_DESERIALIZATION + // TODO: move this code into object-inl.h + template + simdjson_inline simdjson_result get() noexcept { + if (error()) { return error(); } + if constexpr (std::is_same_v) { + return first; + } + return first.get(); + } + template + simdjson_inline error_code get(T& out) noexcept { + if (error()) { return error(); } + if constexpr (std::is_same_v) { + out = first; + } else { + SIMDJSON_TRY( first.get(out) ); + } + return SUCCESS; + } +#endif // SIMDJSON_SUPPORTS_DESERIALIZATION }; } // namespace simdjson @@ -69536,7 +74135,7 @@ public: * Be mindful that the document instance must remain in scope while you are accessing object, array and value instances. * * @param out This is set to a value of the given type, parsed from the JSON. If there is an error, this may not be initialized. - * @returns INCORRECT_TYPE If the JSON value is not an object. + * @returns INCORRECT_TYPE If the JSON value is of the given type. * @returns SUCCESS If the parse succeeded and the out parameter was set to the value. */ template @@ -71023,6 +75622,36 @@ public: */ simdjson_inline simdjson_result raw_json() noexcept; +#if SIMDJSON_SUPPORTS_DESERIALIZATION + /** + * Get this object as the given type. + * + * @param out This is set to a value of the given type, parsed from the JSON. If there is an error, this may not be initialized. + * @returns INCORRECT_TYPE If the JSON object is not of the given type. + * @returns SUCCESS If the parse succeeded and the out parameter was set to the value. + */ + template + simdjson_inline error_code get(T &out) + noexcept(custom_deserializable ? nothrow_custom_deserializable : true) { + static_assert(custom_deserializable); + return deserialize(*this, out); + } + /** + * Get this array as the given type. + * + * @returns A value of the given type, parsed from the JSON. + * @returns INCORRECT_TYPE If the JSON value is not the given type. + */ + template + simdjson_inline simdjson_result get() + noexcept(custom_deserializable ? nothrow_custom_deserializable : true) + { + static_assert(std::is_default_constructible::value, "The specified type is not default constructible."); + T out{}; + SIMDJSON_TRY(get(out)); + return out; + } +#endif // SIMDJSON_SUPPORTS_DESERIALIZATION protected: /** * Go to the end of the object, no matter where you are right now. @@ -71066,12 +75695,32 @@ public: simdjson_inline simdjson_result operator[](std::string_view key) && noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - inline simdjson_result reset() noexcept; inline simdjson_result is_empty() noexcept; inline simdjson_result count_fields() & noexcept; inline simdjson_result raw_json() noexcept; + #if SIMDJSON_SUPPORTS_DESERIALIZATION + // TODO: move this code into object-inl.h + template + simdjson_inline simdjson_result get() noexcept { + if (error()) { return error(); } + if constexpr (std::is_same_v) { + return first; + } + return first.get(); + } + template + simdjson_inline error_code get(T& out) noexcept { + if (error()) { return error(); } + if constexpr (std::is_same_v) { + out = first; + } else { + SIMDJSON_TRY( first.get(out) ); + } + return SUCCESS; + } +#endif // SIMDJSON_SUPPORTS_DESERIALIZATION }; } // namespace simdjson @@ -71276,12 +75925,17 @@ inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result #include +#if SIMDJSON_STATIC_REFLECTION +#include +// #include // for std::define_static_string - header not available yet +#endif namespace simdjson { template @@ -71328,6 +75982,22 @@ error_code tag_invoke(deserialize_tag, auto &val, T &out) noexcept { return SUCCESS; } +////////////////////////////// +// String deserialization +////////////////////////////// + +// just a character! +error_code tag_invoke(deserialize_tag, auto &val, char &out) noexcept { + std::string_view x; + SIMDJSON_TRY(val.get_string().get(x)); + if(x.size() != 1) { + return INCORRECT_TYPE; + } + out = x[0]; + return SUCCESS; +} + +// any string-like type (can be constructed from std::string_view) template requires(!require_custom_serialization) error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept(std::is_nothrow_constructible_v) { @@ -71349,15 +76019,20 @@ template requires(!require_custom_serialization) error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept(false) { using value_type = typename std::remove_cvref_t::value_type; - static_assert( + /*static_assert( deserializable, - "The specified type inside the container must itself be deserializable"); + "The specified type inside the container must itself be deserializable");*/ static_assert( std::is_default_constructible_v, "The specified type inside the container must default constructible."); icelake::ondemand::array arr; - SIMDJSON_TRY(val.get_array().get(arr)); + if constexpr (std::is_same_v, icelake::ondemand::array>) { + arr = val; + } else { + SIMDJSON_TRY(val.get_array().get(arr)); + } + for (auto v : arr) { if constexpr (concepts::returns_reference) { if (auto const err = v.get().get(concepts::emplace_one(out)); @@ -71414,7 +76089,45 @@ error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept(false) { return SUCCESS; } +template +error_code tag_invoke(deserialize_tag, icelake::ondemand::object &obj, T &out) noexcept { + using value_type = typename std::remove_cvref_t::mapped_type; + out.clear(); + for (auto field : obj) { + std::string_view key; + SIMDJSON_TRY(field.unescaped_key().get(key)); + + icelake::ondemand::value value_obj; + SIMDJSON_TRY(field.value().get(value_obj)); + + value_type this_value; + SIMDJSON_TRY(value_obj.get(this_value)); + out.emplace(typename T::key_type(key), std::move(this_value)); + } + return SUCCESS; +} + +template +error_code tag_invoke(deserialize_tag, icelake::ondemand::value &val, T &out) noexcept { + icelake::ondemand::object obj; + SIMDJSON_TRY(val.get_object().get(obj)); + return simdjson::deserialize(obj, out); +} + +template +error_code tag_invoke(deserialize_tag, icelake::ondemand::document &doc, T &out) noexcept { + icelake::ondemand::object obj; + SIMDJSON_TRY(doc.get_object().get(obj)); + return simdjson::deserialize(obj, out); +} + +template +error_code tag_invoke(deserialize_tag, icelake::ondemand::document_reference &doc, T &out) noexcept { + icelake::ondemand::object obj; + SIMDJSON_TRY(doc.get_object().get(obj)); + return simdjson::deserialize(obj, out); +} /** @@ -71476,6 +76189,199 @@ error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept(nothrow_deser return SUCCESS; } + +#if SIMDJSON_STATIC_REFLECTION + + +template +constexpr bool user_defined_type = (std::is_class_v +&& !std::is_same_v && !std::is_same_v && !concepts::optional_type && +!concepts::appendable_containers && !require_custom_serialization); + + +// workaround from +// https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2024/p2996r10.html#back-and-forth +// for missing expansion statements +namespace __impl { + template + struct replicator_type { + template + constexpr void operator>>(F body) const { + (body.template operator()(), ...); + } + }; + + template + replicator_type replicator = {}; +} + +template +consteval auto expand(R range) { + std::vector args; + for (auto r : range) { + args.push_back(reflect_value(r)); + } + return substitute(^^__impl::replicator, args); +} +// end of workaround + +template + requires(user_defined_type && std::is_class_v) +error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept { + icelake::ondemand::object obj; + if constexpr (std::is_same_v, icelake::ondemand::object>) { + obj = val; + } else { + SIMDJSON_TRY(val.get_object().get(obj)); + } + error_code e = simdjson::SUCCESS; + + [:expand(std::meta::nonstatic_data_members_of(^^T, std::meta::access_context::unchecked())):] >> [&]() { + if constexpr (!std::meta::is_const(mem) && std::meta::is_public(mem)) { + constexpr std::string_view key = std::define_static_string(std::meta::identifier_of(mem)); + static_assert( + deserializable, + "The specified type inside the class must itself be deserializable"); + // as long we are succesful or the field is not found, we continue + if(e == simdjson::SUCCESS || e == simdjson::NO_SUCH_FIELD) { + obj[key].get(out.[:mem:]); + } + } + }; + return e; +} +template + requires(user_defined_type>) +error_code tag_invoke(deserialize_tag, simdjson_value &val, std::unique_ptr &out) noexcept { + if (!out) { + out = std::make_unique(); + if (!out) { + return MEMALLOC; + } + } + if (auto err = val.get(*out)) { + out.reset(); + return err; + } + return SUCCESS; +} + +template + requires(user_defined_type>) +error_code tag_invoke(deserialize_tag, simdjson_value &val, std::shared_ptr &out) noexcept { + if (!out) { + out = std::make_shared(); + if (!out) { + return MEMALLOC; + } + } + if (auto err = val.get(*out)) { + out.reset(); + return err; + } + return SUCCESS; +} + +#endif // SIMDJSON_STATIC_REFLECTION + +//////////////////////////////////////// +// Unique pointers +//////////////////////////////////////// +error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) noexcept { + if (!out) { + out = std::make_unique(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_bool().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) noexcept { + if (!out) { + out = std::make_unique(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_int64().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) noexcept { + if (!out) { + out = std::make_unique(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_uint64().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) noexcept { + if (!out) { + out = std::make_unique(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_double().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) noexcept { + if (!out) { + out = std::make_unique(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_string().get(*out)); + return SUCCESS; +} + + +//////////////////////////////////////// +// Shared pointers +//////////////////////////////////////// +error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) noexcept { + if (!out) { + out = std::make_shared(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_bool().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) noexcept { + if (!out) { + out = std::make_shared(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_int64().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) noexcept { + if (!out) { + out = std::make_shared(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_uint64().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) noexcept { + if (!out) { + out = std::make_shared(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_double().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) noexcept { + if (!out) { + out = std::make_shared(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_string().get(*out)); + return SUCCESS; +} + + } // namespace simdjson #endif // SIMDJSON_ONDEMAND_DESERIALIZE_H @@ -74963,6 +79869,7 @@ simdjson_inline simdjson_result simdjson_result::simdjson_res #endif // SIMDJSON_GENERIC_ONDEMAND_VALUE_ITERATOR_INL_H /* end file simdjson/generic/ondemand/value_iterator-inl.h for icelake */ +// JSON builder, ideally they should not be part of the ondemand directory +// but it is convenient for now to have them here. +/* including simdjson/generic/ondemand/json_string_builder.h for icelake: #include "simdjson/generic/ondemand/json_string_builder.h" */ +/* begin file simdjson/generic/ondemand/json_string_builder.h for icelake */ +/** + * This file is part of the builder API. It is temporarily in the ondemand directory + * but we will move it to a builder directory later. + */ +#ifndef SIMDJSON_GENERIC_STRING_BUILDER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { +namespace icelake { +namespace builder { + +/** + * A builder for JSON strings representing documents. This is a low-level + * builder that is not meant to be used directly by end-users. Though it + * supports atomic types (Booleans, strings), it does not support composed + * types (arrays and objects). + * + * Ultimately, this class should support kernel-specific optimizations. E.g., + * it may make use of SIMD instructions to escape strings faster. + */ +class string_builder { +public: + simdjson_inline string_builder(size_t initial_capacity = 1024); + + /** + * Append number (includes Booleans). Booleans are mapped to the strings + * false and true. Numbers are converted to strings abiding by the JSON standard. + * Floating-point numbers are converted to the shortest string that 'correctly' + * represents the number. + */ + template::value>::type> + simdjson_inline void append(number_type v) noexcept; + + /** + * Append character c. + */ + simdjson_inline void append(char c) noexcept; + + /** + * Append the string 'null'. + */ + simdjson_inline void append_null() noexcept; + + /** + * Clear the content. + */ + simdjson_inline void clear() noexcept; + + /** + * Append the std::string_view, after escaping it. + * There is no UTF-8 validation. + */ + simdjson_inline void escape_and_append(std::string_view input) noexcept; + + /** + * Append the std::string_view surrounded by double quotes, after escaping it. + * There is no UTF-8 validation. + */ + simdjson_inline void escape_and_append_with_quotes(std::string_view input) noexcept; + + /** + * Append the character surrounded by double quotes, after escaping it. + * There is no UTF-8 validation. + */ + simdjson_inline void escape_and_append_with_quotes(char input) noexcept; + + /** + * Append the character surrounded by double quotes, after escaping it. + * There is no UTF-8 validation. + */ + simdjson_inline void escape_and_append_with_quotes(const char* input) noexcept; + + /** + * Append the C string directly, without escaping. + * There is no UTF-8 validation. + */ + simdjson_inline void append_raw(const char *c) noexcept; + + /** + * Append "{" to the buffer. + */ + simdjson_inline void start_object() noexcept; + + /** + * Append "}" to the buffer. + */ + simdjson_inline void end_object() noexcept; + + /** + * Append "[" to the buffer. + */ + simdjson_inline void start_array() noexcept; + + /** + * Append "]" to the buffer. + */ + simdjson_inline void end_array() noexcept; + + /** + * Append "," to the buffer. + */ + simdjson_inline void append_comma() noexcept; + + /** + * Append ":" to the buffer. + */ + simdjson_inline void append_colon() noexcept; + + /** + * Append a key-value pair to the buffer. + * The key is escaped and surrounded by double quotes. + * The value is escaped if it is a string. + */ + template + simdjson_inline void append_key_value(key_type key, value_type value) noexcept; + /** + * Append the std::string_view directly, without escaping. + * There is no UTF-8 validation. + */ + simdjson_inline void append_raw(std::string_view input) noexcept; + + /** + * Append len characters from str. + * There is no UTF-8 validation. + */ + simdjson_inline void append_raw(const char *str, size_t len) noexcept; +#if SIMDJSON_EXCEPTIONS + /** + * Creates an std::string from the written JSON buffer. + * Throws if memory allocation failed + * + * The result may not be valid UTF-8 if some of your content was not valid UTF-8. + * Use validate_unicode() to check the content if needed. + */ + simdjson_inline operator std::string() const noexcept(false); + + /** + * Creates an std::string_view from the written JSON buffer. + * Throws if memory allocation failed. + * + * The result may not be valid UTF-8 if some of your content was not valid UTF-8. + * Use validate_unicode() to check the content if needed. + */ + simdjson_inline operator std::string_view() const noexcept(false); +#endif + + /** + * Returns a view on the written JSON buffer. Returns an error + * if memory allocation failed. + * + * The result may not be valid UTF-8 if some of your content was not valid UTF-8. + * Use validate_unicode() to check the content. + */ + simdjson_inline simdjson_result view() const noexcept; + + /** + * Appends the null character to the buffer and returns + * a pointer to the beginning of the written JSON buffer. + * Returns an error if memory allocation failed. + * The result is null-terminated. + * + * The result may not be valid UTF-8 if some of your content was not valid UTF-8. + * Use validate_unicode() to check the content. + */ + simdjson_inline simdjson_result c_str() noexcept; + + /** + * Return true if the content is valid UTF-8. + */ + simdjson_inline bool validate_unicode() const noexcept; + + /** + * Returns the current size of the written JSON buffer. + * If an error occurred, returns 0. + */ + simdjson_inline size_t size() const noexcept; + +private: + /** + * Returns true if we can write at least upcoming_bytes bytes. + * The underlying buffer is reallocated if needed. It is designed + * to be called before writing to the buffer. It should be fast. + */ + simdjson_inline bool capacity_check(size_t upcoming_bytes); + + /** + * Grow the buffer to at least desired_capacity bytes. + * If the allocation fails, is_valid is set to false. We expect + * that this function would not be repeatedly called. + */ + simdjson_inline void grow_buffer(size_t desired_capacity); + + /** + * We use this helper function to make sure that is_valid is kept consistent. + */ + simdjson_inline void set_valid(bool valid) noexcept; + + std::unique_ptr buffer{}; + size_t position{0}; + size_t capacity{0}; + bool is_valid{true}; +}; + + + +} +} +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_STRING_BUILDER_H +/* end file simdjson/generic/ondemand/json_string_builder.h for icelake */ +/* including simdjson/generic/ondemand/json_string_builder-inl.h for icelake: #include "simdjson/generic/ondemand/json_string_builder-inl.h" */ +/* begin file simdjson/generic/ondemand/json_string_builder-inl.h for icelake */ +/** + * This file is part of the builder API. It is temporarily in the ondemand + * directory but we will move it to a builder directory later. + */ +#include +#include +#include +#ifndef SIMDJSON_GENERIC_STRING_BUILDER_INL_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_INL_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +/* + * Empirically, we have found that an inlined optimization is important for + * performance. The following macros are not ideal. We should find a better + * way to inline the code. + */ + +#if defined(__SSE2__) || defined(__x86_64__) || defined(__x86_64) || \ + (defined(_M_AMD64) || defined(_M_X64) || \ + (defined(_M_IX86_FP) && _M_IX86_FP == 2)) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_SSE2 +#define SIMDJSON_EXPERIMENTAL_HAS_SSE2 1 +#endif +#endif + +#if defined(__aarch64__) || defined(_M_ARM64) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_NEON +#define SIMDJSON_EXPERIMENTAL_HAS_NEON 1 +#endif +#endif +#if SIMDJSON_EXPERIMENTAL_HAS_NEON +#include +#endif +#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 +#include +#endif + +namespace simdjson { +namespace icelake { +namespace builder { + +static SIMDJSON_CONSTEXPR_LAMBDA std::array json_quotable_character = { + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + +/** + +A possible SWAR implementation of has_json_escapable_byte. It is not used because +it is slower than the current implementation. It is kept here for reference (to show +that we tried it). + +inline bool has_json_escapable_byte(uint64_t x) { + uint64_t is_ascii = 0x8080808080808080ULL & ~x; + uint64_t xor2 = x ^ 0x0202020202020202ULL; + uint64_t lt32_or_eq34 = xor2 - 0x2121212121212121ULL; + uint64_t sub92 = x ^ 0x5C5C5C5C5C5C5C5CULL; + uint64_t eq92 = (sub92 - 0x0101010101010101ULL); + return ((lt32_or_eq34 | eq92) & is_ascii) != 0; +} + +**/ + +SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline bool +simple_needs_escaping(std::string_view v) { + for (char c : v) { + // a table lookup is faster than a series of comparisons + if(json_quotable_character[static_cast(c)]) { + return true; + } + } + return false; +} + +#if SIMDJSON_EXPERIMENTAL_HAS_NEON +simdjson_inline bool fast_needs_escaping(std::string_view view) { + if (view.size() < 16) { + return simple_needs_escaping(view); + } + size_t i = 0; + uint8x16_t running = vdupq_n_u8(0); + uint8x16_t v34 = vdupq_n_u8(34); + uint8x16_t v92 = vdupq_n_u8(92); + + for (; i + 15 < view.size(); i += 16) { + uint8x16_t word = vld1q_u8((const uint8_t *)view.data() + i); + running = vorrq_u8(running, vceqq_u8(word, v34)); + running = vorrq_u8(running, vceqq_u8(word, v92)); + running = vorrq_u8(running, vcltq_u8(word, vdupq_n_u8(32))); + } + if (i < view.size()) { + uint8x16_t word = + vld1q_u8((const uint8_t *)view.data() + view.length() - 16); + running = vorrq_u8(running, vceqq_u8(word, v34)); + running = vorrq_u8(running, vceqq_u8(word, v92)); + running = vorrq_u8(running, vcltq_u8(word, vdupq_n_u8(32))); + } + return vmaxvq_u32(vreinterpretq_u32_u8(running)) != 0; +} +#elif SIMDJSON_EXPERIMENTAL_HAS_SSE2 +simdjson_inline bool fast_needs_escaping(std::string_view view) { + if (view.size() < 16) { + return simple_needs_escaping(view); + } + size_t i = 0; + __m128i running = _mm_setzero_si128(); + for (; i + 15 < view.size(); i += 16) { + + __m128i word = _mm_loadu_si128(reinterpret_cast(view.data() + i)); + running = _mm_or_si128(running, _mm_cmpeq_epi8(word, _mm_set1_epi8(34))); + running = _mm_or_si128(running, _mm_cmpeq_epi8(word, _mm_set1_epi8(92))); + running = _mm_or_si128( + running, _mm_cmpeq_epi8(_mm_subs_epu8(word, _mm_set1_epi8(31)), + _mm_setzero_si128())); + } + if (i < view.size()) { + __m128i word = + _mm_loadu_si128(reinterpret_cast(view.data() + view.length() - 16)); + running = _mm_or_si128(running, _mm_cmpeq_epi8(word, _mm_set1_epi8(34))); + running = _mm_or_si128(running, _mm_cmpeq_epi8(word, _mm_set1_epi8(92))); + running = _mm_or_si128( + running, _mm_cmpeq_epi8(_mm_subs_epu8(word, _mm_set1_epi8(31)), + _mm_setzero_si128())); + } + return _mm_movemask_epi8(running) != 0; +} +#else +simdjson_inline bool fast_needs_escaping(std::string_view view) { + return simple_needs_escaping(view); +} +#endif + + +SIMDJSON_CONSTEXPR_LAMBDA inline size_t +find_next_json_quotable_character(const std::string_view view, + size_t location) noexcept { + + for (auto pos = view.begin() + location; pos != view.end(); ++pos) { + if (json_quotable_character[static_cast(*pos)]) { + return pos - view.begin(); + } + } + return size_t(view.size()); +} + +SIMDJSON_CONSTEXPR_LAMBDA static std::string_view control_chars[] = { + "\\x0000", "\\x0001", "\\x0002", "\\x0003", "\\x0004", "\\x0005", "\\x0006", + "\\x0007", "\\x0008", "\\t", "\\n", "\\x000b", "\\f", "\\r", + "\\x000e", "\\x000f", "\\x0010", "\\x0011", "\\x0012", "\\x0013", "\\x0014", + "\\x0015", "\\x0016", "\\x0017", "\\x0018", "\\x0019", "\\x001a", "\\x001b", + "\\x001c", "\\x001d", "\\x001e", "\\x001f"}; + +SIMDJSON_CONSTEXPR_LAMBDA void escape_json_char(char c, char *&out) { + if (c == '"') { + memcpy(out, "\\\"", 2); + out += 2; + } else if (c == '\\') { + memcpy(out, "\\\\", 2); + out += 2; + } else { + std::string_view v = control_chars[uint8_t(c)]; + memcpy(out, v.data(), v.size()); + out += v.size(); + } +} + +inline size_t write_string_escaped(const std::string_view input, char *out) { + size_t mysize = input.size(); + if (!fast_needs_escaping(input)) { // fast path! + memcpy(out, input.data(), input.size()); + return input.size(); + } + const char *const initout = out; + size_t location = find_next_json_quotable_character(input, 0); + memcpy(out, input.data(), location); + out += location; + escape_json_char(input[location], out); + location += 1; + while (location < mysize) { + size_t newlocation = find_next_json_quotable_character(input, location); + memcpy(out, input.data() + location, newlocation - location); + out += newlocation - location; + location = newlocation; + if (location == mysize) { + break; + } + escape_json_char(input[location], out); + location += 1; + } + return out - initout; +} + +#if SIMDJSON_CONSTEVAL +// unoptimized, meant for compile-time execution +consteval std::string consteval_to_quoted_escaped(std::string_view input) { + std::string out = "\""; + for (char c : input) { + if (json_quotable_character[uint8_t(c)]) { + if (c == '"') { + out.append("\\\""); + } else if (c == '\\') { + out.append("\\\\"); + } else { + std::string_view v = control_chars[uint8_t(c)]; + out.append(v); + } + } else { + out.push_back(c); + } + } + out.push_back('"'); + return out; +} +#endif // SIMDJSON_CONSTEVAL + +simdjson_inline string_builder::string_builder(size_t initial_capacity) + : buffer(new(std::nothrow) char[initial_capacity]), position(0), + capacity(buffer.get() != nullptr ? initial_capacity : 0), + is_valid(buffer.get() != nullptr) {} + +simdjson_inline bool string_builder::capacity_check(size_t upcoming_bytes) { + // We use the convention that when is_valid is false, then the capacity and + // the position are 0. + // Most of the time, this function will return true. + if (simdjson_likely(upcoming_bytes <= capacity - position)) { + return true; + } + // check for overflow, most of the time there is no overflow + if (simdjson_likely(position + upcoming_bytes < position)) { + return false; + } + // We will rarely get here. + grow_buffer((std::max)(capacity * 2, position + upcoming_bytes)); + // If the buffer allocation failed, we set is_valid to false. + return is_valid; +} + +simdjson_inline void string_builder::grow_buffer(size_t desired_capacity) { + if (!is_valid) { + return; + } + std::unique_ptr new_buffer(new (std::nothrow) char[desired_capacity]); + if (new_buffer.get() == nullptr) { + set_valid(false); + return; + } + std::memcpy(new_buffer.get(), buffer.get(), position); + buffer.swap(new_buffer); + capacity = desired_capacity; +} + +simdjson_inline void string_builder::set_valid(bool valid) noexcept { + if (!valid) { + is_valid = false; + capacity = 0; + position = 0; + buffer.reset(); + } else { + is_valid = true; + } +} + +simdjson_inline size_t string_builder::size() const noexcept { + return position; +} + +simdjson_inline void string_builder::append(char c) noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = c; + } +} + +simdjson_inline void string_builder::append_null() noexcept { + constexpr char null_literal[] = "null"; + constexpr size_t null_len = sizeof(null_literal) - 1; + if (capacity_check(null_len)) { + std::memcpy(buffer.get() + position, null_literal, null_len); + position += null_len; + } +} + +simdjson_inline void string_builder::clear() noexcept { + position = 0; + // if it was invalid, we should try to repair it + if (!is_valid) { + capacity = 0; + buffer.reset(); + is_valid = true; + } +} + +namespace internal { + +// We could specialize further for 32-bit integers. +int int_log2(uint32_t x) { return (63 - leading_zeroes(x | 1)); } + +int fast_digit_count(uint32_t x) { + static uint64_t table[] = { + 4294967296, 8589934582, 8589934582, 8589934582, 12884901788, + 12884901788, 12884901788, 17179868184, 17179868184, 17179868184, + 21474826480, 21474826480, 21474826480, 21474826480, 25769703776, + 25769703776, 25769703776, 30063771072, 30063771072, 30063771072, + 34349738368, 34349738368, 34349738368, 34349738368, 38554705664, + 38554705664, 38554705664, 41949672960, 41949672960, 41949672960, + 42949672960, 42949672960}; + return uint32_t((x + table[int_log2(x)]) >> 32); +} + +int int_log2(uint64_t x) { return 63 - leading_zeroes(x | 1); } + +int fast_digit_count(uint64_t x) { + static uint64_t table[] = {9, + 99, + 999, + 9999, + 99999, + 999999, + 9999999, + 99999999, + 999999999, + 9999999999, + 99999999999, + 999999999999, + 9999999999999, + 99999999999999, + 999999999999999ULL, + 9999999999999999ULL, + 99999999999999999ULL, + 999999999999999999ULL, + 9999999999999999999ULL}; + int y = (19 * int_log2(x) >> 6); + y += x > table[y]; + return y + 1; +} + +template ::value>::type> +simdjson_inline size_t digit_count(number_type v) noexcept { + static_assert(sizeof(number_type) == 8 || sizeof(number_type) == 4 || + sizeof(number_type) == 2 || sizeof(number_type) == 1, + "We only support 8-bit, 16-bit, 32-bit and 64-bit numbers"); + return fast_digit_count(v); +} +static const char decimal_table[200] = { + 0x30, 0x30, 0x30, 0x31, 0x30, 0x32, 0x30, 0x33, 0x30, 0x34, 0x30, 0x35, + 0x30, 0x36, 0x30, 0x37, 0x30, 0x38, 0x30, 0x39, 0x31, 0x30, 0x31, 0x31, + 0x31, 0x32, 0x31, 0x33, 0x31, 0x34, 0x31, 0x35, 0x31, 0x36, 0x31, 0x37, + 0x31, 0x38, 0x31, 0x39, 0x32, 0x30, 0x32, 0x31, 0x32, 0x32, 0x32, 0x33, + 0x32, 0x34, 0x32, 0x35, 0x32, 0x36, 0x32, 0x37, 0x32, 0x38, 0x32, 0x39, + 0x33, 0x30, 0x33, 0x31, 0x33, 0x32, 0x33, 0x33, 0x33, 0x34, 0x33, 0x35, + 0x33, 0x36, 0x33, 0x37, 0x33, 0x38, 0x33, 0x39, 0x34, 0x30, 0x34, 0x31, + 0x34, 0x32, 0x34, 0x33, 0x34, 0x34, 0x34, 0x35, 0x34, 0x36, 0x34, 0x37, + 0x34, 0x38, 0x34, 0x39, 0x35, 0x30, 0x35, 0x31, 0x35, 0x32, 0x35, 0x33, + 0x35, 0x34, 0x35, 0x35, 0x35, 0x36, 0x35, 0x37, 0x35, 0x38, 0x35, 0x39, + 0x36, 0x30, 0x36, 0x31, 0x36, 0x32, 0x36, 0x33, 0x36, 0x34, 0x36, 0x35, + 0x36, 0x36, 0x36, 0x37, 0x36, 0x38, 0x36, 0x39, 0x37, 0x30, 0x37, 0x31, + 0x37, 0x32, 0x37, 0x33, 0x37, 0x34, 0x37, 0x35, 0x37, 0x36, 0x37, 0x37, + 0x37, 0x38, 0x37, 0x39, 0x38, 0x30, 0x38, 0x31, 0x38, 0x32, 0x38, 0x33, + 0x38, 0x34, 0x38, 0x35, 0x38, 0x36, 0x38, 0x37, 0x38, 0x38, 0x38, 0x39, + 0x39, 0x30, 0x39, 0x31, 0x39, 0x32, 0x39, 0x33, 0x39, 0x34, 0x39, 0x35, + 0x39, 0x36, 0x39, 0x37, 0x39, 0x38, 0x39, 0x39, +}; +} // namespace internal + +template +simdjson_inline void string_builder::append(number_type v) noexcept { + static_assert(std::is_same::value || + std::is_integral::value || + std::is_floating_point::value, + "Unsupported number type"); + // If C++17 is available, we can 'if constexpr' here. + SIMDJSON_IF_CONSTEXPR(std::is_same::value) { + if (v) { + constexpr char true_literal[] = "true"; + constexpr size_t true_len = sizeof(true_literal) - 1; + if (capacity_check(true_len)) { + std::memcpy(buffer.get() + position, true_literal, true_len); + position += true_len; + } + } else { + constexpr char false_literal[] = "false"; + constexpr size_t false_len = sizeof(false_literal) - 1; + if (capacity_check(false_len)) { + std::memcpy(buffer.get() + position, false_literal, false_len); + position += false_len; + } + } + } + else SIMDJSON_IF_CONSTEXPR(std::is_unsigned::value) { + constexpr size_t max_number_size = 20; + if (capacity_check(max_number_size)) { + using unsigned_type = typename std::make_unsigned::type; + unsigned_type pv = static_cast(v); + size_t dc = internal::digit_count(pv); + char *write_pointer = buffer.get() + position + dc - 1; + while (pv >= 100) { + memcpy(write_pointer - 1, &internal::decimal_table[(pv % 100)*2], 2); + write_pointer -= 2; + pv /= 100; + } + if (pv >= 10) { + *write_pointer-- = char('0' + (pv % 10)); + pv /= 10; + } + *write_pointer = char('0' + pv); + position += dc; + } + } + else SIMDJSON_IF_CONSTEXPR(std::is_integral::value) { + constexpr size_t max_number_size = 20; + if (capacity_check(max_number_size)) { + using unsigned_type = typename std::make_unsigned::type; + bool negative = v < 0; + unsigned_type pv = static_cast(v); + if (negative) { + pv = 0 - pv; // the 0 is for Microsoft + } + size_t dc = internal::digit_count(pv); + if (negative) { + buffer.get()[position++] = '-'; + } + char *write_pointer = buffer.get() + position + dc - 1; + while (pv >= 100) { + memcpy(write_pointer - 1, &internal::decimal_table[(pv % 100)*2], 2); + write_pointer -= 2; + pv /= 100; + } + if (pv >= 10) { + *write_pointer-- = char('0' + (pv % 10)); + pv /= 10; + } + *write_pointer = char('0' + pv); + position += dc; + } + } + else SIMDJSON_IF_CONSTEXPR(std::is_floating_point::value) { + constexpr size_t max_number_size = 24; + if (capacity_check(max_number_size)) { + // We could specialize for float. + char *end = simdjson::internal::to_chars(buffer.get() + position, nullptr, + double(v)); + position = end - buffer.get(); + } + } +} + +simdjson_inline void +string_builder::escape_and_append(std::string_view input) noexcept { + // escaping might turn a control character into \x00xx so 6 characters. + if (capacity_check(6 * input.size())) { + position += write_string_escaped(input, buffer.get() + position); + } +} + +simdjson_inline void +string_builder::escape_and_append_with_quotes(std::string_view input) noexcept { + // escaping might turn a control character into \x00xx so 6 characters. + if (capacity_check(2 + 6 * input.size())) { + buffer.get()[position++] = '"'; + position += write_string_escaped(input, buffer.get() + position); + buffer.get()[position++] = '"'; + } +} + +simdjson_inline void +string_builder::escape_and_append_with_quotes(char input) noexcept { + // escaping might turn a control character into \x00xx so 6 characters. + if (capacity_check(2 + 6 * 1)) { + buffer.get()[position++] = '"'; + std::string_view cinput(&input, 1); + position += write_string_escaped(cinput, buffer.get() + position); + buffer.get()[position++] = '"'; + } +} + +simdjson_inline void string_builder::escape_and_append_with_quotes(const char* input) noexcept { + std::string_view cinput(input); + escape_and_append_with_quotes(cinput); +} + +simdjson_inline void string_builder::append_raw(const char *c) noexcept { + size_t len = std::strlen(c); + append_raw(c, len); +} + +simdjson_inline void +string_builder::append_raw(std::string_view input) noexcept { + if (capacity_check(input.size())) { + std::memcpy(buffer.get() + position, input.data(), input.size()); + position += input.size(); + } +} + +simdjson_inline void string_builder::append_raw(const char *str, + size_t len) noexcept { + if (capacity_check(len)) { + std::memcpy(buffer.get() + position, str, len); + position += len; + } +} + +#if SIMDJSON_EXCEPTIONS +simdjson_inline string_builder::operator std::string() const noexcept(false) { + return std::string(std::string_view()); +} + +simdjson_inline string_builder::operator std::string_view() const + noexcept(false) { + return view(); +} +#endif + +simdjson_inline simdjson_result +string_builder::view() const noexcept { + if (!is_valid) { + return simdjson::OUT_OF_CAPACITY; + } + return std::string_view(buffer.get(), position); +} + +simdjson_inline simdjson_result string_builder::c_str() noexcept { + if (capacity_check(1)) { + buffer.get()[position] = '\0'; + return buffer.get(); + } + return simdjson::OUT_OF_CAPACITY; +} + +simdjson_inline bool string_builder::validate_unicode() const noexcept { + return simdjson::validate_utf8(buffer.get(), position); +} + +simdjson_inline void string_builder::start_object() noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = '{'; + } +} + + +simdjson_inline void string_builder::end_object() noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = '}'; + } +} + + +simdjson_inline void string_builder::start_array() noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = '['; + } +} + + +simdjson_inline void string_builder::end_array() noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = ']'; + } +} + + +simdjson_inline void string_builder::append_comma() noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = ','; + } +} + + +simdjson_inline void string_builder::append_colon() noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = ':'; + } +} + +template +simdjson_inline void string_builder::append_key_value(key_type key, value_type value) noexcept { + static_assert( + std::is_arithmetic::value || + std::is_same::value || + std::is_same::value || + std::is_convertible::value || + std::is_same::value, + "Unsupported value type"); + static_assert( + std::is_same::value || + std::is_convertible::value, + "Unsupported key type"); + escape_and_append_with_quotes(key); + append_colon(); + SIMDJSON_IF_CONSTEXPR(std::is_same::value) { + append_null(); + } else SIMDJSON_IF_CONSTEXPR(std::is_same::value) { + escape_and_append_with_quotes(value); + } else SIMDJSON_IF_CONSTEXPR(std::is_convertible::value) { + escape_and_append_with_quotes(value); + } else SIMDJSON_IF_CONSTEXPR(std::is_same::value) { + escape_and_append_with_quotes(value); + } else { + append(value); + } +} + +} // namespace builder +} // namespace icelake +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_STRING_BUILDER_INL_H +/* end file simdjson/generic/ondemand/json_string_builder-inl.h for icelake */ +/* including simdjson/generic/ondemand/json_builder.h for icelake: #include "simdjson/generic/ondemand/json_builder.h" */ +/* begin file simdjson/generic/ondemand/json_builder.h for icelake */ +/** + * This file is part of the builder API. It is temporarily in the ondemand directory + * but we will move it to a builder directory later. + */ +#ifndef SIMDJSON_GENERIC_BUILDER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/concepts.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ +#if SIMDJSON_STATIC_REFLECTION + +#include +#include +#include +#include +#include +#include +// #include // for std::define_static_string - header not available yet + +namespace simdjson { +namespace icelake { +namespace builder { + +// Concept that checks if a type is a container but not a string (because +// strings handling must be handled differently) +template +concept container_but_not_string = + requires(T a) { + { a.size() } -> std::convertible_to; + { + a[std::declval()] + }; // check if elements are accessible for the subscript operator + } && !std::is_same_v && + !std::is_same_v && !std::is_same_v; + +template + requires(container_but_not_string) +constexpr void atom(string_builder &b, const T &t) { + if (t.size() == 0) { + b.append_raw("[]"); + return; + } + b.append('['); + atom(b, t[0]); + for (size_t i = 1; i < t.size(); ++i) { + b.append(','); + atom(b, t[i]); + } + b.append(']'); +} + +template + requires(std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v) +constexpr void atom(string_builder &b, const T &t) { + b.escape_and_append_with_quotes(t); +} + +template +constexpr void atom(string_builder &b, const T &m) { + if (m.empty()) { + b.append_raw("{}"); + return; + } + b.append('{'); + bool first = true; + for (const auto& [key, value] : m) { + if (!first) { + b.append(','); + } + first = false; + // Keys must be convertible to string_view per the concept + b.escape_and_append_with_quotes(key); + b.append(':'); + atom(b, value); + } + b.append('}'); +} + + +template::value && !std::is_same_v>::type> +constexpr void atom(string_builder &b, const number_type t) { + b.append(t); +} +#if SIMDJSON_CONSTEVAL +consteval std::string consteval_to_quoted_escaped(std::string_view input); +#endif + +template + requires(std::is_class_v && !container_but_not_string && + !concepts::string_view_keyed_map && + !std::is_same_v && + !std::is_same_v) +constexpr void atom(string_builder &b, const T &t) { + int i = 0; + b.append('{'); + [:expand(std::meta::nonstatic_data_members_of(^^T, std::meta::access_context::unchecked())):] >> [&]() { + if (i != 0) + b.append(','); + constexpr auto key = std::define_static_string(consteval_to_quoted_escaped(std::meta::identifier_of(dm))); + b.append_raw(key); + b.append(':'); + atom(b, t.[:dm:]); + i++; + }; + b.append('}'); +} + +// works for struct +template void append(string_builder &b, const Z &z) { + int i = 0; + b.append('{'); + [:expand(std::meta::nonstatic_data_members_of(^^Z, std::meta::access_context::unchecked())):] >> [&]() { + if (i != 0) + b.append(','); + constexpr auto key = std::define_static_string(consteval_to_quoted_escaped(std::meta::identifier_of(dm))); + b.append_raw(key); + b.append(':'); + atom(b, z.[:dm:]); + i++; + }; + b.append('}'); +} + +// works for container +template + requires(container_but_not_string) +void append(string_builder &b, const Z &z) { + if (z.size() == 0) { + b.append_raw("[]"); + return; + } + b.append('['); + atom(b, z[0]); + for (size_t i = 1; i < z.size(); ++i) { + b.append(','); + atom(b, z[i]); + } + b.append(']'); +} + +template +simdjson_result to_json_string(const Z &z) { + string_builder b; + append(b, z); + std::string_view s; + if(auto e = b.view().get(s); e) { return e; } + return std::string(s); +} + +template +simdjson_error to_json(const Z &z, std::string &s) { + string_builder b; + append(b, z); + std::string_view view; + if(auto e = b.view().get(view); e) { return e; } + s.assign(view); + return SUCCESS; +} +} // namespace json_builder +} // namespace icelake +} // namespace simdjson +#endif // SIMDJSON_STATIC_REFLECTION + +#endif +/* end file simdjson/generic/ondemand/json_builder.h for icelake */ /* end file simdjson/generic/ondemand/amalgamated.h for icelake */ /* including simdjson/icelake/end.h: #include "simdjson/icelake/end.h" */ @@ -77783,6 +83703,32 @@ backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { }; } + +struct escaping { + static constexpr uint32_t BYTES_PROCESSED = 16; + simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst); + + simdjson_inline bool has_escape() { return escape_bits != 0; } + simdjson_inline int escape_index() { return trailing_zeroes(escape_bits); } + + uint64_t escape_bits; +}; // struct escaping + + + +simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) { + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v(src); + v.store(dst); + simd8 is_quote = (v == '"'); + simd8 is_backslash = (v == '\\'); + simd8 is_control = (v < 32); + return { + // We store it as a 64-bit bitmask even though we only need 16 bits. + uint64_t((is_backslash | is_quote | is_control).to_bitmask()) + }; +} + } // unnamed namespace } // namespace ppc64 } // namespace simdjson @@ -77941,10 +83887,26 @@ concept nothrow_deserializable = nothrow_custom_deserializable || is_bu /// Deserialize Tag inline constexpr struct deserialize_tag { + using array_type = ppc64::ondemand::array; + using object_type = ppc64::ondemand::object; using value_type = ppc64::ondemand::value; using document_type = ppc64::ondemand::document; using document_reference_type = ppc64::ondemand::document_reference; + // Customization Point for array + template + requires custom_deserializable + [[nodiscard]] constexpr /* error_code */ auto operator()(array_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + return tag_invoke(*this, object, output); + } + + // Customization Point for object + template + requires custom_deserializable + [[nodiscard]] constexpr /* error_code */ auto operator()(object_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + return tag_invoke(*this, object, output); + } + // Customization Point for value template requires custom_deserializable @@ -78512,6 +84474,7 @@ public: * * You may use get_double(), get_bool(), get_uint64(), get_int64(), * get_object(), get_array(), get_raw_json_string(), or get_string() instead. + * When SIMDJSON_SUPPORTS_DESERIALIZATION is set, custom types are also supported. * * @returns A value of the given type, parsed from the JSON. * @returns INCORRECT_TYPE If the JSON value is not the given type. @@ -78535,6 +84498,7 @@ public: * Get this value as the given type. * * Supported types: object, array, raw_json_string, string_view, uint64_t, int64_t, double, bool + * If the macro SIMDJSON_SUPPORTS_DESERIALIZATION is set, then custom types are also supported. * * @param out This is set to a value of the given type, parsed from the JSON. If there is an error, this may not be initialized. * @returns INCORRECT_TYPE If the JSON value is not an object. @@ -80777,6 +86741,37 @@ public: * - INDEX_OUT_OF_BOUNDS if the array index is larger than an array length */ simdjson_inline simdjson_result at(size_t index) noexcept; + +#if SIMDJSON_SUPPORTS_DESERIALIZATION + /** + * Get this array as the given type. + * + * @param out This is set to a value of the given type, parsed from the JSON. If there is an error, this may not be initialized. + * @returns INCORRECT_TYPE If the JSON array is not of the given type. + * @returns SUCCESS If the parse succeeded and the out parameter was set to the value. + */ + template + simdjson_inline error_code get(T &out) + noexcept(custom_deserializable ? nothrow_custom_deserializable : true) { + static_assert(custom_deserializable); + return deserialize(*this, out); + } + /** + * Get this array as the given type. + * + * @returns A value of the given type, parsed from the JSON. + * @returns INCORRECT_TYPE If the JSON value is not the given type. + */ + template + simdjson_inline simdjson_result get() + noexcept(custom_deserializable ? nothrow_custom_deserializable : true) + { + static_assert(std::is_default_constructible::value, "The specified type is not default constructible."); + T out{}; + SIMDJSON_TRY(get(out)); + return out; + } +#endif // SIMDJSON_SUPPORTS_DESERIALIZATION protected: /** * Go to the end of the array, no matter where you are right now. @@ -80855,7 +86850,28 @@ public: simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; simdjson_inline simdjson_result raw_json() noexcept; +#if SIMDJSON_SUPPORTS_DESERIALIZATION + // TODO: move this code into object-inl.h + template + simdjson_inline simdjson_result get() noexcept { + if (error()) { return error(); } + if constexpr (std::is_same_v) { + return first; + } + return first.get(); + } + template + simdjson_inline error_code get(T& out) noexcept { + if (error()) { return error(); } + if constexpr (std::is_same_v) { + out = first; + } else { + SIMDJSON_TRY( first.get(out) ); + } + return SUCCESS; + } +#endif // SIMDJSON_SUPPORTS_DESERIALIZATION }; } // namespace simdjson @@ -81189,7 +87205,7 @@ public: * Be mindful that the document instance must remain in scope while you are accessing object, array and value instances. * * @param out This is set to a value of the given type, parsed from the JSON. If there is an error, this may not be initialized. - * @returns INCORRECT_TYPE If the JSON value is not an object. + * @returns INCORRECT_TYPE If the JSON value is of the given type. * @returns SUCCESS If the parse succeeded and the out parameter was set to the value. */ template @@ -82676,6 +88692,36 @@ public: */ simdjson_inline simdjson_result raw_json() noexcept; +#if SIMDJSON_SUPPORTS_DESERIALIZATION + /** + * Get this object as the given type. + * + * @param out This is set to a value of the given type, parsed from the JSON. If there is an error, this may not be initialized. + * @returns INCORRECT_TYPE If the JSON object is not of the given type. + * @returns SUCCESS If the parse succeeded and the out parameter was set to the value. + */ + template + simdjson_inline error_code get(T &out) + noexcept(custom_deserializable ? nothrow_custom_deserializable : true) { + static_assert(custom_deserializable); + return deserialize(*this, out); + } + /** + * Get this array as the given type. + * + * @returns A value of the given type, parsed from the JSON. + * @returns INCORRECT_TYPE If the JSON value is not the given type. + */ + template + simdjson_inline simdjson_result get() + noexcept(custom_deserializable ? nothrow_custom_deserializable : true) + { + static_assert(std::is_default_constructible::value, "The specified type is not default constructible."); + T out{}; + SIMDJSON_TRY(get(out)); + return out; + } +#endif // SIMDJSON_SUPPORTS_DESERIALIZATION protected: /** * Go to the end of the object, no matter where you are right now. @@ -82719,12 +88765,32 @@ public: simdjson_inline simdjson_result operator[](std::string_view key) && noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - inline simdjson_result reset() noexcept; inline simdjson_result is_empty() noexcept; inline simdjson_result count_fields() & noexcept; inline simdjson_result raw_json() noexcept; + #if SIMDJSON_SUPPORTS_DESERIALIZATION + // TODO: move this code into object-inl.h + template + simdjson_inline simdjson_result get() noexcept { + if (error()) { return error(); } + if constexpr (std::is_same_v) { + return first; + } + return first.get(); + } + template + simdjson_inline error_code get(T& out) noexcept { + if (error()) { return error(); } + if constexpr (std::is_same_v) { + out = first; + } else { + SIMDJSON_TRY( first.get(out) ); + } + return SUCCESS; + } +#endif // SIMDJSON_SUPPORTS_DESERIALIZATION }; } // namespace simdjson @@ -82929,12 +88995,17 @@ inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result #include +#if SIMDJSON_STATIC_REFLECTION +#include +// #include // for std::define_static_string - header not available yet +#endif namespace simdjson { template @@ -82981,6 +89052,22 @@ error_code tag_invoke(deserialize_tag, auto &val, T &out) noexcept { return SUCCESS; } +////////////////////////////// +// String deserialization +////////////////////////////// + +// just a character! +error_code tag_invoke(deserialize_tag, auto &val, char &out) noexcept { + std::string_view x; + SIMDJSON_TRY(val.get_string().get(x)); + if(x.size() != 1) { + return INCORRECT_TYPE; + } + out = x[0]; + return SUCCESS; +} + +// any string-like type (can be constructed from std::string_view) template requires(!require_custom_serialization) error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept(std::is_nothrow_constructible_v) { @@ -83002,15 +89089,20 @@ template requires(!require_custom_serialization) error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept(false) { using value_type = typename std::remove_cvref_t::value_type; - static_assert( + /*static_assert( deserializable, - "The specified type inside the container must itself be deserializable"); + "The specified type inside the container must itself be deserializable");*/ static_assert( std::is_default_constructible_v, "The specified type inside the container must default constructible."); ppc64::ondemand::array arr; - SIMDJSON_TRY(val.get_array().get(arr)); + if constexpr (std::is_same_v, ppc64::ondemand::array>) { + arr = val; + } else { + SIMDJSON_TRY(val.get_array().get(arr)); + } + for (auto v : arr) { if constexpr (concepts::returns_reference) { if (auto const err = v.get().get(concepts::emplace_one(out)); @@ -83067,7 +89159,45 @@ error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept(false) { return SUCCESS; } +template +error_code tag_invoke(deserialize_tag, ppc64::ondemand::object &obj, T &out) noexcept { + using value_type = typename std::remove_cvref_t::mapped_type; + out.clear(); + for (auto field : obj) { + std::string_view key; + SIMDJSON_TRY(field.unescaped_key().get(key)); + + ppc64::ondemand::value value_obj; + SIMDJSON_TRY(field.value().get(value_obj)); + + value_type this_value; + SIMDJSON_TRY(value_obj.get(this_value)); + out.emplace(typename T::key_type(key), std::move(this_value)); + } + return SUCCESS; +} + +template +error_code tag_invoke(deserialize_tag, ppc64::ondemand::value &val, T &out) noexcept { + ppc64::ondemand::object obj; + SIMDJSON_TRY(val.get_object().get(obj)); + return simdjson::deserialize(obj, out); +} + +template +error_code tag_invoke(deserialize_tag, ppc64::ondemand::document &doc, T &out) noexcept { + ppc64::ondemand::object obj; + SIMDJSON_TRY(doc.get_object().get(obj)); + return simdjson::deserialize(obj, out); +} + +template +error_code tag_invoke(deserialize_tag, ppc64::ondemand::document_reference &doc, T &out) noexcept { + ppc64::ondemand::object obj; + SIMDJSON_TRY(doc.get_object().get(obj)); + return simdjson::deserialize(obj, out); +} /** @@ -83129,6 +89259,199 @@ error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept(nothrow_deser return SUCCESS; } + +#if SIMDJSON_STATIC_REFLECTION + + +template +constexpr bool user_defined_type = (std::is_class_v +&& !std::is_same_v && !std::is_same_v && !concepts::optional_type && +!concepts::appendable_containers && !require_custom_serialization); + + +// workaround from +// https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2024/p2996r10.html#back-and-forth +// for missing expansion statements +namespace __impl { + template + struct replicator_type { + template + constexpr void operator>>(F body) const { + (body.template operator()(), ...); + } + }; + + template + replicator_type replicator = {}; +} + +template +consteval auto expand(R range) { + std::vector args; + for (auto r : range) { + args.push_back(reflect_value(r)); + } + return substitute(^^__impl::replicator, args); +} +// end of workaround + +template + requires(user_defined_type && std::is_class_v) +error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept { + ppc64::ondemand::object obj; + if constexpr (std::is_same_v, ppc64::ondemand::object>) { + obj = val; + } else { + SIMDJSON_TRY(val.get_object().get(obj)); + } + error_code e = simdjson::SUCCESS; + + [:expand(std::meta::nonstatic_data_members_of(^^T, std::meta::access_context::unchecked())):] >> [&]() { + if constexpr (!std::meta::is_const(mem) && std::meta::is_public(mem)) { + constexpr std::string_view key = std::define_static_string(std::meta::identifier_of(mem)); + static_assert( + deserializable, + "The specified type inside the class must itself be deserializable"); + // as long we are succesful or the field is not found, we continue + if(e == simdjson::SUCCESS || e == simdjson::NO_SUCH_FIELD) { + obj[key].get(out.[:mem:]); + } + } + }; + return e; +} +template + requires(user_defined_type>) +error_code tag_invoke(deserialize_tag, simdjson_value &val, std::unique_ptr &out) noexcept { + if (!out) { + out = std::make_unique(); + if (!out) { + return MEMALLOC; + } + } + if (auto err = val.get(*out)) { + out.reset(); + return err; + } + return SUCCESS; +} + +template + requires(user_defined_type>) +error_code tag_invoke(deserialize_tag, simdjson_value &val, std::shared_ptr &out) noexcept { + if (!out) { + out = std::make_shared(); + if (!out) { + return MEMALLOC; + } + } + if (auto err = val.get(*out)) { + out.reset(); + return err; + } + return SUCCESS; +} + +#endif // SIMDJSON_STATIC_REFLECTION + +//////////////////////////////////////// +// Unique pointers +//////////////////////////////////////// +error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) noexcept { + if (!out) { + out = std::make_unique(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_bool().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) noexcept { + if (!out) { + out = std::make_unique(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_int64().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) noexcept { + if (!out) { + out = std::make_unique(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_uint64().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) noexcept { + if (!out) { + out = std::make_unique(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_double().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) noexcept { + if (!out) { + out = std::make_unique(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_string().get(*out)); + return SUCCESS; +} + + +//////////////////////////////////////// +// Shared pointers +//////////////////////////////////////// +error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) noexcept { + if (!out) { + out = std::make_shared(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_bool().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) noexcept { + if (!out) { + out = std::make_shared(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_int64().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) noexcept { + if (!out) { + out = std::make_shared(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_uint64().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) noexcept { + if (!out) { + out = std::make_shared(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_double().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) noexcept { + if (!out) { + out = std::make_shared(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_string().get(*out)); + return SUCCESS; +} + + } // namespace simdjson #endif // SIMDJSON_ONDEMAND_DESERIALIZE_H @@ -86616,6 +92939,7 @@ simdjson_inline simdjson_result simdjson_result::simdjson_resul #endif // SIMDJSON_GENERIC_ONDEMAND_VALUE_ITERATOR_INL_H /* end file simdjson/generic/ondemand/value_iterator-inl.h for ppc64 */ +// JSON builder, ideally they should not be part of the ondemand directory +// but it is convenient for now to have them here. +/* including simdjson/generic/ondemand/json_string_builder.h for ppc64: #include "simdjson/generic/ondemand/json_string_builder.h" */ +/* begin file simdjson/generic/ondemand/json_string_builder.h for ppc64 */ +/** + * This file is part of the builder API. It is temporarily in the ondemand directory + * but we will move it to a builder directory later. + */ +#ifndef SIMDJSON_GENERIC_STRING_BUILDER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { +namespace ppc64 { +namespace builder { + +/** + * A builder for JSON strings representing documents. This is a low-level + * builder that is not meant to be used directly by end-users. Though it + * supports atomic types (Booleans, strings), it does not support composed + * types (arrays and objects). + * + * Ultimately, this class should support kernel-specific optimizations. E.g., + * it may make use of SIMD instructions to escape strings faster. + */ +class string_builder { +public: + simdjson_inline string_builder(size_t initial_capacity = 1024); + + /** + * Append number (includes Booleans). Booleans are mapped to the strings + * false and true. Numbers are converted to strings abiding by the JSON standard. + * Floating-point numbers are converted to the shortest string that 'correctly' + * represents the number. + */ + template::value>::type> + simdjson_inline void append(number_type v) noexcept; + + /** + * Append character c. + */ + simdjson_inline void append(char c) noexcept; + + /** + * Append the string 'null'. + */ + simdjson_inline void append_null() noexcept; + + /** + * Clear the content. + */ + simdjson_inline void clear() noexcept; + + /** + * Append the std::string_view, after escaping it. + * There is no UTF-8 validation. + */ + simdjson_inline void escape_and_append(std::string_view input) noexcept; + + /** + * Append the std::string_view surrounded by double quotes, after escaping it. + * There is no UTF-8 validation. + */ + simdjson_inline void escape_and_append_with_quotes(std::string_view input) noexcept; + + /** + * Append the character surrounded by double quotes, after escaping it. + * There is no UTF-8 validation. + */ + simdjson_inline void escape_and_append_with_quotes(char input) noexcept; + + /** + * Append the character surrounded by double quotes, after escaping it. + * There is no UTF-8 validation. + */ + simdjson_inline void escape_and_append_with_quotes(const char* input) noexcept; + + /** + * Append the C string directly, without escaping. + * There is no UTF-8 validation. + */ + simdjson_inline void append_raw(const char *c) noexcept; + + /** + * Append "{" to the buffer. + */ + simdjson_inline void start_object() noexcept; + + /** + * Append "}" to the buffer. + */ + simdjson_inline void end_object() noexcept; + + /** + * Append "[" to the buffer. + */ + simdjson_inline void start_array() noexcept; + + /** + * Append "]" to the buffer. + */ + simdjson_inline void end_array() noexcept; + + /** + * Append "," to the buffer. + */ + simdjson_inline void append_comma() noexcept; + + /** + * Append ":" to the buffer. + */ + simdjson_inline void append_colon() noexcept; + + /** + * Append a key-value pair to the buffer. + * The key is escaped and surrounded by double quotes. + * The value is escaped if it is a string. + */ + template + simdjson_inline void append_key_value(key_type key, value_type value) noexcept; + /** + * Append the std::string_view directly, without escaping. + * There is no UTF-8 validation. + */ + simdjson_inline void append_raw(std::string_view input) noexcept; + + /** + * Append len characters from str. + * There is no UTF-8 validation. + */ + simdjson_inline void append_raw(const char *str, size_t len) noexcept; +#if SIMDJSON_EXCEPTIONS + /** + * Creates an std::string from the written JSON buffer. + * Throws if memory allocation failed + * + * The result may not be valid UTF-8 if some of your content was not valid UTF-8. + * Use validate_unicode() to check the content if needed. + */ + simdjson_inline operator std::string() const noexcept(false); + + /** + * Creates an std::string_view from the written JSON buffer. + * Throws if memory allocation failed. + * + * The result may not be valid UTF-8 if some of your content was not valid UTF-8. + * Use validate_unicode() to check the content if needed. + */ + simdjson_inline operator std::string_view() const noexcept(false); +#endif + + /** + * Returns a view on the written JSON buffer. Returns an error + * if memory allocation failed. + * + * The result may not be valid UTF-8 if some of your content was not valid UTF-8. + * Use validate_unicode() to check the content. + */ + simdjson_inline simdjson_result view() const noexcept; + + /** + * Appends the null character to the buffer and returns + * a pointer to the beginning of the written JSON buffer. + * Returns an error if memory allocation failed. + * The result is null-terminated. + * + * The result may not be valid UTF-8 if some of your content was not valid UTF-8. + * Use validate_unicode() to check the content. + */ + simdjson_inline simdjson_result c_str() noexcept; + + /** + * Return true if the content is valid UTF-8. + */ + simdjson_inline bool validate_unicode() const noexcept; + + /** + * Returns the current size of the written JSON buffer. + * If an error occurred, returns 0. + */ + simdjson_inline size_t size() const noexcept; + +private: + /** + * Returns true if we can write at least upcoming_bytes bytes. + * The underlying buffer is reallocated if needed. It is designed + * to be called before writing to the buffer. It should be fast. + */ + simdjson_inline bool capacity_check(size_t upcoming_bytes); + + /** + * Grow the buffer to at least desired_capacity bytes. + * If the allocation fails, is_valid is set to false. We expect + * that this function would not be repeatedly called. + */ + simdjson_inline void grow_buffer(size_t desired_capacity); + + /** + * We use this helper function to make sure that is_valid is kept consistent. + */ + simdjson_inline void set_valid(bool valid) noexcept; + + std::unique_ptr buffer{}; + size_t position{0}; + size_t capacity{0}; + bool is_valid{true}; +}; + + + +} +} +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_STRING_BUILDER_H +/* end file simdjson/generic/ondemand/json_string_builder.h for ppc64 */ +/* including simdjson/generic/ondemand/json_string_builder-inl.h for ppc64: #include "simdjson/generic/ondemand/json_string_builder-inl.h" */ +/* begin file simdjson/generic/ondemand/json_string_builder-inl.h for ppc64 */ +/** + * This file is part of the builder API. It is temporarily in the ondemand + * directory but we will move it to a builder directory later. + */ +#include +#include +#include +#ifndef SIMDJSON_GENERIC_STRING_BUILDER_INL_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_INL_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +/* + * Empirically, we have found that an inlined optimization is important for + * performance. The following macros are not ideal. We should find a better + * way to inline the code. + */ + +#if defined(__SSE2__) || defined(__x86_64__) || defined(__x86_64) || \ + (defined(_M_AMD64) || defined(_M_X64) || \ + (defined(_M_IX86_FP) && _M_IX86_FP == 2)) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_SSE2 +#define SIMDJSON_EXPERIMENTAL_HAS_SSE2 1 +#endif +#endif + +#if defined(__aarch64__) || defined(_M_ARM64) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_NEON +#define SIMDJSON_EXPERIMENTAL_HAS_NEON 1 +#endif +#endif +#if SIMDJSON_EXPERIMENTAL_HAS_NEON +#include +#endif +#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 +#include +#endif + +namespace simdjson { +namespace ppc64 { +namespace builder { + +static SIMDJSON_CONSTEXPR_LAMBDA std::array json_quotable_character = { + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + +/** + +A possible SWAR implementation of has_json_escapable_byte. It is not used because +it is slower than the current implementation. It is kept here for reference (to show +that we tried it). + +inline bool has_json_escapable_byte(uint64_t x) { + uint64_t is_ascii = 0x8080808080808080ULL & ~x; + uint64_t xor2 = x ^ 0x0202020202020202ULL; + uint64_t lt32_or_eq34 = xor2 - 0x2121212121212121ULL; + uint64_t sub92 = x ^ 0x5C5C5C5C5C5C5C5CULL; + uint64_t eq92 = (sub92 - 0x0101010101010101ULL); + return ((lt32_or_eq34 | eq92) & is_ascii) != 0; +} + +**/ + +SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline bool +simple_needs_escaping(std::string_view v) { + for (char c : v) { + // a table lookup is faster than a series of comparisons + if(json_quotable_character[static_cast(c)]) { + return true; + } + } + return false; +} + +#if SIMDJSON_EXPERIMENTAL_HAS_NEON +simdjson_inline bool fast_needs_escaping(std::string_view view) { + if (view.size() < 16) { + return simple_needs_escaping(view); + } + size_t i = 0; + uint8x16_t running = vdupq_n_u8(0); + uint8x16_t v34 = vdupq_n_u8(34); + uint8x16_t v92 = vdupq_n_u8(92); + + for (; i + 15 < view.size(); i += 16) { + uint8x16_t word = vld1q_u8((const uint8_t *)view.data() + i); + running = vorrq_u8(running, vceqq_u8(word, v34)); + running = vorrq_u8(running, vceqq_u8(word, v92)); + running = vorrq_u8(running, vcltq_u8(word, vdupq_n_u8(32))); + } + if (i < view.size()) { + uint8x16_t word = + vld1q_u8((const uint8_t *)view.data() + view.length() - 16); + running = vorrq_u8(running, vceqq_u8(word, v34)); + running = vorrq_u8(running, vceqq_u8(word, v92)); + running = vorrq_u8(running, vcltq_u8(word, vdupq_n_u8(32))); + } + return vmaxvq_u32(vreinterpretq_u32_u8(running)) != 0; +} +#elif SIMDJSON_EXPERIMENTAL_HAS_SSE2 +simdjson_inline bool fast_needs_escaping(std::string_view view) { + if (view.size() < 16) { + return simple_needs_escaping(view); + } + size_t i = 0; + __m128i running = _mm_setzero_si128(); + for (; i + 15 < view.size(); i += 16) { + + __m128i word = _mm_loadu_si128(reinterpret_cast(view.data() + i)); + running = _mm_or_si128(running, _mm_cmpeq_epi8(word, _mm_set1_epi8(34))); + running = _mm_or_si128(running, _mm_cmpeq_epi8(word, _mm_set1_epi8(92))); + running = _mm_or_si128( + running, _mm_cmpeq_epi8(_mm_subs_epu8(word, _mm_set1_epi8(31)), + _mm_setzero_si128())); + } + if (i < view.size()) { + __m128i word = + _mm_loadu_si128(reinterpret_cast(view.data() + view.length() - 16)); + running = _mm_or_si128(running, _mm_cmpeq_epi8(word, _mm_set1_epi8(34))); + running = _mm_or_si128(running, _mm_cmpeq_epi8(word, _mm_set1_epi8(92))); + running = _mm_or_si128( + running, _mm_cmpeq_epi8(_mm_subs_epu8(word, _mm_set1_epi8(31)), + _mm_setzero_si128())); + } + return _mm_movemask_epi8(running) != 0; +} +#else +simdjson_inline bool fast_needs_escaping(std::string_view view) { + return simple_needs_escaping(view); +} +#endif + + +SIMDJSON_CONSTEXPR_LAMBDA inline size_t +find_next_json_quotable_character(const std::string_view view, + size_t location) noexcept { + + for (auto pos = view.begin() + location; pos != view.end(); ++pos) { + if (json_quotable_character[static_cast(*pos)]) { + return pos - view.begin(); + } + } + return size_t(view.size()); +} + +SIMDJSON_CONSTEXPR_LAMBDA static std::string_view control_chars[] = { + "\\x0000", "\\x0001", "\\x0002", "\\x0003", "\\x0004", "\\x0005", "\\x0006", + "\\x0007", "\\x0008", "\\t", "\\n", "\\x000b", "\\f", "\\r", + "\\x000e", "\\x000f", "\\x0010", "\\x0011", "\\x0012", "\\x0013", "\\x0014", + "\\x0015", "\\x0016", "\\x0017", "\\x0018", "\\x0019", "\\x001a", "\\x001b", + "\\x001c", "\\x001d", "\\x001e", "\\x001f"}; + +SIMDJSON_CONSTEXPR_LAMBDA void escape_json_char(char c, char *&out) { + if (c == '"') { + memcpy(out, "\\\"", 2); + out += 2; + } else if (c == '\\') { + memcpy(out, "\\\\", 2); + out += 2; + } else { + std::string_view v = control_chars[uint8_t(c)]; + memcpy(out, v.data(), v.size()); + out += v.size(); + } +} + +inline size_t write_string_escaped(const std::string_view input, char *out) { + size_t mysize = input.size(); + if (!fast_needs_escaping(input)) { // fast path! + memcpy(out, input.data(), input.size()); + return input.size(); + } + const char *const initout = out; + size_t location = find_next_json_quotable_character(input, 0); + memcpy(out, input.data(), location); + out += location; + escape_json_char(input[location], out); + location += 1; + while (location < mysize) { + size_t newlocation = find_next_json_quotable_character(input, location); + memcpy(out, input.data() + location, newlocation - location); + out += newlocation - location; + location = newlocation; + if (location == mysize) { + break; + } + escape_json_char(input[location], out); + location += 1; + } + return out - initout; +} + +#if SIMDJSON_CONSTEVAL +// unoptimized, meant for compile-time execution +consteval std::string consteval_to_quoted_escaped(std::string_view input) { + std::string out = "\""; + for (char c : input) { + if (json_quotable_character[uint8_t(c)]) { + if (c == '"') { + out.append("\\\""); + } else if (c == '\\') { + out.append("\\\\"); + } else { + std::string_view v = control_chars[uint8_t(c)]; + out.append(v); + } + } else { + out.push_back(c); + } + } + out.push_back('"'); + return out; +} +#endif // SIMDJSON_CONSTEVAL + +simdjson_inline string_builder::string_builder(size_t initial_capacity) + : buffer(new(std::nothrow) char[initial_capacity]), position(0), + capacity(buffer.get() != nullptr ? initial_capacity : 0), + is_valid(buffer.get() != nullptr) {} + +simdjson_inline bool string_builder::capacity_check(size_t upcoming_bytes) { + // We use the convention that when is_valid is false, then the capacity and + // the position are 0. + // Most of the time, this function will return true. + if (simdjson_likely(upcoming_bytes <= capacity - position)) { + return true; + } + // check for overflow, most of the time there is no overflow + if (simdjson_likely(position + upcoming_bytes < position)) { + return false; + } + // We will rarely get here. + grow_buffer((std::max)(capacity * 2, position + upcoming_bytes)); + // If the buffer allocation failed, we set is_valid to false. + return is_valid; +} + +simdjson_inline void string_builder::grow_buffer(size_t desired_capacity) { + if (!is_valid) { + return; + } + std::unique_ptr new_buffer(new (std::nothrow) char[desired_capacity]); + if (new_buffer.get() == nullptr) { + set_valid(false); + return; + } + std::memcpy(new_buffer.get(), buffer.get(), position); + buffer.swap(new_buffer); + capacity = desired_capacity; +} + +simdjson_inline void string_builder::set_valid(bool valid) noexcept { + if (!valid) { + is_valid = false; + capacity = 0; + position = 0; + buffer.reset(); + } else { + is_valid = true; + } +} + +simdjson_inline size_t string_builder::size() const noexcept { + return position; +} + +simdjson_inline void string_builder::append(char c) noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = c; + } +} + +simdjson_inline void string_builder::append_null() noexcept { + constexpr char null_literal[] = "null"; + constexpr size_t null_len = sizeof(null_literal) - 1; + if (capacity_check(null_len)) { + std::memcpy(buffer.get() + position, null_literal, null_len); + position += null_len; + } +} + +simdjson_inline void string_builder::clear() noexcept { + position = 0; + // if it was invalid, we should try to repair it + if (!is_valid) { + capacity = 0; + buffer.reset(); + is_valid = true; + } +} + +namespace internal { + +// We could specialize further for 32-bit integers. +int int_log2(uint32_t x) { return (63 - leading_zeroes(x | 1)); } + +int fast_digit_count(uint32_t x) { + static uint64_t table[] = { + 4294967296, 8589934582, 8589934582, 8589934582, 12884901788, + 12884901788, 12884901788, 17179868184, 17179868184, 17179868184, + 21474826480, 21474826480, 21474826480, 21474826480, 25769703776, + 25769703776, 25769703776, 30063771072, 30063771072, 30063771072, + 34349738368, 34349738368, 34349738368, 34349738368, 38554705664, + 38554705664, 38554705664, 41949672960, 41949672960, 41949672960, + 42949672960, 42949672960}; + return uint32_t((x + table[int_log2(x)]) >> 32); +} + +int int_log2(uint64_t x) { return 63 - leading_zeroes(x | 1); } + +int fast_digit_count(uint64_t x) { + static uint64_t table[] = {9, + 99, + 999, + 9999, + 99999, + 999999, + 9999999, + 99999999, + 999999999, + 9999999999, + 99999999999, + 999999999999, + 9999999999999, + 99999999999999, + 999999999999999ULL, + 9999999999999999ULL, + 99999999999999999ULL, + 999999999999999999ULL, + 9999999999999999999ULL}; + int y = (19 * int_log2(x) >> 6); + y += x > table[y]; + return y + 1; +} + +template ::value>::type> +simdjson_inline size_t digit_count(number_type v) noexcept { + static_assert(sizeof(number_type) == 8 || sizeof(number_type) == 4 || + sizeof(number_type) == 2 || sizeof(number_type) == 1, + "We only support 8-bit, 16-bit, 32-bit and 64-bit numbers"); + return fast_digit_count(v); +} +static const char decimal_table[200] = { + 0x30, 0x30, 0x30, 0x31, 0x30, 0x32, 0x30, 0x33, 0x30, 0x34, 0x30, 0x35, + 0x30, 0x36, 0x30, 0x37, 0x30, 0x38, 0x30, 0x39, 0x31, 0x30, 0x31, 0x31, + 0x31, 0x32, 0x31, 0x33, 0x31, 0x34, 0x31, 0x35, 0x31, 0x36, 0x31, 0x37, + 0x31, 0x38, 0x31, 0x39, 0x32, 0x30, 0x32, 0x31, 0x32, 0x32, 0x32, 0x33, + 0x32, 0x34, 0x32, 0x35, 0x32, 0x36, 0x32, 0x37, 0x32, 0x38, 0x32, 0x39, + 0x33, 0x30, 0x33, 0x31, 0x33, 0x32, 0x33, 0x33, 0x33, 0x34, 0x33, 0x35, + 0x33, 0x36, 0x33, 0x37, 0x33, 0x38, 0x33, 0x39, 0x34, 0x30, 0x34, 0x31, + 0x34, 0x32, 0x34, 0x33, 0x34, 0x34, 0x34, 0x35, 0x34, 0x36, 0x34, 0x37, + 0x34, 0x38, 0x34, 0x39, 0x35, 0x30, 0x35, 0x31, 0x35, 0x32, 0x35, 0x33, + 0x35, 0x34, 0x35, 0x35, 0x35, 0x36, 0x35, 0x37, 0x35, 0x38, 0x35, 0x39, + 0x36, 0x30, 0x36, 0x31, 0x36, 0x32, 0x36, 0x33, 0x36, 0x34, 0x36, 0x35, + 0x36, 0x36, 0x36, 0x37, 0x36, 0x38, 0x36, 0x39, 0x37, 0x30, 0x37, 0x31, + 0x37, 0x32, 0x37, 0x33, 0x37, 0x34, 0x37, 0x35, 0x37, 0x36, 0x37, 0x37, + 0x37, 0x38, 0x37, 0x39, 0x38, 0x30, 0x38, 0x31, 0x38, 0x32, 0x38, 0x33, + 0x38, 0x34, 0x38, 0x35, 0x38, 0x36, 0x38, 0x37, 0x38, 0x38, 0x38, 0x39, + 0x39, 0x30, 0x39, 0x31, 0x39, 0x32, 0x39, 0x33, 0x39, 0x34, 0x39, 0x35, + 0x39, 0x36, 0x39, 0x37, 0x39, 0x38, 0x39, 0x39, +}; +} // namespace internal + +template +simdjson_inline void string_builder::append(number_type v) noexcept { + static_assert(std::is_same::value || + std::is_integral::value || + std::is_floating_point::value, + "Unsupported number type"); + // If C++17 is available, we can 'if constexpr' here. + SIMDJSON_IF_CONSTEXPR(std::is_same::value) { + if (v) { + constexpr char true_literal[] = "true"; + constexpr size_t true_len = sizeof(true_literal) - 1; + if (capacity_check(true_len)) { + std::memcpy(buffer.get() + position, true_literal, true_len); + position += true_len; + } + } else { + constexpr char false_literal[] = "false"; + constexpr size_t false_len = sizeof(false_literal) - 1; + if (capacity_check(false_len)) { + std::memcpy(buffer.get() + position, false_literal, false_len); + position += false_len; + } + } + } + else SIMDJSON_IF_CONSTEXPR(std::is_unsigned::value) { + constexpr size_t max_number_size = 20; + if (capacity_check(max_number_size)) { + using unsigned_type = typename std::make_unsigned::type; + unsigned_type pv = static_cast(v); + size_t dc = internal::digit_count(pv); + char *write_pointer = buffer.get() + position + dc - 1; + while (pv >= 100) { + memcpy(write_pointer - 1, &internal::decimal_table[(pv % 100)*2], 2); + write_pointer -= 2; + pv /= 100; + } + if (pv >= 10) { + *write_pointer-- = char('0' + (pv % 10)); + pv /= 10; + } + *write_pointer = char('0' + pv); + position += dc; + } + } + else SIMDJSON_IF_CONSTEXPR(std::is_integral::value) { + constexpr size_t max_number_size = 20; + if (capacity_check(max_number_size)) { + using unsigned_type = typename std::make_unsigned::type; + bool negative = v < 0; + unsigned_type pv = static_cast(v); + if (negative) { + pv = 0 - pv; // the 0 is for Microsoft + } + size_t dc = internal::digit_count(pv); + if (negative) { + buffer.get()[position++] = '-'; + } + char *write_pointer = buffer.get() + position + dc - 1; + while (pv >= 100) { + memcpy(write_pointer - 1, &internal::decimal_table[(pv % 100)*2], 2); + write_pointer -= 2; + pv /= 100; + } + if (pv >= 10) { + *write_pointer-- = char('0' + (pv % 10)); + pv /= 10; + } + *write_pointer = char('0' + pv); + position += dc; + } + } + else SIMDJSON_IF_CONSTEXPR(std::is_floating_point::value) { + constexpr size_t max_number_size = 24; + if (capacity_check(max_number_size)) { + // We could specialize for float. + char *end = simdjson::internal::to_chars(buffer.get() + position, nullptr, + double(v)); + position = end - buffer.get(); + } + } +} + +simdjson_inline void +string_builder::escape_and_append(std::string_view input) noexcept { + // escaping might turn a control character into \x00xx so 6 characters. + if (capacity_check(6 * input.size())) { + position += write_string_escaped(input, buffer.get() + position); + } +} + +simdjson_inline void +string_builder::escape_and_append_with_quotes(std::string_view input) noexcept { + // escaping might turn a control character into \x00xx so 6 characters. + if (capacity_check(2 + 6 * input.size())) { + buffer.get()[position++] = '"'; + position += write_string_escaped(input, buffer.get() + position); + buffer.get()[position++] = '"'; + } +} + +simdjson_inline void +string_builder::escape_and_append_with_quotes(char input) noexcept { + // escaping might turn a control character into \x00xx so 6 characters. + if (capacity_check(2 + 6 * 1)) { + buffer.get()[position++] = '"'; + std::string_view cinput(&input, 1); + position += write_string_escaped(cinput, buffer.get() + position); + buffer.get()[position++] = '"'; + } +} + +simdjson_inline void string_builder::escape_and_append_with_quotes(const char* input) noexcept { + std::string_view cinput(input); + escape_and_append_with_quotes(cinput); +} + +simdjson_inline void string_builder::append_raw(const char *c) noexcept { + size_t len = std::strlen(c); + append_raw(c, len); +} + +simdjson_inline void +string_builder::append_raw(std::string_view input) noexcept { + if (capacity_check(input.size())) { + std::memcpy(buffer.get() + position, input.data(), input.size()); + position += input.size(); + } +} + +simdjson_inline void string_builder::append_raw(const char *str, + size_t len) noexcept { + if (capacity_check(len)) { + std::memcpy(buffer.get() + position, str, len); + position += len; + } +} + +#if SIMDJSON_EXCEPTIONS +simdjson_inline string_builder::operator std::string() const noexcept(false) { + return std::string(std::string_view()); +} + +simdjson_inline string_builder::operator std::string_view() const + noexcept(false) { + return view(); +} +#endif + +simdjson_inline simdjson_result +string_builder::view() const noexcept { + if (!is_valid) { + return simdjson::OUT_OF_CAPACITY; + } + return std::string_view(buffer.get(), position); +} + +simdjson_inline simdjson_result string_builder::c_str() noexcept { + if (capacity_check(1)) { + buffer.get()[position] = '\0'; + return buffer.get(); + } + return simdjson::OUT_OF_CAPACITY; +} + +simdjson_inline bool string_builder::validate_unicode() const noexcept { + return simdjson::validate_utf8(buffer.get(), position); +} + +simdjson_inline void string_builder::start_object() noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = '{'; + } +} + + +simdjson_inline void string_builder::end_object() noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = '}'; + } +} + + +simdjson_inline void string_builder::start_array() noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = '['; + } +} + + +simdjson_inline void string_builder::end_array() noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = ']'; + } +} + + +simdjson_inline void string_builder::append_comma() noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = ','; + } +} + + +simdjson_inline void string_builder::append_colon() noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = ':'; + } +} + +template +simdjson_inline void string_builder::append_key_value(key_type key, value_type value) noexcept { + static_assert( + std::is_arithmetic::value || + std::is_same::value || + std::is_same::value || + std::is_convertible::value || + std::is_same::value, + "Unsupported value type"); + static_assert( + std::is_same::value || + std::is_convertible::value, + "Unsupported key type"); + escape_and_append_with_quotes(key); + append_colon(); + SIMDJSON_IF_CONSTEXPR(std::is_same::value) { + append_null(); + } else SIMDJSON_IF_CONSTEXPR(std::is_same::value) { + escape_and_append_with_quotes(value); + } else SIMDJSON_IF_CONSTEXPR(std::is_convertible::value) { + escape_and_append_with_quotes(value); + } else SIMDJSON_IF_CONSTEXPR(std::is_same::value) { + escape_and_append_with_quotes(value); + } else { + append(value); + } +} + +} // namespace builder +} // namespace ppc64 +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_STRING_BUILDER_INL_H +/* end file simdjson/generic/ondemand/json_string_builder-inl.h for ppc64 */ +/* including simdjson/generic/ondemand/json_builder.h for ppc64: #include "simdjson/generic/ondemand/json_builder.h" */ +/* begin file simdjson/generic/ondemand/json_builder.h for ppc64 */ +/** + * This file is part of the builder API. It is temporarily in the ondemand directory + * but we will move it to a builder directory later. + */ +#ifndef SIMDJSON_GENERIC_BUILDER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/concepts.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ +#if SIMDJSON_STATIC_REFLECTION + +#include +#include +#include +#include +#include +#include +// #include // for std::define_static_string - header not available yet + +namespace simdjson { +namespace ppc64 { +namespace builder { + +// Concept that checks if a type is a container but not a string (because +// strings handling must be handled differently) +template +concept container_but_not_string = + requires(T a) { + { a.size() } -> std::convertible_to; + { + a[std::declval()] + }; // check if elements are accessible for the subscript operator + } && !std::is_same_v && + !std::is_same_v && !std::is_same_v; + +template + requires(container_but_not_string) +constexpr void atom(string_builder &b, const T &t) { + if (t.size() == 0) { + b.append_raw("[]"); + return; + } + b.append('['); + atom(b, t[0]); + for (size_t i = 1; i < t.size(); ++i) { + b.append(','); + atom(b, t[i]); + } + b.append(']'); +} + +template + requires(std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v) +constexpr void atom(string_builder &b, const T &t) { + b.escape_and_append_with_quotes(t); +} + +template +constexpr void atom(string_builder &b, const T &m) { + if (m.empty()) { + b.append_raw("{}"); + return; + } + b.append('{'); + bool first = true; + for (const auto& [key, value] : m) { + if (!first) { + b.append(','); + } + first = false; + // Keys must be convertible to string_view per the concept + b.escape_and_append_with_quotes(key); + b.append(':'); + atom(b, value); + } + b.append('}'); +} + + +template::value && !std::is_same_v>::type> +constexpr void atom(string_builder &b, const number_type t) { + b.append(t); +} +#if SIMDJSON_CONSTEVAL +consteval std::string consteval_to_quoted_escaped(std::string_view input); +#endif + +template + requires(std::is_class_v && !container_but_not_string && + !concepts::string_view_keyed_map && + !std::is_same_v && + !std::is_same_v) +constexpr void atom(string_builder &b, const T &t) { + int i = 0; + b.append('{'); + [:expand(std::meta::nonstatic_data_members_of(^^T, std::meta::access_context::unchecked())):] >> [&]() { + if (i != 0) + b.append(','); + constexpr auto key = std::define_static_string(consteval_to_quoted_escaped(std::meta::identifier_of(dm))); + b.append_raw(key); + b.append(':'); + atom(b, t.[:dm:]); + i++; + }; + b.append('}'); +} + +// works for struct +template void append(string_builder &b, const Z &z) { + int i = 0; + b.append('{'); + [:expand(std::meta::nonstatic_data_members_of(^^Z, std::meta::access_context::unchecked())):] >> [&]() { + if (i != 0) + b.append(','); + constexpr auto key = std::define_static_string(consteval_to_quoted_escaped(std::meta::identifier_of(dm))); + b.append_raw(key); + b.append(':'); + atom(b, z.[:dm:]); + i++; + }; + b.append('}'); +} + +// works for container +template + requires(container_but_not_string) +void append(string_builder &b, const Z &z) { + if (z.size() == 0) { + b.append_raw("[]"); + return; + } + b.append('['); + atom(b, z[0]); + for (size_t i = 1; i < z.size(); ++i) { + b.append(','); + atom(b, z[i]); + } + b.append(']'); +} + +template +simdjson_result to_json_string(const Z &z) { + string_builder b; + append(b, z); + std::string_view s; + if(auto e = b.view().get(s); e) { return e; } + return std::string(s); +} + +template +simdjson_error to_json(const Z &z, std::string &s) { + string_builder b; + append(b, z); + std::string_view view; + if(auto e = b.view().get(view); e) { return e; } + s.assign(view); + return SUCCESS; +} +} // namespace json_builder +} // namespace ppc64 +} // namespace simdjson +#endif // SIMDJSON_STATIC_REFLECTION + +#endif +/* end file simdjson/generic/ondemand/json_builder.h for ppc64 */ /* end file simdjson/generic/ondemand/amalgamated.h for ppc64 */ /* including simdjson/ppc64/end.h: #include "simdjson/ppc64/end.h" */ @@ -89755,6 +97092,31 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin }; } + +struct escaping { + static constexpr uint32_t BYTES_PROCESSED = 16; + simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst); + + simdjson_inline bool has_escape() { return escape_bits != 0; } + simdjson_inline int escape_index() { return trailing_zeroes(escape_bits); } + + uint64_t escape_bits; +}; // struct escaping + + + +simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) { + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v(src); + v.store(dst); + simd8 is_quote = (v == '"'); + simd8 is_backslash = (v == '\\'); + simd8 is_control = (v < 32); + return { + uint64_t((is_backslash | is_quote | is_control).to_bitmask()) + }; +} + } // unnamed namespace } // namespace westmere } // namespace simdjson @@ -89911,10 +97273,26 @@ concept nothrow_deserializable = nothrow_custom_deserializable || is_bu /// Deserialize Tag inline constexpr struct deserialize_tag { + using array_type = westmere::ondemand::array; + using object_type = westmere::ondemand::object; using value_type = westmere::ondemand::value; using document_type = westmere::ondemand::document; using document_reference_type = westmere::ondemand::document_reference; + // Customization Point for array + template + requires custom_deserializable + [[nodiscard]] constexpr /* error_code */ auto operator()(array_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + return tag_invoke(*this, object, output); + } + + // Customization Point for object + template + requires custom_deserializable + [[nodiscard]] constexpr /* error_code */ auto operator()(object_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + return tag_invoke(*this, object, output); + } + // Customization Point for value template requires custom_deserializable @@ -90482,6 +97860,7 @@ public: * * You may use get_double(), get_bool(), get_uint64(), get_int64(), * get_object(), get_array(), get_raw_json_string(), or get_string() instead. + * When SIMDJSON_SUPPORTS_DESERIALIZATION is set, custom types are also supported. * * @returns A value of the given type, parsed from the JSON. * @returns INCORRECT_TYPE If the JSON value is not the given type. @@ -90505,6 +97884,7 @@ public: * Get this value as the given type. * * Supported types: object, array, raw_json_string, string_view, uint64_t, int64_t, double, bool + * If the macro SIMDJSON_SUPPORTS_DESERIALIZATION is set, then custom types are also supported. * * @param out This is set to a value of the given type, parsed from the JSON. If there is an error, this may not be initialized. * @returns INCORRECT_TYPE If the JSON value is not an object. @@ -92747,6 +100127,37 @@ public: * - INDEX_OUT_OF_BOUNDS if the array index is larger than an array length */ simdjson_inline simdjson_result at(size_t index) noexcept; + +#if SIMDJSON_SUPPORTS_DESERIALIZATION + /** + * Get this array as the given type. + * + * @param out This is set to a value of the given type, parsed from the JSON. If there is an error, this may not be initialized. + * @returns INCORRECT_TYPE If the JSON array is not of the given type. + * @returns SUCCESS If the parse succeeded and the out parameter was set to the value. + */ + template + simdjson_inline error_code get(T &out) + noexcept(custom_deserializable ? nothrow_custom_deserializable : true) { + static_assert(custom_deserializable); + return deserialize(*this, out); + } + /** + * Get this array as the given type. + * + * @returns A value of the given type, parsed from the JSON. + * @returns INCORRECT_TYPE If the JSON value is not the given type. + */ + template + simdjson_inline simdjson_result get() + noexcept(custom_deserializable ? nothrow_custom_deserializable : true) + { + static_assert(std::is_default_constructible::value, "The specified type is not default constructible."); + T out{}; + SIMDJSON_TRY(get(out)); + return out; + } +#endif // SIMDJSON_SUPPORTS_DESERIALIZATION protected: /** * Go to the end of the array, no matter where you are right now. @@ -92825,7 +100236,28 @@ public: simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; simdjson_inline simdjson_result raw_json() noexcept; +#if SIMDJSON_SUPPORTS_DESERIALIZATION + // TODO: move this code into object-inl.h + template + simdjson_inline simdjson_result get() noexcept { + if (error()) { return error(); } + if constexpr (std::is_same_v) { + return first; + } + return first.get(); + } + template + simdjson_inline error_code get(T& out) noexcept { + if (error()) { return error(); } + if constexpr (std::is_same_v) { + out = first; + } else { + SIMDJSON_TRY( first.get(out) ); + } + return SUCCESS; + } +#endif // SIMDJSON_SUPPORTS_DESERIALIZATION }; } // namespace simdjson @@ -93159,7 +100591,7 @@ public: * Be mindful that the document instance must remain in scope while you are accessing object, array and value instances. * * @param out This is set to a value of the given type, parsed from the JSON. If there is an error, this may not be initialized. - * @returns INCORRECT_TYPE If the JSON value is not an object. + * @returns INCORRECT_TYPE If the JSON value is of the given type. * @returns SUCCESS If the parse succeeded and the out parameter was set to the value. */ template @@ -94646,6 +102078,36 @@ public: */ simdjson_inline simdjson_result raw_json() noexcept; +#if SIMDJSON_SUPPORTS_DESERIALIZATION + /** + * Get this object as the given type. + * + * @param out This is set to a value of the given type, parsed from the JSON. If there is an error, this may not be initialized. + * @returns INCORRECT_TYPE If the JSON object is not of the given type. + * @returns SUCCESS If the parse succeeded and the out parameter was set to the value. + */ + template + simdjson_inline error_code get(T &out) + noexcept(custom_deserializable ? nothrow_custom_deserializable : true) { + static_assert(custom_deserializable); + return deserialize(*this, out); + } + /** + * Get this array as the given type. + * + * @returns A value of the given type, parsed from the JSON. + * @returns INCORRECT_TYPE If the JSON value is not the given type. + */ + template + simdjson_inline simdjson_result get() + noexcept(custom_deserializable ? nothrow_custom_deserializable : true) + { + static_assert(std::is_default_constructible::value, "The specified type is not default constructible."); + T out{}; + SIMDJSON_TRY(get(out)); + return out; + } +#endif // SIMDJSON_SUPPORTS_DESERIALIZATION protected: /** * Go to the end of the object, no matter where you are right now. @@ -94689,12 +102151,32 @@ public: simdjson_inline simdjson_result operator[](std::string_view key) && noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - inline simdjson_result reset() noexcept; inline simdjson_result is_empty() noexcept; inline simdjson_result count_fields() & noexcept; inline simdjson_result raw_json() noexcept; + #if SIMDJSON_SUPPORTS_DESERIALIZATION + // TODO: move this code into object-inl.h + template + simdjson_inline simdjson_result get() noexcept { + if (error()) { return error(); } + if constexpr (std::is_same_v) { + return first; + } + return first.get(); + } + template + simdjson_inline error_code get(T& out) noexcept { + if (error()) { return error(); } + if constexpr (std::is_same_v) { + out = first; + } else { + SIMDJSON_TRY( first.get(out) ); + } + return SUCCESS; + } +#endif // SIMDJSON_SUPPORTS_DESERIALIZATION }; } // namespace simdjson @@ -94899,12 +102381,17 @@ inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result #include +#if SIMDJSON_STATIC_REFLECTION +#include +// #include // for std::define_static_string - header not available yet +#endif namespace simdjson { template @@ -94951,6 +102438,22 @@ error_code tag_invoke(deserialize_tag, auto &val, T &out) noexcept { return SUCCESS; } +////////////////////////////// +// String deserialization +////////////////////////////// + +// just a character! +error_code tag_invoke(deserialize_tag, auto &val, char &out) noexcept { + std::string_view x; + SIMDJSON_TRY(val.get_string().get(x)); + if(x.size() != 1) { + return INCORRECT_TYPE; + } + out = x[0]; + return SUCCESS; +} + +// any string-like type (can be constructed from std::string_view) template requires(!require_custom_serialization) error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept(std::is_nothrow_constructible_v) { @@ -94972,15 +102475,20 @@ template requires(!require_custom_serialization) error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept(false) { using value_type = typename std::remove_cvref_t::value_type; - static_assert( + /*static_assert( deserializable, - "The specified type inside the container must itself be deserializable"); + "The specified type inside the container must itself be deserializable");*/ static_assert( std::is_default_constructible_v, "The specified type inside the container must default constructible."); westmere::ondemand::array arr; - SIMDJSON_TRY(val.get_array().get(arr)); + if constexpr (std::is_same_v, westmere::ondemand::array>) { + arr = val; + } else { + SIMDJSON_TRY(val.get_array().get(arr)); + } + for (auto v : arr) { if constexpr (concepts::returns_reference) { if (auto const err = v.get().get(concepts::emplace_one(out)); @@ -95037,7 +102545,45 @@ error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept(false) { return SUCCESS; } +template +error_code tag_invoke(deserialize_tag, westmere::ondemand::object &obj, T &out) noexcept { + using value_type = typename std::remove_cvref_t::mapped_type; + out.clear(); + for (auto field : obj) { + std::string_view key; + SIMDJSON_TRY(field.unescaped_key().get(key)); + + westmere::ondemand::value value_obj; + SIMDJSON_TRY(field.value().get(value_obj)); + + value_type this_value; + SIMDJSON_TRY(value_obj.get(this_value)); + out.emplace(typename T::key_type(key), std::move(this_value)); + } + return SUCCESS; +} + +template +error_code tag_invoke(deserialize_tag, westmere::ondemand::value &val, T &out) noexcept { + westmere::ondemand::object obj; + SIMDJSON_TRY(val.get_object().get(obj)); + return simdjson::deserialize(obj, out); +} + +template +error_code tag_invoke(deserialize_tag, westmere::ondemand::document &doc, T &out) noexcept { + westmere::ondemand::object obj; + SIMDJSON_TRY(doc.get_object().get(obj)); + return simdjson::deserialize(obj, out); +} + +template +error_code tag_invoke(deserialize_tag, westmere::ondemand::document_reference &doc, T &out) noexcept { + westmere::ondemand::object obj; + SIMDJSON_TRY(doc.get_object().get(obj)); + return simdjson::deserialize(obj, out); +} /** @@ -95099,6 +102645,199 @@ error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept(nothrow_deser return SUCCESS; } + +#if SIMDJSON_STATIC_REFLECTION + + +template +constexpr bool user_defined_type = (std::is_class_v +&& !std::is_same_v && !std::is_same_v && !concepts::optional_type && +!concepts::appendable_containers && !require_custom_serialization); + + +// workaround from +// https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2024/p2996r10.html#back-and-forth +// for missing expansion statements +namespace __impl { + template + struct replicator_type { + template + constexpr void operator>>(F body) const { + (body.template operator()(), ...); + } + }; + + template + replicator_type replicator = {}; +} + +template +consteval auto expand(R range) { + std::vector args; + for (auto r : range) { + args.push_back(reflect_value(r)); + } + return substitute(^^__impl::replicator, args); +} +// end of workaround + +template + requires(user_defined_type && std::is_class_v) +error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept { + westmere::ondemand::object obj; + if constexpr (std::is_same_v, westmere::ondemand::object>) { + obj = val; + } else { + SIMDJSON_TRY(val.get_object().get(obj)); + } + error_code e = simdjson::SUCCESS; + + [:expand(std::meta::nonstatic_data_members_of(^^T, std::meta::access_context::unchecked())):] >> [&]() { + if constexpr (!std::meta::is_const(mem) && std::meta::is_public(mem)) { + constexpr std::string_view key = std::define_static_string(std::meta::identifier_of(mem)); + static_assert( + deserializable, + "The specified type inside the class must itself be deserializable"); + // as long we are succesful or the field is not found, we continue + if(e == simdjson::SUCCESS || e == simdjson::NO_SUCH_FIELD) { + obj[key].get(out.[:mem:]); + } + } + }; + return e; +} +template + requires(user_defined_type>) +error_code tag_invoke(deserialize_tag, simdjson_value &val, std::unique_ptr &out) noexcept { + if (!out) { + out = std::make_unique(); + if (!out) { + return MEMALLOC; + } + } + if (auto err = val.get(*out)) { + out.reset(); + return err; + } + return SUCCESS; +} + +template + requires(user_defined_type>) +error_code tag_invoke(deserialize_tag, simdjson_value &val, std::shared_ptr &out) noexcept { + if (!out) { + out = std::make_shared(); + if (!out) { + return MEMALLOC; + } + } + if (auto err = val.get(*out)) { + out.reset(); + return err; + } + return SUCCESS; +} + +#endif // SIMDJSON_STATIC_REFLECTION + +//////////////////////////////////////// +// Unique pointers +//////////////////////////////////////// +error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) noexcept { + if (!out) { + out = std::make_unique(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_bool().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) noexcept { + if (!out) { + out = std::make_unique(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_int64().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) noexcept { + if (!out) { + out = std::make_unique(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_uint64().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) noexcept { + if (!out) { + out = std::make_unique(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_double().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) noexcept { + if (!out) { + out = std::make_unique(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_string().get(*out)); + return SUCCESS; +} + + +//////////////////////////////////////// +// Shared pointers +//////////////////////////////////////// +error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) noexcept { + if (!out) { + out = std::make_shared(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_bool().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) noexcept { + if (!out) { + out = std::make_shared(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_int64().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) noexcept { + if (!out) { + out = std::make_shared(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_uint64().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) noexcept { + if (!out) { + out = std::make_shared(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_double().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) noexcept { + if (!out) { + out = std::make_shared(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_string().get(*out)); + return SUCCESS; +} + + } // namespace simdjson #endif // SIMDJSON_ONDEMAND_DESERIALIZE_H @@ -98586,6 +106325,7 @@ simdjson_inline simdjson_result simdjson_result::simdjson_re #endif // SIMDJSON_GENERIC_ONDEMAND_VALUE_ITERATOR_INL_H /* end file simdjson/generic/ondemand/value_iterator-inl.h for westmere */ +// JSON builder, ideally they should not be part of the ondemand directory +// but it is convenient for now to have them here. +/* including simdjson/generic/ondemand/json_string_builder.h for westmere: #include "simdjson/generic/ondemand/json_string_builder.h" */ +/* begin file simdjson/generic/ondemand/json_string_builder.h for westmere */ +/** + * This file is part of the builder API. It is temporarily in the ondemand directory + * but we will move it to a builder directory later. + */ +#ifndef SIMDJSON_GENERIC_STRING_BUILDER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { +namespace westmere { +namespace builder { + +/** + * A builder for JSON strings representing documents. This is a low-level + * builder that is not meant to be used directly by end-users. Though it + * supports atomic types (Booleans, strings), it does not support composed + * types (arrays and objects). + * + * Ultimately, this class should support kernel-specific optimizations. E.g., + * it may make use of SIMD instructions to escape strings faster. + */ +class string_builder { +public: + simdjson_inline string_builder(size_t initial_capacity = 1024); + + /** + * Append number (includes Booleans). Booleans are mapped to the strings + * false and true. Numbers are converted to strings abiding by the JSON standard. + * Floating-point numbers are converted to the shortest string that 'correctly' + * represents the number. + */ + template::value>::type> + simdjson_inline void append(number_type v) noexcept; + + /** + * Append character c. + */ + simdjson_inline void append(char c) noexcept; + + /** + * Append the string 'null'. + */ + simdjson_inline void append_null() noexcept; + + /** + * Clear the content. + */ + simdjson_inline void clear() noexcept; + + /** + * Append the std::string_view, after escaping it. + * There is no UTF-8 validation. + */ + simdjson_inline void escape_and_append(std::string_view input) noexcept; + + /** + * Append the std::string_view surrounded by double quotes, after escaping it. + * There is no UTF-8 validation. + */ + simdjson_inline void escape_and_append_with_quotes(std::string_view input) noexcept; + + /** + * Append the character surrounded by double quotes, after escaping it. + * There is no UTF-8 validation. + */ + simdjson_inline void escape_and_append_with_quotes(char input) noexcept; + + /** + * Append the character surrounded by double quotes, after escaping it. + * There is no UTF-8 validation. + */ + simdjson_inline void escape_and_append_with_quotes(const char* input) noexcept; + + /** + * Append the C string directly, without escaping. + * There is no UTF-8 validation. + */ + simdjson_inline void append_raw(const char *c) noexcept; + + /** + * Append "{" to the buffer. + */ + simdjson_inline void start_object() noexcept; + + /** + * Append "}" to the buffer. + */ + simdjson_inline void end_object() noexcept; + + /** + * Append "[" to the buffer. + */ + simdjson_inline void start_array() noexcept; + + /** + * Append "]" to the buffer. + */ + simdjson_inline void end_array() noexcept; + + /** + * Append "," to the buffer. + */ + simdjson_inline void append_comma() noexcept; + + /** + * Append ":" to the buffer. + */ + simdjson_inline void append_colon() noexcept; + + /** + * Append a key-value pair to the buffer. + * The key is escaped and surrounded by double quotes. + * The value is escaped if it is a string. + */ + template + simdjson_inline void append_key_value(key_type key, value_type value) noexcept; + /** + * Append the std::string_view directly, without escaping. + * There is no UTF-8 validation. + */ + simdjson_inline void append_raw(std::string_view input) noexcept; + + /** + * Append len characters from str. + * There is no UTF-8 validation. + */ + simdjson_inline void append_raw(const char *str, size_t len) noexcept; +#if SIMDJSON_EXCEPTIONS + /** + * Creates an std::string from the written JSON buffer. + * Throws if memory allocation failed + * + * The result may not be valid UTF-8 if some of your content was not valid UTF-8. + * Use validate_unicode() to check the content if needed. + */ + simdjson_inline operator std::string() const noexcept(false); + + /** + * Creates an std::string_view from the written JSON buffer. + * Throws if memory allocation failed. + * + * The result may not be valid UTF-8 if some of your content was not valid UTF-8. + * Use validate_unicode() to check the content if needed. + */ + simdjson_inline operator std::string_view() const noexcept(false); +#endif + + /** + * Returns a view on the written JSON buffer. Returns an error + * if memory allocation failed. + * + * The result may not be valid UTF-8 if some of your content was not valid UTF-8. + * Use validate_unicode() to check the content. + */ + simdjson_inline simdjson_result view() const noexcept; + + /** + * Appends the null character to the buffer and returns + * a pointer to the beginning of the written JSON buffer. + * Returns an error if memory allocation failed. + * The result is null-terminated. + * + * The result may not be valid UTF-8 if some of your content was not valid UTF-8. + * Use validate_unicode() to check the content. + */ + simdjson_inline simdjson_result c_str() noexcept; + + /** + * Return true if the content is valid UTF-8. + */ + simdjson_inline bool validate_unicode() const noexcept; + + /** + * Returns the current size of the written JSON buffer. + * If an error occurred, returns 0. + */ + simdjson_inline size_t size() const noexcept; + +private: + /** + * Returns true if we can write at least upcoming_bytes bytes. + * The underlying buffer is reallocated if needed. It is designed + * to be called before writing to the buffer. It should be fast. + */ + simdjson_inline bool capacity_check(size_t upcoming_bytes); + + /** + * Grow the buffer to at least desired_capacity bytes. + * If the allocation fails, is_valid is set to false. We expect + * that this function would not be repeatedly called. + */ + simdjson_inline void grow_buffer(size_t desired_capacity); + + /** + * We use this helper function to make sure that is_valid is kept consistent. + */ + simdjson_inline void set_valid(bool valid) noexcept; + + std::unique_ptr buffer{}; + size_t position{0}; + size_t capacity{0}; + bool is_valid{true}; +}; + + + +} +} +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_STRING_BUILDER_H +/* end file simdjson/generic/ondemand/json_string_builder.h for westmere */ +/* including simdjson/generic/ondemand/json_string_builder-inl.h for westmere: #include "simdjson/generic/ondemand/json_string_builder-inl.h" */ +/* begin file simdjson/generic/ondemand/json_string_builder-inl.h for westmere */ +/** + * This file is part of the builder API. It is temporarily in the ondemand + * directory but we will move it to a builder directory later. + */ +#include +#include +#include +#ifndef SIMDJSON_GENERIC_STRING_BUILDER_INL_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_INL_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +/* + * Empirically, we have found that an inlined optimization is important for + * performance. The following macros are not ideal. We should find a better + * way to inline the code. + */ + +#if defined(__SSE2__) || defined(__x86_64__) || defined(__x86_64) || \ + (defined(_M_AMD64) || defined(_M_X64) || \ + (defined(_M_IX86_FP) && _M_IX86_FP == 2)) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_SSE2 +#define SIMDJSON_EXPERIMENTAL_HAS_SSE2 1 +#endif +#endif + +#if defined(__aarch64__) || defined(_M_ARM64) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_NEON +#define SIMDJSON_EXPERIMENTAL_HAS_NEON 1 +#endif +#endif +#if SIMDJSON_EXPERIMENTAL_HAS_NEON +#include +#endif +#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 +#include +#endif + +namespace simdjson { +namespace westmere { +namespace builder { + +static SIMDJSON_CONSTEXPR_LAMBDA std::array json_quotable_character = { + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + +/** + +A possible SWAR implementation of has_json_escapable_byte. It is not used because +it is slower than the current implementation. It is kept here for reference (to show +that we tried it). + +inline bool has_json_escapable_byte(uint64_t x) { + uint64_t is_ascii = 0x8080808080808080ULL & ~x; + uint64_t xor2 = x ^ 0x0202020202020202ULL; + uint64_t lt32_or_eq34 = xor2 - 0x2121212121212121ULL; + uint64_t sub92 = x ^ 0x5C5C5C5C5C5C5C5CULL; + uint64_t eq92 = (sub92 - 0x0101010101010101ULL); + return ((lt32_or_eq34 | eq92) & is_ascii) != 0; +} + +**/ + +SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline bool +simple_needs_escaping(std::string_view v) { + for (char c : v) { + // a table lookup is faster than a series of comparisons + if(json_quotable_character[static_cast(c)]) { + return true; + } + } + return false; +} + +#if SIMDJSON_EXPERIMENTAL_HAS_NEON +simdjson_inline bool fast_needs_escaping(std::string_view view) { + if (view.size() < 16) { + return simple_needs_escaping(view); + } + size_t i = 0; + uint8x16_t running = vdupq_n_u8(0); + uint8x16_t v34 = vdupq_n_u8(34); + uint8x16_t v92 = vdupq_n_u8(92); + + for (; i + 15 < view.size(); i += 16) { + uint8x16_t word = vld1q_u8((const uint8_t *)view.data() + i); + running = vorrq_u8(running, vceqq_u8(word, v34)); + running = vorrq_u8(running, vceqq_u8(word, v92)); + running = vorrq_u8(running, vcltq_u8(word, vdupq_n_u8(32))); + } + if (i < view.size()) { + uint8x16_t word = + vld1q_u8((const uint8_t *)view.data() + view.length() - 16); + running = vorrq_u8(running, vceqq_u8(word, v34)); + running = vorrq_u8(running, vceqq_u8(word, v92)); + running = vorrq_u8(running, vcltq_u8(word, vdupq_n_u8(32))); + } + return vmaxvq_u32(vreinterpretq_u32_u8(running)) != 0; +} +#elif SIMDJSON_EXPERIMENTAL_HAS_SSE2 +simdjson_inline bool fast_needs_escaping(std::string_view view) { + if (view.size() < 16) { + return simple_needs_escaping(view); + } + size_t i = 0; + __m128i running = _mm_setzero_si128(); + for (; i + 15 < view.size(); i += 16) { + + __m128i word = _mm_loadu_si128(reinterpret_cast(view.data() + i)); + running = _mm_or_si128(running, _mm_cmpeq_epi8(word, _mm_set1_epi8(34))); + running = _mm_or_si128(running, _mm_cmpeq_epi8(word, _mm_set1_epi8(92))); + running = _mm_or_si128( + running, _mm_cmpeq_epi8(_mm_subs_epu8(word, _mm_set1_epi8(31)), + _mm_setzero_si128())); + } + if (i < view.size()) { + __m128i word = + _mm_loadu_si128(reinterpret_cast(view.data() + view.length() - 16)); + running = _mm_or_si128(running, _mm_cmpeq_epi8(word, _mm_set1_epi8(34))); + running = _mm_or_si128(running, _mm_cmpeq_epi8(word, _mm_set1_epi8(92))); + running = _mm_or_si128( + running, _mm_cmpeq_epi8(_mm_subs_epu8(word, _mm_set1_epi8(31)), + _mm_setzero_si128())); + } + return _mm_movemask_epi8(running) != 0; +} +#else +simdjson_inline bool fast_needs_escaping(std::string_view view) { + return simple_needs_escaping(view); +} +#endif + + +SIMDJSON_CONSTEXPR_LAMBDA inline size_t +find_next_json_quotable_character(const std::string_view view, + size_t location) noexcept { + + for (auto pos = view.begin() + location; pos != view.end(); ++pos) { + if (json_quotable_character[static_cast(*pos)]) { + return pos - view.begin(); + } + } + return size_t(view.size()); +} + +SIMDJSON_CONSTEXPR_LAMBDA static std::string_view control_chars[] = { + "\\x0000", "\\x0001", "\\x0002", "\\x0003", "\\x0004", "\\x0005", "\\x0006", + "\\x0007", "\\x0008", "\\t", "\\n", "\\x000b", "\\f", "\\r", + "\\x000e", "\\x000f", "\\x0010", "\\x0011", "\\x0012", "\\x0013", "\\x0014", + "\\x0015", "\\x0016", "\\x0017", "\\x0018", "\\x0019", "\\x001a", "\\x001b", + "\\x001c", "\\x001d", "\\x001e", "\\x001f"}; + +SIMDJSON_CONSTEXPR_LAMBDA void escape_json_char(char c, char *&out) { + if (c == '"') { + memcpy(out, "\\\"", 2); + out += 2; + } else if (c == '\\') { + memcpy(out, "\\\\", 2); + out += 2; + } else { + std::string_view v = control_chars[uint8_t(c)]; + memcpy(out, v.data(), v.size()); + out += v.size(); + } +} + +inline size_t write_string_escaped(const std::string_view input, char *out) { + size_t mysize = input.size(); + if (!fast_needs_escaping(input)) { // fast path! + memcpy(out, input.data(), input.size()); + return input.size(); + } + const char *const initout = out; + size_t location = find_next_json_quotable_character(input, 0); + memcpy(out, input.data(), location); + out += location; + escape_json_char(input[location], out); + location += 1; + while (location < mysize) { + size_t newlocation = find_next_json_quotable_character(input, location); + memcpy(out, input.data() + location, newlocation - location); + out += newlocation - location; + location = newlocation; + if (location == mysize) { + break; + } + escape_json_char(input[location], out); + location += 1; + } + return out - initout; +} + +#if SIMDJSON_CONSTEVAL +// unoptimized, meant for compile-time execution +consteval std::string consteval_to_quoted_escaped(std::string_view input) { + std::string out = "\""; + for (char c : input) { + if (json_quotable_character[uint8_t(c)]) { + if (c == '"') { + out.append("\\\""); + } else if (c == '\\') { + out.append("\\\\"); + } else { + std::string_view v = control_chars[uint8_t(c)]; + out.append(v); + } + } else { + out.push_back(c); + } + } + out.push_back('"'); + return out; +} +#endif // SIMDJSON_CONSTEVAL + +simdjson_inline string_builder::string_builder(size_t initial_capacity) + : buffer(new(std::nothrow) char[initial_capacity]), position(0), + capacity(buffer.get() != nullptr ? initial_capacity : 0), + is_valid(buffer.get() != nullptr) {} + +simdjson_inline bool string_builder::capacity_check(size_t upcoming_bytes) { + // We use the convention that when is_valid is false, then the capacity and + // the position are 0. + // Most of the time, this function will return true. + if (simdjson_likely(upcoming_bytes <= capacity - position)) { + return true; + } + // check for overflow, most of the time there is no overflow + if (simdjson_likely(position + upcoming_bytes < position)) { + return false; + } + // We will rarely get here. + grow_buffer((std::max)(capacity * 2, position + upcoming_bytes)); + // If the buffer allocation failed, we set is_valid to false. + return is_valid; +} + +simdjson_inline void string_builder::grow_buffer(size_t desired_capacity) { + if (!is_valid) { + return; + } + std::unique_ptr new_buffer(new (std::nothrow) char[desired_capacity]); + if (new_buffer.get() == nullptr) { + set_valid(false); + return; + } + std::memcpy(new_buffer.get(), buffer.get(), position); + buffer.swap(new_buffer); + capacity = desired_capacity; +} + +simdjson_inline void string_builder::set_valid(bool valid) noexcept { + if (!valid) { + is_valid = false; + capacity = 0; + position = 0; + buffer.reset(); + } else { + is_valid = true; + } +} + +simdjson_inline size_t string_builder::size() const noexcept { + return position; +} + +simdjson_inline void string_builder::append(char c) noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = c; + } +} + +simdjson_inline void string_builder::append_null() noexcept { + constexpr char null_literal[] = "null"; + constexpr size_t null_len = sizeof(null_literal) - 1; + if (capacity_check(null_len)) { + std::memcpy(buffer.get() + position, null_literal, null_len); + position += null_len; + } +} + +simdjson_inline void string_builder::clear() noexcept { + position = 0; + // if it was invalid, we should try to repair it + if (!is_valid) { + capacity = 0; + buffer.reset(); + is_valid = true; + } +} + +namespace internal { + +// We could specialize further for 32-bit integers. +int int_log2(uint32_t x) { return (63 - leading_zeroes(x | 1)); } + +int fast_digit_count(uint32_t x) { + static uint64_t table[] = { + 4294967296, 8589934582, 8589934582, 8589934582, 12884901788, + 12884901788, 12884901788, 17179868184, 17179868184, 17179868184, + 21474826480, 21474826480, 21474826480, 21474826480, 25769703776, + 25769703776, 25769703776, 30063771072, 30063771072, 30063771072, + 34349738368, 34349738368, 34349738368, 34349738368, 38554705664, + 38554705664, 38554705664, 41949672960, 41949672960, 41949672960, + 42949672960, 42949672960}; + return uint32_t((x + table[int_log2(x)]) >> 32); +} + +int int_log2(uint64_t x) { return 63 - leading_zeroes(x | 1); } + +int fast_digit_count(uint64_t x) { + static uint64_t table[] = {9, + 99, + 999, + 9999, + 99999, + 999999, + 9999999, + 99999999, + 999999999, + 9999999999, + 99999999999, + 999999999999, + 9999999999999, + 99999999999999, + 999999999999999ULL, + 9999999999999999ULL, + 99999999999999999ULL, + 999999999999999999ULL, + 9999999999999999999ULL}; + int y = (19 * int_log2(x) >> 6); + y += x > table[y]; + return y + 1; +} + +template ::value>::type> +simdjson_inline size_t digit_count(number_type v) noexcept { + static_assert(sizeof(number_type) == 8 || sizeof(number_type) == 4 || + sizeof(number_type) == 2 || sizeof(number_type) == 1, + "We only support 8-bit, 16-bit, 32-bit and 64-bit numbers"); + return fast_digit_count(v); +} +static const char decimal_table[200] = { + 0x30, 0x30, 0x30, 0x31, 0x30, 0x32, 0x30, 0x33, 0x30, 0x34, 0x30, 0x35, + 0x30, 0x36, 0x30, 0x37, 0x30, 0x38, 0x30, 0x39, 0x31, 0x30, 0x31, 0x31, + 0x31, 0x32, 0x31, 0x33, 0x31, 0x34, 0x31, 0x35, 0x31, 0x36, 0x31, 0x37, + 0x31, 0x38, 0x31, 0x39, 0x32, 0x30, 0x32, 0x31, 0x32, 0x32, 0x32, 0x33, + 0x32, 0x34, 0x32, 0x35, 0x32, 0x36, 0x32, 0x37, 0x32, 0x38, 0x32, 0x39, + 0x33, 0x30, 0x33, 0x31, 0x33, 0x32, 0x33, 0x33, 0x33, 0x34, 0x33, 0x35, + 0x33, 0x36, 0x33, 0x37, 0x33, 0x38, 0x33, 0x39, 0x34, 0x30, 0x34, 0x31, + 0x34, 0x32, 0x34, 0x33, 0x34, 0x34, 0x34, 0x35, 0x34, 0x36, 0x34, 0x37, + 0x34, 0x38, 0x34, 0x39, 0x35, 0x30, 0x35, 0x31, 0x35, 0x32, 0x35, 0x33, + 0x35, 0x34, 0x35, 0x35, 0x35, 0x36, 0x35, 0x37, 0x35, 0x38, 0x35, 0x39, + 0x36, 0x30, 0x36, 0x31, 0x36, 0x32, 0x36, 0x33, 0x36, 0x34, 0x36, 0x35, + 0x36, 0x36, 0x36, 0x37, 0x36, 0x38, 0x36, 0x39, 0x37, 0x30, 0x37, 0x31, + 0x37, 0x32, 0x37, 0x33, 0x37, 0x34, 0x37, 0x35, 0x37, 0x36, 0x37, 0x37, + 0x37, 0x38, 0x37, 0x39, 0x38, 0x30, 0x38, 0x31, 0x38, 0x32, 0x38, 0x33, + 0x38, 0x34, 0x38, 0x35, 0x38, 0x36, 0x38, 0x37, 0x38, 0x38, 0x38, 0x39, + 0x39, 0x30, 0x39, 0x31, 0x39, 0x32, 0x39, 0x33, 0x39, 0x34, 0x39, 0x35, + 0x39, 0x36, 0x39, 0x37, 0x39, 0x38, 0x39, 0x39, +}; +} // namespace internal + +template +simdjson_inline void string_builder::append(number_type v) noexcept { + static_assert(std::is_same::value || + std::is_integral::value || + std::is_floating_point::value, + "Unsupported number type"); + // If C++17 is available, we can 'if constexpr' here. + SIMDJSON_IF_CONSTEXPR(std::is_same::value) { + if (v) { + constexpr char true_literal[] = "true"; + constexpr size_t true_len = sizeof(true_literal) - 1; + if (capacity_check(true_len)) { + std::memcpy(buffer.get() + position, true_literal, true_len); + position += true_len; + } + } else { + constexpr char false_literal[] = "false"; + constexpr size_t false_len = sizeof(false_literal) - 1; + if (capacity_check(false_len)) { + std::memcpy(buffer.get() + position, false_literal, false_len); + position += false_len; + } + } + } + else SIMDJSON_IF_CONSTEXPR(std::is_unsigned::value) { + constexpr size_t max_number_size = 20; + if (capacity_check(max_number_size)) { + using unsigned_type = typename std::make_unsigned::type; + unsigned_type pv = static_cast(v); + size_t dc = internal::digit_count(pv); + char *write_pointer = buffer.get() + position + dc - 1; + while (pv >= 100) { + memcpy(write_pointer - 1, &internal::decimal_table[(pv % 100)*2], 2); + write_pointer -= 2; + pv /= 100; + } + if (pv >= 10) { + *write_pointer-- = char('0' + (pv % 10)); + pv /= 10; + } + *write_pointer = char('0' + pv); + position += dc; + } + } + else SIMDJSON_IF_CONSTEXPR(std::is_integral::value) { + constexpr size_t max_number_size = 20; + if (capacity_check(max_number_size)) { + using unsigned_type = typename std::make_unsigned::type; + bool negative = v < 0; + unsigned_type pv = static_cast(v); + if (negative) { + pv = 0 - pv; // the 0 is for Microsoft + } + size_t dc = internal::digit_count(pv); + if (negative) { + buffer.get()[position++] = '-'; + } + char *write_pointer = buffer.get() + position + dc - 1; + while (pv >= 100) { + memcpy(write_pointer - 1, &internal::decimal_table[(pv % 100)*2], 2); + write_pointer -= 2; + pv /= 100; + } + if (pv >= 10) { + *write_pointer-- = char('0' + (pv % 10)); + pv /= 10; + } + *write_pointer = char('0' + pv); + position += dc; + } + } + else SIMDJSON_IF_CONSTEXPR(std::is_floating_point::value) { + constexpr size_t max_number_size = 24; + if (capacity_check(max_number_size)) { + // We could specialize for float. + char *end = simdjson::internal::to_chars(buffer.get() + position, nullptr, + double(v)); + position = end - buffer.get(); + } + } +} + +simdjson_inline void +string_builder::escape_and_append(std::string_view input) noexcept { + // escaping might turn a control character into \x00xx so 6 characters. + if (capacity_check(6 * input.size())) { + position += write_string_escaped(input, buffer.get() + position); + } +} + +simdjson_inline void +string_builder::escape_and_append_with_quotes(std::string_view input) noexcept { + // escaping might turn a control character into \x00xx so 6 characters. + if (capacity_check(2 + 6 * input.size())) { + buffer.get()[position++] = '"'; + position += write_string_escaped(input, buffer.get() + position); + buffer.get()[position++] = '"'; + } +} + +simdjson_inline void +string_builder::escape_and_append_with_quotes(char input) noexcept { + // escaping might turn a control character into \x00xx so 6 characters. + if (capacity_check(2 + 6 * 1)) { + buffer.get()[position++] = '"'; + std::string_view cinput(&input, 1); + position += write_string_escaped(cinput, buffer.get() + position); + buffer.get()[position++] = '"'; + } +} + +simdjson_inline void string_builder::escape_and_append_with_quotes(const char* input) noexcept { + std::string_view cinput(input); + escape_and_append_with_quotes(cinput); +} + +simdjson_inline void string_builder::append_raw(const char *c) noexcept { + size_t len = std::strlen(c); + append_raw(c, len); +} + +simdjson_inline void +string_builder::append_raw(std::string_view input) noexcept { + if (capacity_check(input.size())) { + std::memcpy(buffer.get() + position, input.data(), input.size()); + position += input.size(); + } +} + +simdjson_inline void string_builder::append_raw(const char *str, + size_t len) noexcept { + if (capacity_check(len)) { + std::memcpy(buffer.get() + position, str, len); + position += len; + } +} + +#if SIMDJSON_EXCEPTIONS +simdjson_inline string_builder::operator std::string() const noexcept(false) { + return std::string(std::string_view()); +} + +simdjson_inline string_builder::operator std::string_view() const + noexcept(false) { + return view(); +} +#endif + +simdjson_inline simdjson_result +string_builder::view() const noexcept { + if (!is_valid) { + return simdjson::OUT_OF_CAPACITY; + } + return std::string_view(buffer.get(), position); +} + +simdjson_inline simdjson_result string_builder::c_str() noexcept { + if (capacity_check(1)) { + buffer.get()[position] = '\0'; + return buffer.get(); + } + return simdjson::OUT_OF_CAPACITY; +} + +simdjson_inline bool string_builder::validate_unicode() const noexcept { + return simdjson::validate_utf8(buffer.get(), position); +} + +simdjson_inline void string_builder::start_object() noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = '{'; + } +} + + +simdjson_inline void string_builder::end_object() noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = '}'; + } +} + + +simdjson_inline void string_builder::start_array() noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = '['; + } +} + + +simdjson_inline void string_builder::end_array() noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = ']'; + } +} + + +simdjson_inline void string_builder::append_comma() noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = ','; + } +} + + +simdjson_inline void string_builder::append_colon() noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = ':'; + } +} + +template +simdjson_inline void string_builder::append_key_value(key_type key, value_type value) noexcept { + static_assert( + std::is_arithmetic::value || + std::is_same::value || + std::is_same::value || + std::is_convertible::value || + std::is_same::value, + "Unsupported value type"); + static_assert( + std::is_same::value || + std::is_convertible::value, + "Unsupported key type"); + escape_and_append_with_quotes(key); + append_colon(); + SIMDJSON_IF_CONSTEXPR(std::is_same::value) { + append_null(); + } else SIMDJSON_IF_CONSTEXPR(std::is_same::value) { + escape_and_append_with_quotes(value); + } else SIMDJSON_IF_CONSTEXPR(std::is_convertible::value) { + escape_and_append_with_quotes(value); + } else SIMDJSON_IF_CONSTEXPR(std::is_same::value) { + escape_and_append_with_quotes(value); + } else { + append(value); + } +} + +} // namespace builder +} // namespace westmere +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_STRING_BUILDER_INL_H +/* end file simdjson/generic/ondemand/json_string_builder-inl.h for westmere */ +/* including simdjson/generic/ondemand/json_builder.h for westmere: #include "simdjson/generic/ondemand/json_builder.h" */ +/* begin file simdjson/generic/ondemand/json_builder.h for westmere */ +/** + * This file is part of the builder API. It is temporarily in the ondemand directory + * but we will move it to a builder directory later. + */ +#ifndef SIMDJSON_GENERIC_BUILDER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/concepts.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ +#if SIMDJSON_STATIC_REFLECTION + +#include +#include +#include +#include +#include +#include +// #include // for std::define_static_string - header not available yet + +namespace simdjson { +namespace westmere { +namespace builder { + +// Concept that checks if a type is a container but not a string (because +// strings handling must be handled differently) +template +concept container_but_not_string = + requires(T a) { + { a.size() } -> std::convertible_to; + { + a[std::declval()] + }; // check if elements are accessible for the subscript operator + } && !std::is_same_v && + !std::is_same_v && !std::is_same_v; + +template + requires(container_but_not_string) +constexpr void atom(string_builder &b, const T &t) { + if (t.size() == 0) { + b.append_raw("[]"); + return; + } + b.append('['); + atom(b, t[0]); + for (size_t i = 1; i < t.size(); ++i) { + b.append(','); + atom(b, t[i]); + } + b.append(']'); +} + +template + requires(std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v) +constexpr void atom(string_builder &b, const T &t) { + b.escape_and_append_with_quotes(t); +} + +template +constexpr void atom(string_builder &b, const T &m) { + if (m.empty()) { + b.append_raw("{}"); + return; + } + b.append('{'); + bool first = true; + for (const auto& [key, value] : m) { + if (!first) { + b.append(','); + } + first = false; + // Keys must be convertible to string_view per the concept + b.escape_and_append_with_quotes(key); + b.append(':'); + atom(b, value); + } + b.append('}'); +} + + +template::value && !std::is_same_v>::type> +constexpr void atom(string_builder &b, const number_type t) { + b.append(t); +} +#if SIMDJSON_CONSTEVAL +consteval std::string consteval_to_quoted_escaped(std::string_view input); +#endif + +template + requires(std::is_class_v && !container_but_not_string && + !concepts::string_view_keyed_map && + !std::is_same_v && + !std::is_same_v) +constexpr void atom(string_builder &b, const T &t) { + int i = 0; + b.append('{'); + [:expand(std::meta::nonstatic_data_members_of(^^T, std::meta::access_context::unchecked())):] >> [&]() { + if (i != 0) + b.append(','); + constexpr auto key = std::define_static_string(consteval_to_quoted_escaped(std::meta::identifier_of(dm))); + b.append_raw(key); + b.append(':'); + atom(b, t.[:dm:]); + i++; + }; + b.append('}'); +} + +// works for struct +template void append(string_builder &b, const Z &z) { + int i = 0; + b.append('{'); + [:expand(std::meta::nonstatic_data_members_of(^^Z, std::meta::access_context::unchecked())):] >> [&]() { + if (i != 0) + b.append(','); + constexpr auto key = std::define_static_string(consteval_to_quoted_escaped(std::meta::identifier_of(dm))); + b.append_raw(key); + b.append(':'); + atom(b, z.[:dm:]); + i++; + }; + b.append('}'); +} + +// works for container +template + requires(container_but_not_string) +void append(string_builder &b, const Z &z) { + if (z.size() == 0) { + b.append_raw("[]"); + return; + } + b.append('['); + atom(b, z[0]); + for (size_t i = 1; i < z.size(); ++i) { + b.append(','); + atom(b, z[i]); + } + b.append(']'); +} + +template +simdjson_result to_json_string(const Z &z) { + string_builder b; + append(b, z); + std::string_view s; + if(auto e = b.view().get(s); e) { return e; } + return std::string(s); +} + +template +simdjson_error to_json(const Z &z, std::string &s) { + string_builder b; + append(b, z); + std::string_view view; + if(auto e = b.view().get(view); e) { return e; } + s.assign(view); + return SUCCESS; +} +} // namespace json_builder +} // namespace westmere +} // namespace simdjson +#endif // SIMDJSON_STATIC_REFLECTION + +#endif +/* end file simdjson/generic/ondemand/json_builder.h for westmere */ /* end file simdjson/generic/ondemand/amalgamated.h for westmere */ /* including simdjson/westmere/end.h: #include "simdjson/westmere/end.h" */ @@ -101200,6 +109953,31 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin }; } + +struct escaping { + static constexpr uint32_t BYTES_PROCESSED = 16; + simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst); + + simdjson_inline bool has_escape() { return escape_bits != 0; } + simdjson_inline int escape_index() { return trailing_zeroes(escape_bits); } + + uint64_t escape_bits; +}; // struct escaping + + + +simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) { + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v(src); + v.store(dst); + simd8 is_quote = (v == '"'); + simd8 is_backslash = (v == '\\'); + simd8 is_control = (v < 32); + return { + (is_backslash | is_quote | is_control).to_bitmask() + }; +} + } // unnamed namespace } // namespace lsx } // namespace simdjson @@ -101358,10 +110136,26 @@ concept nothrow_deserializable = nothrow_custom_deserializable || is_bu /// Deserialize Tag inline constexpr struct deserialize_tag { + using array_type = lsx::ondemand::array; + using object_type = lsx::ondemand::object; using value_type = lsx::ondemand::value; using document_type = lsx::ondemand::document; using document_reference_type = lsx::ondemand::document_reference; + // Customization Point for array + template + requires custom_deserializable + [[nodiscard]] constexpr /* error_code */ auto operator()(array_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + return tag_invoke(*this, object, output); + } + + // Customization Point for object + template + requires custom_deserializable + [[nodiscard]] constexpr /* error_code */ auto operator()(object_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + return tag_invoke(*this, object, output); + } + // Customization Point for value template requires custom_deserializable @@ -101929,6 +110723,7 @@ public: * * You may use get_double(), get_bool(), get_uint64(), get_int64(), * get_object(), get_array(), get_raw_json_string(), or get_string() instead. + * When SIMDJSON_SUPPORTS_DESERIALIZATION is set, custom types are also supported. * * @returns A value of the given type, parsed from the JSON. * @returns INCORRECT_TYPE If the JSON value is not the given type. @@ -101952,6 +110747,7 @@ public: * Get this value as the given type. * * Supported types: object, array, raw_json_string, string_view, uint64_t, int64_t, double, bool + * If the macro SIMDJSON_SUPPORTS_DESERIALIZATION is set, then custom types are also supported. * * @param out This is set to a value of the given type, parsed from the JSON. If there is an error, this may not be initialized. * @returns INCORRECT_TYPE If the JSON value is not an object. @@ -104194,6 +112990,37 @@ public: * - INDEX_OUT_OF_BOUNDS if the array index is larger than an array length */ simdjson_inline simdjson_result at(size_t index) noexcept; + +#if SIMDJSON_SUPPORTS_DESERIALIZATION + /** + * Get this array as the given type. + * + * @param out This is set to a value of the given type, parsed from the JSON. If there is an error, this may not be initialized. + * @returns INCORRECT_TYPE If the JSON array is not of the given type. + * @returns SUCCESS If the parse succeeded and the out parameter was set to the value. + */ + template + simdjson_inline error_code get(T &out) + noexcept(custom_deserializable ? nothrow_custom_deserializable : true) { + static_assert(custom_deserializable); + return deserialize(*this, out); + } + /** + * Get this array as the given type. + * + * @returns A value of the given type, parsed from the JSON. + * @returns INCORRECT_TYPE If the JSON value is not the given type. + */ + template + simdjson_inline simdjson_result get() + noexcept(custom_deserializable ? nothrow_custom_deserializable : true) + { + static_assert(std::is_default_constructible::value, "The specified type is not default constructible."); + T out{}; + SIMDJSON_TRY(get(out)); + return out; + } +#endif // SIMDJSON_SUPPORTS_DESERIALIZATION protected: /** * Go to the end of the array, no matter where you are right now. @@ -104272,7 +113099,28 @@ public: simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; simdjson_inline simdjson_result raw_json() noexcept; +#if SIMDJSON_SUPPORTS_DESERIALIZATION + // TODO: move this code into object-inl.h + template + simdjson_inline simdjson_result get() noexcept { + if (error()) { return error(); } + if constexpr (std::is_same_v) { + return first; + } + return first.get(); + } + template + simdjson_inline error_code get(T& out) noexcept { + if (error()) { return error(); } + if constexpr (std::is_same_v) { + out = first; + } else { + SIMDJSON_TRY( first.get(out) ); + } + return SUCCESS; + } +#endif // SIMDJSON_SUPPORTS_DESERIALIZATION }; } // namespace simdjson @@ -104606,7 +113454,7 @@ public: * Be mindful that the document instance must remain in scope while you are accessing object, array and value instances. * * @param out This is set to a value of the given type, parsed from the JSON. If there is an error, this may not be initialized. - * @returns INCORRECT_TYPE If the JSON value is not an object. + * @returns INCORRECT_TYPE If the JSON value is of the given type. * @returns SUCCESS If the parse succeeded and the out parameter was set to the value. */ template @@ -106093,6 +114941,36 @@ public: */ simdjson_inline simdjson_result raw_json() noexcept; +#if SIMDJSON_SUPPORTS_DESERIALIZATION + /** + * Get this object as the given type. + * + * @param out This is set to a value of the given type, parsed from the JSON. If there is an error, this may not be initialized. + * @returns INCORRECT_TYPE If the JSON object is not of the given type. + * @returns SUCCESS If the parse succeeded and the out parameter was set to the value. + */ + template + simdjson_inline error_code get(T &out) + noexcept(custom_deserializable ? nothrow_custom_deserializable : true) { + static_assert(custom_deserializable); + return deserialize(*this, out); + } + /** + * Get this array as the given type. + * + * @returns A value of the given type, parsed from the JSON. + * @returns INCORRECT_TYPE If the JSON value is not the given type. + */ + template + simdjson_inline simdjson_result get() + noexcept(custom_deserializable ? nothrow_custom_deserializable : true) + { + static_assert(std::is_default_constructible::value, "The specified type is not default constructible."); + T out{}; + SIMDJSON_TRY(get(out)); + return out; + } +#endif // SIMDJSON_SUPPORTS_DESERIALIZATION protected: /** * Go to the end of the object, no matter where you are right now. @@ -106136,12 +115014,32 @@ public: simdjson_inline simdjson_result operator[](std::string_view key) && noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - inline simdjson_result reset() noexcept; inline simdjson_result is_empty() noexcept; inline simdjson_result count_fields() & noexcept; inline simdjson_result raw_json() noexcept; + #if SIMDJSON_SUPPORTS_DESERIALIZATION + // TODO: move this code into object-inl.h + template + simdjson_inline simdjson_result get() noexcept { + if (error()) { return error(); } + if constexpr (std::is_same_v) { + return first; + } + return first.get(); + } + template + simdjson_inline error_code get(T& out) noexcept { + if (error()) { return error(); } + if constexpr (std::is_same_v) { + out = first; + } else { + SIMDJSON_TRY( first.get(out) ); + } + return SUCCESS; + } +#endif // SIMDJSON_SUPPORTS_DESERIALIZATION }; } // namespace simdjson @@ -106346,12 +115244,17 @@ inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result #include +#if SIMDJSON_STATIC_REFLECTION +#include +// #include // for std::define_static_string - header not available yet +#endif namespace simdjson { template @@ -106398,6 +115301,22 @@ error_code tag_invoke(deserialize_tag, auto &val, T &out) noexcept { return SUCCESS; } +////////////////////////////// +// String deserialization +////////////////////////////// + +// just a character! +error_code tag_invoke(deserialize_tag, auto &val, char &out) noexcept { + std::string_view x; + SIMDJSON_TRY(val.get_string().get(x)); + if(x.size() != 1) { + return INCORRECT_TYPE; + } + out = x[0]; + return SUCCESS; +} + +// any string-like type (can be constructed from std::string_view) template requires(!require_custom_serialization) error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept(std::is_nothrow_constructible_v) { @@ -106419,15 +115338,20 @@ template requires(!require_custom_serialization) error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept(false) { using value_type = typename std::remove_cvref_t::value_type; - static_assert( + /*static_assert( deserializable, - "The specified type inside the container must itself be deserializable"); + "The specified type inside the container must itself be deserializable");*/ static_assert( std::is_default_constructible_v, "The specified type inside the container must default constructible."); lsx::ondemand::array arr; - SIMDJSON_TRY(val.get_array().get(arr)); + if constexpr (std::is_same_v, lsx::ondemand::array>) { + arr = val; + } else { + SIMDJSON_TRY(val.get_array().get(arr)); + } + for (auto v : arr) { if constexpr (concepts::returns_reference) { if (auto const err = v.get().get(concepts::emplace_one(out)); @@ -106484,7 +115408,45 @@ error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept(false) { return SUCCESS; } +template +error_code tag_invoke(deserialize_tag, lsx::ondemand::object &obj, T &out) noexcept { + using value_type = typename std::remove_cvref_t::mapped_type; + out.clear(); + for (auto field : obj) { + std::string_view key; + SIMDJSON_TRY(field.unescaped_key().get(key)); + + lsx::ondemand::value value_obj; + SIMDJSON_TRY(field.value().get(value_obj)); + + value_type this_value; + SIMDJSON_TRY(value_obj.get(this_value)); + out.emplace(typename T::key_type(key), std::move(this_value)); + } + return SUCCESS; +} + +template +error_code tag_invoke(deserialize_tag, lsx::ondemand::value &val, T &out) noexcept { + lsx::ondemand::object obj; + SIMDJSON_TRY(val.get_object().get(obj)); + return simdjson::deserialize(obj, out); +} + +template +error_code tag_invoke(deserialize_tag, lsx::ondemand::document &doc, T &out) noexcept { + lsx::ondemand::object obj; + SIMDJSON_TRY(doc.get_object().get(obj)); + return simdjson::deserialize(obj, out); +} + +template +error_code tag_invoke(deserialize_tag, lsx::ondemand::document_reference &doc, T &out) noexcept { + lsx::ondemand::object obj; + SIMDJSON_TRY(doc.get_object().get(obj)); + return simdjson::deserialize(obj, out); +} /** @@ -106546,6 +115508,199 @@ error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept(nothrow_deser return SUCCESS; } + +#if SIMDJSON_STATIC_REFLECTION + + +template +constexpr bool user_defined_type = (std::is_class_v +&& !std::is_same_v && !std::is_same_v && !concepts::optional_type && +!concepts::appendable_containers && !require_custom_serialization); + + +// workaround from +// https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2024/p2996r10.html#back-and-forth +// for missing expansion statements +namespace __impl { + template + struct replicator_type { + template + constexpr void operator>>(F body) const { + (body.template operator()(), ...); + } + }; + + template + replicator_type replicator = {}; +} + +template +consteval auto expand(R range) { + std::vector args; + for (auto r : range) { + args.push_back(reflect_value(r)); + } + return substitute(^^__impl::replicator, args); +} +// end of workaround + +template + requires(user_defined_type && std::is_class_v) +error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept { + lsx::ondemand::object obj; + if constexpr (std::is_same_v, lsx::ondemand::object>) { + obj = val; + } else { + SIMDJSON_TRY(val.get_object().get(obj)); + } + error_code e = simdjson::SUCCESS; + + [:expand(std::meta::nonstatic_data_members_of(^^T, std::meta::access_context::unchecked())):] >> [&]() { + if constexpr (!std::meta::is_const(mem) && std::meta::is_public(mem)) { + constexpr std::string_view key = std::define_static_string(std::meta::identifier_of(mem)); + static_assert( + deserializable, + "The specified type inside the class must itself be deserializable"); + // as long we are succesful or the field is not found, we continue + if(e == simdjson::SUCCESS || e == simdjson::NO_SUCH_FIELD) { + obj[key].get(out.[:mem:]); + } + } + }; + return e; +} +template + requires(user_defined_type>) +error_code tag_invoke(deserialize_tag, simdjson_value &val, std::unique_ptr &out) noexcept { + if (!out) { + out = std::make_unique(); + if (!out) { + return MEMALLOC; + } + } + if (auto err = val.get(*out)) { + out.reset(); + return err; + } + return SUCCESS; +} + +template + requires(user_defined_type>) +error_code tag_invoke(deserialize_tag, simdjson_value &val, std::shared_ptr &out) noexcept { + if (!out) { + out = std::make_shared(); + if (!out) { + return MEMALLOC; + } + } + if (auto err = val.get(*out)) { + out.reset(); + return err; + } + return SUCCESS; +} + +#endif // SIMDJSON_STATIC_REFLECTION + +//////////////////////////////////////// +// Unique pointers +//////////////////////////////////////// +error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) noexcept { + if (!out) { + out = std::make_unique(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_bool().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) noexcept { + if (!out) { + out = std::make_unique(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_int64().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) noexcept { + if (!out) { + out = std::make_unique(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_uint64().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) noexcept { + if (!out) { + out = std::make_unique(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_double().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) noexcept { + if (!out) { + out = std::make_unique(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_string().get(*out)); + return SUCCESS; +} + + +//////////////////////////////////////// +// Shared pointers +//////////////////////////////////////// +error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) noexcept { + if (!out) { + out = std::make_shared(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_bool().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) noexcept { + if (!out) { + out = std::make_shared(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_int64().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) noexcept { + if (!out) { + out = std::make_shared(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_uint64().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) noexcept { + if (!out) { + out = std::make_shared(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_double().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) noexcept { + if (!out) { + out = std::make_shared(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_string().get(*out)); + return SUCCESS; +} + + } // namespace simdjson #endif // SIMDJSON_ONDEMAND_DESERIALIZE_H @@ -110033,6 +119188,7 @@ simdjson_inline simdjson_result simdjson_result::simdjson_result( #endif // SIMDJSON_GENERIC_ONDEMAND_VALUE_ITERATOR_INL_H /* end file simdjson/generic/ondemand/value_iterator-inl.h for lsx */ +// JSON builder, ideally they should not be part of the ondemand directory +// but it is convenient for now to have them here. +/* including simdjson/generic/ondemand/json_string_builder.h for lsx: #include "simdjson/generic/ondemand/json_string_builder.h" */ +/* begin file simdjson/generic/ondemand/json_string_builder.h for lsx */ +/** + * This file is part of the builder API. It is temporarily in the ondemand directory + * but we will move it to a builder directory later. + */ +#ifndef SIMDJSON_GENERIC_STRING_BUILDER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { +namespace lsx { +namespace builder { + +/** + * A builder for JSON strings representing documents. This is a low-level + * builder that is not meant to be used directly by end-users. Though it + * supports atomic types (Booleans, strings), it does not support composed + * types (arrays and objects). + * + * Ultimately, this class should support kernel-specific optimizations. E.g., + * it may make use of SIMD instructions to escape strings faster. + */ +class string_builder { +public: + simdjson_inline string_builder(size_t initial_capacity = 1024); + + /** + * Append number (includes Booleans). Booleans are mapped to the strings + * false and true. Numbers are converted to strings abiding by the JSON standard. + * Floating-point numbers are converted to the shortest string that 'correctly' + * represents the number. + */ + template::value>::type> + simdjson_inline void append(number_type v) noexcept; + + /** + * Append character c. + */ + simdjson_inline void append(char c) noexcept; + + /** + * Append the string 'null'. + */ + simdjson_inline void append_null() noexcept; + + /** + * Clear the content. + */ + simdjson_inline void clear() noexcept; + + /** + * Append the std::string_view, after escaping it. + * There is no UTF-8 validation. + */ + simdjson_inline void escape_and_append(std::string_view input) noexcept; + + /** + * Append the std::string_view surrounded by double quotes, after escaping it. + * There is no UTF-8 validation. + */ + simdjson_inline void escape_and_append_with_quotes(std::string_view input) noexcept; + + /** + * Append the character surrounded by double quotes, after escaping it. + * There is no UTF-8 validation. + */ + simdjson_inline void escape_and_append_with_quotes(char input) noexcept; + + /** + * Append the character surrounded by double quotes, after escaping it. + * There is no UTF-8 validation. + */ + simdjson_inline void escape_and_append_with_quotes(const char* input) noexcept; + + /** + * Append the C string directly, without escaping. + * There is no UTF-8 validation. + */ + simdjson_inline void append_raw(const char *c) noexcept; + + /** + * Append "{" to the buffer. + */ + simdjson_inline void start_object() noexcept; + + /** + * Append "}" to the buffer. + */ + simdjson_inline void end_object() noexcept; + + /** + * Append "[" to the buffer. + */ + simdjson_inline void start_array() noexcept; + + /** + * Append "]" to the buffer. + */ + simdjson_inline void end_array() noexcept; + + /** + * Append "," to the buffer. + */ + simdjson_inline void append_comma() noexcept; + + /** + * Append ":" to the buffer. + */ + simdjson_inline void append_colon() noexcept; + + /** + * Append a key-value pair to the buffer. + * The key is escaped and surrounded by double quotes. + * The value is escaped if it is a string. + */ + template + simdjson_inline void append_key_value(key_type key, value_type value) noexcept; + /** + * Append the std::string_view directly, without escaping. + * There is no UTF-8 validation. + */ + simdjson_inline void append_raw(std::string_view input) noexcept; + + /** + * Append len characters from str. + * There is no UTF-8 validation. + */ + simdjson_inline void append_raw(const char *str, size_t len) noexcept; +#if SIMDJSON_EXCEPTIONS + /** + * Creates an std::string from the written JSON buffer. + * Throws if memory allocation failed + * + * The result may not be valid UTF-8 if some of your content was not valid UTF-8. + * Use validate_unicode() to check the content if needed. + */ + simdjson_inline operator std::string() const noexcept(false); + + /** + * Creates an std::string_view from the written JSON buffer. + * Throws if memory allocation failed. + * + * The result may not be valid UTF-8 if some of your content was not valid UTF-8. + * Use validate_unicode() to check the content if needed. + */ + simdjson_inline operator std::string_view() const noexcept(false); +#endif + + /** + * Returns a view on the written JSON buffer. Returns an error + * if memory allocation failed. + * + * The result may not be valid UTF-8 if some of your content was not valid UTF-8. + * Use validate_unicode() to check the content. + */ + simdjson_inline simdjson_result view() const noexcept; + + /** + * Appends the null character to the buffer and returns + * a pointer to the beginning of the written JSON buffer. + * Returns an error if memory allocation failed. + * The result is null-terminated. + * + * The result may not be valid UTF-8 if some of your content was not valid UTF-8. + * Use validate_unicode() to check the content. + */ + simdjson_inline simdjson_result c_str() noexcept; + + /** + * Return true if the content is valid UTF-8. + */ + simdjson_inline bool validate_unicode() const noexcept; + + /** + * Returns the current size of the written JSON buffer. + * If an error occurred, returns 0. + */ + simdjson_inline size_t size() const noexcept; + +private: + /** + * Returns true if we can write at least upcoming_bytes bytes. + * The underlying buffer is reallocated if needed. It is designed + * to be called before writing to the buffer. It should be fast. + */ + simdjson_inline bool capacity_check(size_t upcoming_bytes); + + /** + * Grow the buffer to at least desired_capacity bytes. + * If the allocation fails, is_valid is set to false. We expect + * that this function would not be repeatedly called. + */ + simdjson_inline void grow_buffer(size_t desired_capacity); + + /** + * We use this helper function to make sure that is_valid is kept consistent. + */ + simdjson_inline void set_valid(bool valid) noexcept; + + std::unique_ptr buffer{}; + size_t position{0}; + size_t capacity{0}; + bool is_valid{true}; +}; + + + +} +} +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_STRING_BUILDER_H +/* end file simdjson/generic/ondemand/json_string_builder.h for lsx */ +/* including simdjson/generic/ondemand/json_string_builder-inl.h for lsx: #include "simdjson/generic/ondemand/json_string_builder-inl.h" */ +/* begin file simdjson/generic/ondemand/json_string_builder-inl.h for lsx */ +/** + * This file is part of the builder API. It is temporarily in the ondemand + * directory but we will move it to a builder directory later. + */ +#include +#include +#include +#ifndef SIMDJSON_GENERIC_STRING_BUILDER_INL_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_INL_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +/* + * Empirically, we have found that an inlined optimization is important for + * performance. The following macros are not ideal. We should find a better + * way to inline the code. + */ + +#if defined(__SSE2__) || defined(__x86_64__) || defined(__x86_64) || \ + (defined(_M_AMD64) || defined(_M_X64) || \ + (defined(_M_IX86_FP) && _M_IX86_FP == 2)) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_SSE2 +#define SIMDJSON_EXPERIMENTAL_HAS_SSE2 1 +#endif +#endif + +#if defined(__aarch64__) || defined(_M_ARM64) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_NEON +#define SIMDJSON_EXPERIMENTAL_HAS_NEON 1 +#endif +#endif +#if SIMDJSON_EXPERIMENTAL_HAS_NEON +#include +#endif +#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 +#include +#endif + +namespace simdjson { +namespace lsx { +namespace builder { + +static SIMDJSON_CONSTEXPR_LAMBDA std::array json_quotable_character = { + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + +/** + +A possible SWAR implementation of has_json_escapable_byte. It is not used because +it is slower than the current implementation. It is kept here for reference (to show +that we tried it). + +inline bool has_json_escapable_byte(uint64_t x) { + uint64_t is_ascii = 0x8080808080808080ULL & ~x; + uint64_t xor2 = x ^ 0x0202020202020202ULL; + uint64_t lt32_or_eq34 = xor2 - 0x2121212121212121ULL; + uint64_t sub92 = x ^ 0x5C5C5C5C5C5C5C5CULL; + uint64_t eq92 = (sub92 - 0x0101010101010101ULL); + return ((lt32_or_eq34 | eq92) & is_ascii) != 0; +} + +**/ + +SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline bool +simple_needs_escaping(std::string_view v) { + for (char c : v) { + // a table lookup is faster than a series of comparisons + if(json_quotable_character[static_cast(c)]) { + return true; + } + } + return false; +} + +#if SIMDJSON_EXPERIMENTAL_HAS_NEON +simdjson_inline bool fast_needs_escaping(std::string_view view) { + if (view.size() < 16) { + return simple_needs_escaping(view); + } + size_t i = 0; + uint8x16_t running = vdupq_n_u8(0); + uint8x16_t v34 = vdupq_n_u8(34); + uint8x16_t v92 = vdupq_n_u8(92); + + for (; i + 15 < view.size(); i += 16) { + uint8x16_t word = vld1q_u8((const uint8_t *)view.data() + i); + running = vorrq_u8(running, vceqq_u8(word, v34)); + running = vorrq_u8(running, vceqq_u8(word, v92)); + running = vorrq_u8(running, vcltq_u8(word, vdupq_n_u8(32))); + } + if (i < view.size()) { + uint8x16_t word = + vld1q_u8((const uint8_t *)view.data() + view.length() - 16); + running = vorrq_u8(running, vceqq_u8(word, v34)); + running = vorrq_u8(running, vceqq_u8(word, v92)); + running = vorrq_u8(running, vcltq_u8(word, vdupq_n_u8(32))); + } + return vmaxvq_u32(vreinterpretq_u32_u8(running)) != 0; +} +#elif SIMDJSON_EXPERIMENTAL_HAS_SSE2 +simdjson_inline bool fast_needs_escaping(std::string_view view) { + if (view.size() < 16) { + return simple_needs_escaping(view); + } + size_t i = 0; + __m128i running = _mm_setzero_si128(); + for (; i + 15 < view.size(); i += 16) { + + __m128i word = _mm_loadu_si128(reinterpret_cast(view.data() + i)); + running = _mm_or_si128(running, _mm_cmpeq_epi8(word, _mm_set1_epi8(34))); + running = _mm_or_si128(running, _mm_cmpeq_epi8(word, _mm_set1_epi8(92))); + running = _mm_or_si128( + running, _mm_cmpeq_epi8(_mm_subs_epu8(word, _mm_set1_epi8(31)), + _mm_setzero_si128())); + } + if (i < view.size()) { + __m128i word = + _mm_loadu_si128(reinterpret_cast(view.data() + view.length() - 16)); + running = _mm_or_si128(running, _mm_cmpeq_epi8(word, _mm_set1_epi8(34))); + running = _mm_or_si128(running, _mm_cmpeq_epi8(word, _mm_set1_epi8(92))); + running = _mm_or_si128( + running, _mm_cmpeq_epi8(_mm_subs_epu8(word, _mm_set1_epi8(31)), + _mm_setzero_si128())); + } + return _mm_movemask_epi8(running) != 0; +} +#else +simdjson_inline bool fast_needs_escaping(std::string_view view) { + return simple_needs_escaping(view); +} +#endif + + +SIMDJSON_CONSTEXPR_LAMBDA inline size_t +find_next_json_quotable_character(const std::string_view view, + size_t location) noexcept { + + for (auto pos = view.begin() + location; pos != view.end(); ++pos) { + if (json_quotable_character[static_cast(*pos)]) { + return pos - view.begin(); + } + } + return size_t(view.size()); +} + +SIMDJSON_CONSTEXPR_LAMBDA static std::string_view control_chars[] = { + "\\x0000", "\\x0001", "\\x0002", "\\x0003", "\\x0004", "\\x0005", "\\x0006", + "\\x0007", "\\x0008", "\\t", "\\n", "\\x000b", "\\f", "\\r", + "\\x000e", "\\x000f", "\\x0010", "\\x0011", "\\x0012", "\\x0013", "\\x0014", + "\\x0015", "\\x0016", "\\x0017", "\\x0018", "\\x0019", "\\x001a", "\\x001b", + "\\x001c", "\\x001d", "\\x001e", "\\x001f"}; + +SIMDJSON_CONSTEXPR_LAMBDA void escape_json_char(char c, char *&out) { + if (c == '"') { + memcpy(out, "\\\"", 2); + out += 2; + } else if (c == '\\') { + memcpy(out, "\\\\", 2); + out += 2; + } else { + std::string_view v = control_chars[uint8_t(c)]; + memcpy(out, v.data(), v.size()); + out += v.size(); + } +} + +inline size_t write_string_escaped(const std::string_view input, char *out) { + size_t mysize = input.size(); + if (!fast_needs_escaping(input)) { // fast path! + memcpy(out, input.data(), input.size()); + return input.size(); + } + const char *const initout = out; + size_t location = find_next_json_quotable_character(input, 0); + memcpy(out, input.data(), location); + out += location; + escape_json_char(input[location], out); + location += 1; + while (location < mysize) { + size_t newlocation = find_next_json_quotable_character(input, location); + memcpy(out, input.data() + location, newlocation - location); + out += newlocation - location; + location = newlocation; + if (location == mysize) { + break; + } + escape_json_char(input[location], out); + location += 1; + } + return out - initout; +} + +#if SIMDJSON_CONSTEVAL +// unoptimized, meant for compile-time execution +consteval std::string consteval_to_quoted_escaped(std::string_view input) { + std::string out = "\""; + for (char c : input) { + if (json_quotable_character[uint8_t(c)]) { + if (c == '"') { + out.append("\\\""); + } else if (c == '\\') { + out.append("\\\\"); + } else { + std::string_view v = control_chars[uint8_t(c)]; + out.append(v); + } + } else { + out.push_back(c); + } + } + out.push_back('"'); + return out; +} +#endif // SIMDJSON_CONSTEVAL + +simdjson_inline string_builder::string_builder(size_t initial_capacity) + : buffer(new(std::nothrow) char[initial_capacity]), position(0), + capacity(buffer.get() != nullptr ? initial_capacity : 0), + is_valid(buffer.get() != nullptr) {} + +simdjson_inline bool string_builder::capacity_check(size_t upcoming_bytes) { + // We use the convention that when is_valid is false, then the capacity and + // the position are 0. + // Most of the time, this function will return true. + if (simdjson_likely(upcoming_bytes <= capacity - position)) { + return true; + } + // check for overflow, most of the time there is no overflow + if (simdjson_likely(position + upcoming_bytes < position)) { + return false; + } + // We will rarely get here. + grow_buffer((std::max)(capacity * 2, position + upcoming_bytes)); + // If the buffer allocation failed, we set is_valid to false. + return is_valid; +} + +simdjson_inline void string_builder::grow_buffer(size_t desired_capacity) { + if (!is_valid) { + return; + } + std::unique_ptr new_buffer(new (std::nothrow) char[desired_capacity]); + if (new_buffer.get() == nullptr) { + set_valid(false); + return; + } + std::memcpy(new_buffer.get(), buffer.get(), position); + buffer.swap(new_buffer); + capacity = desired_capacity; +} + +simdjson_inline void string_builder::set_valid(bool valid) noexcept { + if (!valid) { + is_valid = false; + capacity = 0; + position = 0; + buffer.reset(); + } else { + is_valid = true; + } +} + +simdjson_inline size_t string_builder::size() const noexcept { + return position; +} + +simdjson_inline void string_builder::append(char c) noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = c; + } +} + +simdjson_inline void string_builder::append_null() noexcept { + constexpr char null_literal[] = "null"; + constexpr size_t null_len = sizeof(null_literal) - 1; + if (capacity_check(null_len)) { + std::memcpy(buffer.get() + position, null_literal, null_len); + position += null_len; + } +} + +simdjson_inline void string_builder::clear() noexcept { + position = 0; + // if it was invalid, we should try to repair it + if (!is_valid) { + capacity = 0; + buffer.reset(); + is_valid = true; + } +} + +namespace internal { + +// We could specialize further for 32-bit integers. +int int_log2(uint32_t x) { return (63 - leading_zeroes(x | 1)); } + +int fast_digit_count(uint32_t x) { + static uint64_t table[] = { + 4294967296, 8589934582, 8589934582, 8589934582, 12884901788, + 12884901788, 12884901788, 17179868184, 17179868184, 17179868184, + 21474826480, 21474826480, 21474826480, 21474826480, 25769703776, + 25769703776, 25769703776, 30063771072, 30063771072, 30063771072, + 34349738368, 34349738368, 34349738368, 34349738368, 38554705664, + 38554705664, 38554705664, 41949672960, 41949672960, 41949672960, + 42949672960, 42949672960}; + return uint32_t((x + table[int_log2(x)]) >> 32); +} + +int int_log2(uint64_t x) { return 63 - leading_zeroes(x | 1); } + +int fast_digit_count(uint64_t x) { + static uint64_t table[] = {9, + 99, + 999, + 9999, + 99999, + 999999, + 9999999, + 99999999, + 999999999, + 9999999999, + 99999999999, + 999999999999, + 9999999999999, + 99999999999999, + 999999999999999ULL, + 9999999999999999ULL, + 99999999999999999ULL, + 999999999999999999ULL, + 9999999999999999999ULL}; + int y = (19 * int_log2(x) >> 6); + y += x > table[y]; + return y + 1; +} + +template ::value>::type> +simdjson_inline size_t digit_count(number_type v) noexcept { + static_assert(sizeof(number_type) == 8 || sizeof(number_type) == 4 || + sizeof(number_type) == 2 || sizeof(number_type) == 1, + "We only support 8-bit, 16-bit, 32-bit and 64-bit numbers"); + return fast_digit_count(v); +} +static const char decimal_table[200] = { + 0x30, 0x30, 0x30, 0x31, 0x30, 0x32, 0x30, 0x33, 0x30, 0x34, 0x30, 0x35, + 0x30, 0x36, 0x30, 0x37, 0x30, 0x38, 0x30, 0x39, 0x31, 0x30, 0x31, 0x31, + 0x31, 0x32, 0x31, 0x33, 0x31, 0x34, 0x31, 0x35, 0x31, 0x36, 0x31, 0x37, + 0x31, 0x38, 0x31, 0x39, 0x32, 0x30, 0x32, 0x31, 0x32, 0x32, 0x32, 0x33, + 0x32, 0x34, 0x32, 0x35, 0x32, 0x36, 0x32, 0x37, 0x32, 0x38, 0x32, 0x39, + 0x33, 0x30, 0x33, 0x31, 0x33, 0x32, 0x33, 0x33, 0x33, 0x34, 0x33, 0x35, + 0x33, 0x36, 0x33, 0x37, 0x33, 0x38, 0x33, 0x39, 0x34, 0x30, 0x34, 0x31, + 0x34, 0x32, 0x34, 0x33, 0x34, 0x34, 0x34, 0x35, 0x34, 0x36, 0x34, 0x37, + 0x34, 0x38, 0x34, 0x39, 0x35, 0x30, 0x35, 0x31, 0x35, 0x32, 0x35, 0x33, + 0x35, 0x34, 0x35, 0x35, 0x35, 0x36, 0x35, 0x37, 0x35, 0x38, 0x35, 0x39, + 0x36, 0x30, 0x36, 0x31, 0x36, 0x32, 0x36, 0x33, 0x36, 0x34, 0x36, 0x35, + 0x36, 0x36, 0x36, 0x37, 0x36, 0x38, 0x36, 0x39, 0x37, 0x30, 0x37, 0x31, + 0x37, 0x32, 0x37, 0x33, 0x37, 0x34, 0x37, 0x35, 0x37, 0x36, 0x37, 0x37, + 0x37, 0x38, 0x37, 0x39, 0x38, 0x30, 0x38, 0x31, 0x38, 0x32, 0x38, 0x33, + 0x38, 0x34, 0x38, 0x35, 0x38, 0x36, 0x38, 0x37, 0x38, 0x38, 0x38, 0x39, + 0x39, 0x30, 0x39, 0x31, 0x39, 0x32, 0x39, 0x33, 0x39, 0x34, 0x39, 0x35, + 0x39, 0x36, 0x39, 0x37, 0x39, 0x38, 0x39, 0x39, +}; +} // namespace internal + +template +simdjson_inline void string_builder::append(number_type v) noexcept { + static_assert(std::is_same::value || + std::is_integral::value || + std::is_floating_point::value, + "Unsupported number type"); + // If C++17 is available, we can 'if constexpr' here. + SIMDJSON_IF_CONSTEXPR(std::is_same::value) { + if (v) { + constexpr char true_literal[] = "true"; + constexpr size_t true_len = sizeof(true_literal) - 1; + if (capacity_check(true_len)) { + std::memcpy(buffer.get() + position, true_literal, true_len); + position += true_len; + } + } else { + constexpr char false_literal[] = "false"; + constexpr size_t false_len = sizeof(false_literal) - 1; + if (capacity_check(false_len)) { + std::memcpy(buffer.get() + position, false_literal, false_len); + position += false_len; + } + } + } + else SIMDJSON_IF_CONSTEXPR(std::is_unsigned::value) { + constexpr size_t max_number_size = 20; + if (capacity_check(max_number_size)) { + using unsigned_type = typename std::make_unsigned::type; + unsigned_type pv = static_cast(v); + size_t dc = internal::digit_count(pv); + char *write_pointer = buffer.get() + position + dc - 1; + while (pv >= 100) { + memcpy(write_pointer - 1, &internal::decimal_table[(pv % 100)*2], 2); + write_pointer -= 2; + pv /= 100; + } + if (pv >= 10) { + *write_pointer-- = char('0' + (pv % 10)); + pv /= 10; + } + *write_pointer = char('0' + pv); + position += dc; + } + } + else SIMDJSON_IF_CONSTEXPR(std::is_integral::value) { + constexpr size_t max_number_size = 20; + if (capacity_check(max_number_size)) { + using unsigned_type = typename std::make_unsigned::type; + bool negative = v < 0; + unsigned_type pv = static_cast(v); + if (negative) { + pv = 0 - pv; // the 0 is for Microsoft + } + size_t dc = internal::digit_count(pv); + if (negative) { + buffer.get()[position++] = '-'; + } + char *write_pointer = buffer.get() + position + dc - 1; + while (pv >= 100) { + memcpy(write_pointer - 1, &internal::decimal_table[(pv % 100)*2], 2); + write_pointer -= 2; + pv /= 100; + } + if (pv >= 10) { + *write_pointer-- = char('0' + (pv % 10)); + pv /= 10; + } + *write_pointer = char('0' + pv); + position += dc; + } + } + else SIMDJSON_IF_CONSTEXPR(std::is_floating_point::value) { + constexpr size_t max_number_size = 24; + if (capacity_check(max_number_size)) { + // We could specialize for float. + char *end = simdjson::internal::to_chars(buffer.get() + position, nullptr, + double(v)); + position = end - buffer.get(); + } + } +} + +simdjson_inline void +string_builder::escape_and_append(std::string_view input) noexcept { + // escaping might turn a control character into \x00xx so 6 characters. + if (capacity_check(6 * input.size())) { + position += write_string_escaped(input, buffer.get() + position); + } +} + +simdjson_inline void +string_builder::escape_and_append_with_quotes(std::string_view input) noexcept { + // escaping might turn a control character into \x00xx so 6 characters. + if (capacity_check(2 + 6 * input.size())) { + buffer.get()[position++] = '"'; + position += write_string_escaped(input, buffer.get() + position); + buffer.get()[position++] = '"'; + } +} + +simdjson_inline void +string_builder::escape_and_append_with_quotes(char input) noexcept { + // escaping might turn a control character into \x00xx so 6 characters. + if (capacity_check(2 + 6 * 1)) { + buffer.get()[position++] = '"'; + std::string_view cinput(&input, 1); + position += write_string_escaped(cinput, buffer.get() + position); + buffer.get()[position++] = '"'; + } +} + +simdjson_inline void string_builder::escape_and_append_with_quotes(const char* input) noexcept { + std::string_view cinput(input); + escape_and_append_with_quotes(cinput); +} + +simdjson_inline void string_builder::append_raw(const char *c) noexcept { + size_t len = std::strlen(c); + append_raw(c, len); +} + +simdjson_inline void +string_builder::append_raw(std::string_view input) noexcept { + if (capacity_check(input.size())) { + std::memcpy(buffer.get() + position, input.data(), input.size()); + position += input.size(); + } +} + +simdjson_inline void string_builder::append_raw(const char *str, + size_t len) noexcept { + if (capacity_check(len)) { + std::memcpy(buffer.get() + position, str, len); + position += len; + } +} + +#if SIMDJSON_EXCEPTIONS +simdjson_inline string_builder::operator std::string() const noexcept(false) { + return std::string(std::string_view()); +} + +simdjson_inline string_builder::operator std::string_view() const + noexcept(false) { + return view(); +} +#endif + +simdjson_inline simdjson_result +string_builder::view() const noexcept { + if (!is_valid) { + return simdjson::OUT_OF_CAPACITY; + } + return std::string_view(buffer.get(), position); +} + +simdjson_inline simdjson_result string_builder::c_str() noexcept { + if (capacity_check(1)) { + buffer.get()[position] = '\0'; + return buffer.get(); + } + return simdjson::OUT_OF_CAPACITY; +} + +simdjson_inline bool string_builder::validate_unicode() const noexcept { + return simdjson::validate_utf8(buffer.get(), position); +} + +simdjson_inline void string_builder::start_object() noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = '{'; + } +} + + +simdjson_inline void string_builder::end_object() noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = '}'; + } +} + + +simdjson_inline void string_builder::start_array() noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = '['; + } +} + + +simdjson_inline void string_builder::end_array() noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = ']'; + } +} + + +simdjson_inline void string_builder::append_comma() noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = ','; + } +} + + +simdjson_inline void string_builder::append_colon() noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = ':'; + } +} + +template +simdjson_inline void string_builder::append_key_value(key_type key, value_type value) noexcept { + static_assert( + std::is_arithmetic::value || + std::is_same::value || + std::is_same::value || + std::is_convertible::value || + std::is_same::value, + "Unsupported value type"); + static_assert( + std::is_same::value || + std::is_convertible::value, + "Unsupported key type"); + escape_and_append_with_quotes(key); + append_colon(); + SIMDJSON_IF_CONSTEXPR(std::is_same::value) { + append_null(); + } else SIMDJSON_IF_CONSTEXPR(std::is_same::value) { + escape_and_append_with_quotes(value); + } else SIMDJSON_IF_CONSTEXPR(std::is_convertible::value) { + escape_and_append_with_quotes(value); + } else SIMDJSON_IF_CONSTEXPR(std::is_same::value) { + escape_and_append_with_quotes(value); + } else { + append(value); + } +} + +} // namespace builder +} // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_STRING_BUILDER_INL_H +/* end file simdjson/generic/ondemand/json_string_builder-inl.h for lsx */ +/* including simdjson/generic/ondemand/json_builder.h for lsx: #include "simdjson/generic/ondemand/json_builder.h" */ +/* begin file simdjson/generic/ondemand/json_builder.h for lsx */ +/** + * This file is part of the builder API. It is temporarily in the ondemand directory + * but we will move it to a builder directory later. + */ +#ifndef SIMDJSON_GENERIC_BUILDER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/concepts.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ +#if SIMDJSON_STATIC_REFLECTION + +#include +#include +#include +#include +#include +#include +// #include // for std::define_static_string - header not available yet + +namespace simdjson { +namespace lsx { +namespace builder { + +// Concept that checks if a type is a container but not a string (because +// strings handling must be handled differently) +template +concept container_but_not_string = + requires(T a) { + { a.size() } -> std::convertible_to; + { + a[std::declval()] + }; // check if elements are accessible for the subscript operator + } && !std::is_same_v && + !std::is_same_v && !std::is_same_v; + +template + requires(container_but_not_string) +constexpr void atom(string_builder &b, const T &t) { + if (t.size() == 0) { + b.append_raw("[]"); + return; + } + b.append('['); + atom(b, t[0]); + for (size_t i = 1; i < t.size(); ++i) { + b.append(','); + atom(b, t[i]); + } + b.append(']'); +} + +template + requires(std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v) +constexpr void atom(string_builder &b, const T &t) { + b.escape_and_append_with_quotes(t); +} + +template +constexpr void atom(string_builder &b, const T &m) { + if (m.empty()) { + b.append_raw("{}"); + return; + } + b.append('{'); + bool first = true; + for (const auto& [key, value] : m) { + if (!first) { + b.append(','); + } + first = false; + // Keys must be convertible to string_view per the concept + b.escape_and_append_with_quotes(key); + b.append(':'); + atom(b, value); + } + b.append('}'); +} + + +template::value && !std::is_same_v>::type> +constexpr void atom(string_builder &b, const number_type t) { + b.append(t); +} +#if SIMDJSON_CONSTEVAL +consteval std::string consteval_to_quoted_escaped(std::string_view input); +#endif + +template + requires(std::is_class_v && !container_but_not_string && + !concepts::string_view_keyed_map && + !std::is_same_v && + !std::is_same_v) +constexpr void atom(string_builder &b, const T &t) { + int i = 0; + b.append('{'); + [:expand(std::meta::nonstatic_data_members_of(^^T, std::meta::access_context::unchecked())):] >> [&]() { + if (i != 0) + b.append(','); + constexpr auto key = std::define_static_string(consteval_to_quoted_escaped(std::meta::identifier_of(dm))); + b.append_raw(key); + b.append(':'); + atom(b, t.[:dm:]); + i++; + }; + b.append('}'); +} + +// works for struct +template void append(string_builder &b, const Z &z) { + int i = 0; + b.append('{'); + [:expand(std::meta::nonstatic_data_members_of(^^Z, std::meta::access_context::unchecked())):] >> [&]() { + if (i != 0) + b.append(','); + constexpr auto key = std::define_static_string(consteval_to_quoted_escaped(std::meta::identifier_of(dm))); + b.append_raw(key); + b.append(':'); + atom(b, z.[:dm:]); + i++; + }; + b.append('}'); +} + +// works for container +template + requires(container_but_not_string) +void append(string_builder &b, const Z &z) { + if (z.size() == 0) { + b.append_raw("[]"); + return; + } + b.append('['); + atom(b, z[0]); + for (size_t i = 1; i < z.size(); ++i) { + b.append(','); + atom(b, z[i]); + } + b.append(']'); +} + +template +simdjson_result to_json_string(const Z &z) { + string_builder b; + append(b, z); + std::string_view s; + if(auto e = b.view().get(s); e) { return e; } + return std::string(s); +} + +template +simdjson_error to_json(const Z &z, std::string &s) { + string_builder b; + append(b, z); + std::string_view view; + if(auto e = b.view().get(view); e) { return e; } + s.assign(view); + return SUCCESS; +} +} // namespace json_builder +} // namespace lsx +} // namespace simdjson +#endif // SIMDJSON_STATIC_REFLECTION + +#endif +/* end file simdjson/generic/ondemand/json_builder.h for lsx */ /* end file simdjson/generic/ondemand/amalgamated.h for lsx */ /* including simdjson/lsx/end.h: #include "simdjson/lsx/end.h" */ @@ -112660,6 +122829,31 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin }; } + +struct escaping { + static constexpr uint32_t BYTES_PROCESSED = 16; + simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst); + + simdjson_inline bool has_escape() { return escape_bits != 0; } + simdjson_inline int escape_index() { return trailing_zeroes(escape_bits); } + + uint64_t escape_bits; +}; // struct escaping + + + +simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) { + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v(src); + v.store(dst); + simd8 is_quote = (v == '"'); + simd8 is_backslash = (v == '\\'); + simd8 is_control = (v < 32); + return { + (is_backslash | is_quote | is_control).to_bitmask() + }; +} + } // unnamed namespace } // namespace lasx } // namespace simdjson @@ -112818,10 +123012,26 @@ concept nothrow_deserializable = nothrow_custom_deserializable || is_bu /// Deserialize Tag inline constexpr struct deserialize_tag { + using array_type = lasx::ondemand::array; + using object_type = lasx::ondemand::object; using value_type = lasx::ondemand::value; using document_type = lasx::ondemand::document; using document_reference_type = lasx::ondemand::document_reference; + // Customization Point for array + template + requires custom_deserializable + [[nodiscard]] constexpr /* error_code */ auto operator()(array_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + return tag_invoke(*this, object, output); + } + + // Customization Point for object + template + requires custom_deserializable + [[nodiscard]] constexpr /* error_code */ auto operator()(object_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + return tag_invoke(*this, object, output); + } + // Customization Point for value template requires custom_deserializable @@ -113389,6 +123599,7 @@ public: * * You may use get_double(), get_bool(), get_uint64(), get_int64(), * get_object(), get_array(), get_raw_json_string(), or get_string() instead. + * When SIMDJSON_SUPPORTS_DESERIALIZATION is set, custom types are also supported. * * @returns A value of the given type, parsed from the JSON. * @returns INCORRECT_TYPE If the JSON value is not the given type. @@ -113412,6 +123623,7 @@ public: * Get this value as the given type. * * Supported types: object, array, raw_json_string, string_view, uint64_t, int64_t, double, bool + * If the macro SIMDJSON_SUPPORTS_DESERIALIZATION is set, then custom types are also supported. * * @param out This is set to a value of the given type, parsed from the JSON. If there is an error, this may not be initialized. * @returns INCORRECT_TYPE If the JSON value is not an object. @@ -115654,6 +125866,37 @@ public: * - INDEX_OUT_OF_BOUNDS if the array index is larger than an array length */ simdjson_inline simdjson_result at(size_t index) noexcept; + +#if SIMDJSON_SUPPORTS_DESERIALIZATION + /** + * Get this array as the given type. + * + * @param out This is set to a value of the given type, parsed from the JSON. If there is an error, this may not be initialized. + * @returns INCORRECT_TYPE If the JSON array is not of the given type. + * @returns SUCCESS If the parse succeeded and the out parameter was set to the value. + */ + template + simdjson_inline error_code get(T &out) + noexcept(custom_deserializable ? nothrow_custom_deserializable : true) { + static_assert(custom_deserializable); + return deserialize(*this, out); + } + /** + * Get this array as the given type. + * + * @returns A value of the given type, parsed from the JSON. + * @returns INCORRECT_TYPE If the JSON value is not the given type. + */ + template + simdjson_inline simdjson_result get() + noexcept(custom_deserializable ? nothrow_custom_deserializable : true) + { + static_assert(std::is_default_constructible::value, "The specified type is not default constructible."); + T out{}; + SIMDJSON_TRY(get(out)); + return out; + } +#endif // SIMDJSON_SUPPORTS_DESERIALIZATION protected: /** * Go to the end of the array, no matter where you are right now. @@ -115732,7 +125975,28 @@ public: simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; simdjson_inline simdjson_result raw_json() noexcept; +#if SIMDJSON_SUPPORTS_DESERIALIZATION + // TODO: move this code into object-inl.h + template + simdjson_inline simdjson_result get() noexcept { + if (error()) { return error(); } + if constexpr (std::is_same_v) { + return first; + } + return first.get(); + } + template + simdjson_inline error_code get(T& out) noexcept { + if (error()) { return error(); } + if constexpr (std::is_same_v) { + out = first; + } else { + SIMDJSON_TRY( first.get(out) ); + } + return SUCCESS; + } +#endif // SIMDJSON_SUPPORTS_DESERIALIZATION }; } // namespace simdjson @@ -116066,7 +126330,7 @@ public: * Be mindful that the document instance must remain in scope while you are accessing object, array and value instances. * * @param out This is set to a value of the given type, parsed from the JSON. If there is an error, this may not be initialized. - * @returns INCORRECT_TYPE If the JSON value is not an object. + * @returns INCORRECT_TYPE If the JSON value is of the given type. * @returns SUCCESS If the parse succeeded and the out parameter was set to the value. */ template @@ -117553,6 +127817,36 @@ public: */ simdjson_inline simdjson_result raw_json() noexcept; +#if SIMDJSON_SUPPORTS_DESERIALIZATION + /** + * Get this object as the given type. + * + * @param out This is set to a value of the given type, parsed from the JSON. If there is an error, this may not be initialized. + * @returns INCORRECT_TYPE If the JSON object is not of the given type. + * @returns SUCCESS If the parse succeeded and the out parameter was set to the value. + */ + template + simdjson_inline error_code get(T &out) + noexcept(custom_deserializable ? nothrow_custom_deserializable : true) { + static_assert(custom_deserializable); + return deserialize(*this, out); + } + /** + * Get this array as the given type. + * + * @returns A value of the given type, parsed from the JSON. + * @returns INCORRECT_TYPE If the JSON value is not the given type. + */ + template + simdjson_inline simdjson_result get() + noexcept(custom_deserializable ? nothrow_custom_deserializable : true) + { + static_assert(std::is_default_constructible::value, "The specified type is not default constructible."); + T out{}; + SIMDJSON_TRY(get(out)); + return out; + } +#endif // SIMDJSON_SUPPORTS_DESERIALIZATION protected: /** * Go to the end of the object, no matter where you are right now. @@ -117596,12 +127890,32 @@ public: simdjson_inline simdjson_result operator[](std::string_view key) && noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; - inline simdjson_result reset() noexcept; inline simdjson_result is_empty() noexcept; inline simdjson_result count_fields() & noexcept; inline simdjson_result raw_json() noexcept; + #if SIMDJSON_SUPPORTS_DESERIALIZATION + // TODO: move this code into object-inl.h + template + simdjson_inline simdjson_result get() noexcept { + if (error()) { return error(); } + if constexpr (std::is_same_v) { + return first; + } + return first.get(); + } + template + simdjson_inline error_code get(T& out) noexcept { + if (error()) { return error(); } + if constexpr (std::is_same_v) { + out = first; + } else { + SIMDJSON_TRY( first.get(out) ); + } + return SUCCESS; + } +#endif // SIMDJSON_SUPPORTS_DESERIALIZATION }; } // namespace simdjson @@ -117806,12 +128120,17 @@ inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result #include +#if SIMDJSON_STATIC_REFLECTION +#include +// #include // for std::define_static_string - header not available yet +#endif namespace simdjson { template @@ -117858,6 +128177,22 @@ error_code tag_invoke(deserialize_tag, auto &val, T &out) noexcept { return SUCCESS; } +////////////////////////////// +// String deserialization +////////////////////////////// + +// just a character! +error_code tag_invoke(deserialize_tag, auto &val, char &out) noexcept { + std::string_view x; + SIMDJSON_TRY(val.get_string().get(x)); + if(x.size() != 1) { + return INCORRECT_TYPE; + } + out = x[0]; + return SUCCESS; +} + +// any string-like type (can be constructed from std::string_view) template requires(!require_custom_serialization) error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept(std::is_nothrow_constructible_v) { @@ -117879,15 +128214,20 @@ template requires(!require_custom_serialization) error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept(false) { using value_type = typename std::remove_cvref_t::value_type; - static_assert( + /*static_assert( deserializable, - "The specified type inside the container must itself be deserializable"); + "The specified type inside the container must itself be deserializable");*/ static_assert( std::is_default_constructible_v, "The specified type inside the container must default constructible."); lasx::ondemand::array arr; - SIMDJSON_TRY(val.get_array().get(arr)); + if constexpr (std::is_same_v, lasx::ondemand::array>) { + arr = val; + } else { + SIMDJSON_TRY(val.get_array().get(arr)); + } + for (auto v : arr) { if constexpr (concepts::returns_reference) { if (auto const err = v.get().get(concepts::emplace_one(out)); @@ -117944,7 +128284,45 @@ error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept(false) { return SUCCESS; } +template +error_code tag_invoke(deserialize_tag, lasx::ondemand::object &obj, T &out) noexcept { + using value_type = typename std::remove_cvref_t::mapped_type; + out.clear(); + for (auto field : obj) { + std::string_view key; + SIMDJSON_TRY(field.unescaped_key().get(key)); + + lasx::ondemand::value value_obj; + SIMDJSON_TRY(field.value().get(value_obj)); + + value_type this_value; + SIMDJSON_TRY(value_obj.get(this_value)); + out.emplace(typename T::key_type(key), std::move(this_value)); + } + return SUCCESS; +} + +template +error_code tag_invoke(deserialize_tag, lasx::ondemand::value &val, T &out) noexcept { + lasx::ondemand::object obj; + SIMDJSON_TRY(val.get_object().get(obj)); + return simdjson::deserialize(obj, out); +} + +template +error_code tag_invoke(deserialize_tag, lasx::ondemand::document &doc, T &out) noexcept { + lasx::ondemand::object obj; + SIMDJSON_TRY(doc.get_object().get(obj)); + return simdjson::deserialize(obj, out); +} + +template +error_code tag_invoke(deserialize_tag, lasx::ondemand::document_reference &doc, T &out) noexcept { + lasx::ondemand::object obj; + SIMDJSON_TRY(doc.get_object().get(obj)); + return simdjson::deserialize(obj, out); +} /** @@ -118006,6 +128384,199 @@ error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept(nothrow_deser return SUCCESS; } + +#if SIMDJSON_STATIC_REFLECTION + + +template +constexpr bool user_defined_type = (std::is_class_v +&& !std::is_same_v && !std::is_same_v && !concepts::optional_type && +!concepts::appendable_containers && !require_custom_serialization); + + +// workaround from +// https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2024/p2996r10.html#back-and-forth +// for missing expansion statements +namespace __impl { + template + struct replicator_type { + template + constexpr void operator>>(F body) const { + (body.template operator()(), ...); + } + }; + + template + replicator_type replicator = {}; +} + +template +consteval auto expand(R range) { + std::vector args; + for (auto r : range) { + args.push_back(reflect_value(r)); + } + return substitute(^^__impl::replicator, args); +} +// end of workaround + +template + requires(user_defined_type && std::is_class_v) +error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept { + lasx::ondemand::object obj; + if constexpr (std::is_same_v, lasx::ondemand::object>) { + obj = val; + } else { + SIMDJSON_TRY(val.get_object().get(obj)); + } + error_code e = simdjson::SUCCESS; + + [:expand(std::meta::nonstatic_data_members_of(^^T, std::meta::access_context::unchecked())):] >> [&]() { + if constexpr (!std::meta::is_const(mem) && std::meta::is_public(mem)) { + constexpr std::string_view key = std::define_static_string(std::meta::identifier_of(mem)); + static_assert( + deserializable, + "The specified type inside the class must itself be deserializable"); + // as long we are succesful or the field is not found, we continue + if(e == simdjson::SUCCESS || e == simdjson::NO_SUCH_FIELD) { + obj[key].get(out.[:mem:]); + } + } + }; + return e; +} +template + requires(user_defined_type>) +error_code tag_invoke(deserialize_tag, simdjson_value &val, std::unique_ptr &out) noexcept { + if (!out) { + out = std::make_unique(); + if (!out) { + return MEMALLOC; + } + } + if (auto err = val.get(*out)) { + out.reset(); + return err; + } + return SUCCESS; +} + +template + requires(user_defined_type>) +error_code tag_invoke(deserialize_tag, simdjson_value &val, std::shared_ptr &out) noexcept { + if (!out) { + out = std::make_shared(); + if (!out) { + return MEMALLOC; + } + } + if (auto err = val.get(*out)) { + out.reset(); + return err; + } + return SUCCESS; +} + +#endif // SIMDJSON_STATIC_REFLECTION + +//////////////////////////////////////// +// Unique pointers +//////////////////////////////////////// +error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) noexcept { + if (!out) { + out = std::make_unique(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_bool().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) noexcept { + if (!out) { + out = std::make_unique(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_int64().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) noexcept { + if (!out) { + out = std::make_unique(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_uint64().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) noexcept { + if (!out) { + out = std::make_unique(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_double().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) noexcept { + if (!out) { + out = std::make_unique(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_string().get(*out)); + return SUCCESS; +} + + +//////////////////////////////////////// +// Shared pointers +//////////////////////////////////////// +error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) noexcept { + if (!out) { + out = std::make_shared(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_bool().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) noexcept { + if (!out) { + out = std::make_shared(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_int64().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) noexcept { + if (!out) { + out = std::make_shared(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_uint64().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) noexcept { + if (!out) { + out = std::make_shared(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_double().get(*out)); + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) noexcept { + if (!out) { + out = std::make_shared(); + if (!out) { return MEMALLOC; } + } + SIMDJSON_TRY(val.get_string().get(*out)); + return SUCCESS; +} + + } // namespace simdjson #endif // SIMDJSON_ONDEMAND_DESERIALIZE_H @@ -121493,6 +132064,7 @@ simdjson_inline simdjson_result simdjson_result::simdjson_result #endif // SIMDJSON_GENERIC_ONDEMAND_VALUE_ITERATOR_INL_H /* end file simdjson/generic/ondemand/value_iterator-inl.h for lasx */ +// JSON builder, ideally they should not be part of the ondemand directory +// but it is convenient for now to have them here. +/* including simdjson/generic/ondemand/json_string_builder.h for lasx: #include "simdjson/generic/ondemand/json_string_builder.h" */ +/* begin file simdjson/generic/ondemand/json_string_builder.h for lasx */ +/** + * This file is part of the builder API. It is temporarily in the ondemand directory + * but we will move it to a builder directory later. + */ +#ifndef SIMDJSON_GENERIC_STRING_BUILDER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { +namespace lasx { +namespace builder { + +/** + * A builder for JSON strings representing documents. This is a low-level + * builder that is not meant to be used directly by end-users. Though it + * supports atomic types (Booleans, strings), it does not support composed + * types (arrays and objects). + * + * Ultimately, this class should support kernel-specific optimizations. E.g., + * it may make use of SIMD instructions to escape strings faster. + */ +class string_builder { +public: + simdjson_inline string_builder(size_t initial_capacity = 1024); + + /** + * Append number (includes Booleans). Booleans are mapped to the strings + * false and true. Numbers are converted to strings abiding by the JSON standard. + * Floating-point numbers are converted to the shortest string that 'correctly' + * represents the number. + */ + template::value>::type> + simdjson_inline void append(number_type v) noexcept; + + /** + * Append character c. + */ + simdjson_inline void append(char c) noexcept; + + /** + * Append the string 'null'. + */ + simdjson_inline void append_null() noexcept; + + /** + * Clear the content. + */ + simdjson_inline void clear() noexcept; + + /** + * Append the std::string_view, after escaping it. + * There is no UTF-8 validation. + */ + simdjson_inline void escape_and_append(std::string_view input) noexcept; + + /** + * Append the std::string_view surrounded by double quotes, after escaping it. + * There is no UTF-8 validation. + */ + simdjson_inline void escape_and_append_with_quotes(std::string_view input) noexcept; + + /** + * Append the character surrounded by double quotes, after escaping it. + * There is no UTF-8 validation. + */ + simdjson_inline void escape_and_append_with_quotes(char input) noexcept; + + /** + * Append the character surrounded by double quotes, after escaping it. + * There is no UTF-8 validation. + */ + simdjson_inline void escape_and_append_with_quotes(const char* input) noexcept; + + /** + * Append the C string directly, without escaping. + * There is no UTF-8 validation. + */ + simdjson_inline void append_raw(const char *c) noexcept; + + /** + * Append "{" to the buffer. + */ + simdjson_inline void start_object() noexcept; + + /** + * Append "}" to the buffer. + */ + simdjson_inline void end_object() noexcept; + + /** + * Append "[" to the buffer. + */ + simdjson_inline void start_array() noexcept; + + /** + * Append "]" to the buffer. + */ + simdjson_inline void end_array() noexcept; + + /** + * Append "," to the buffer. + */ + simdjson_inline void append_comma() noexcept; + + /** + * Append ":" to the buffer. + */ + simdjson_inline void append_colon() noexcept; + + /** + * Append a key-value pair to the buffer. + * The key is escaped and surrounded by double quotes. + * The value is escaped if it is a string. + */ + template + simdjson_inline void append_key_value(key_type key, value_type value) noexcept; + /** + * Append the std::string_view directly, without escaping. + * There is no UTF-8 validation. + */ + simdjson_inline void append_raw(std::string_view input) noexcept; + + /** + * Append len characters from str. + * There is no UTF-8 validation. + */ + simdjson_inline void append_raw(const char *str, size_t len) noexcept; +#if SIMDJSON_EXCEPTIONS + /** + * Creates an std::string from the written JSON buffer. + * Throws if memory allocation failed + * + * The result may not be valid UTF-8 if some of your content was not valid UTF-8. + * Use validate_unicode() to check the content if needed. + */ + simdjson_inline operator std::string() const noexcept(false); + + /** + * Creates an std::string_view from the written JSON buffer. + * Throws if memory allocation failed. + * + * The result may not be valid UTF-8 if some of your content was not valid UTF-8. + * Use validate_unicode() to check the content if needed. + */ + simdjson_inline operator std::string_view() const noexcept(false); +#endif + + /** + * Returns a view on the written JSON buffer. Returns an error + * if memory allocation failed. + * + * The result may not be valid UTF-8 if some of your content was not valid UTF-8. + * Use validate_unicode() to check the content. + */ + simdjson_inline simdjson_result view() const noexcept; + + /** + * Appends the null character to the buffer and returns + * a pointer to the beginning of the written JSON buffer. + * Returns an error if memory allocation failed. + * The result is null-terminated. + * + * The result may not be valid UTF-8 if some of your content was not valid UTF-8. + * Use validate_unicode() to check the content. + */ + simdjson_inline simdjson_result c_str() noexcept; + + /** + * Return true if the content is valid UTF-8. + */ + simdjson_inline bool validate_unicode() const noexcept; + + /** + * Returns the current size of the written JSON buffer. + * If an error occurred, returns 0. + */ + simdjson_inline size_t size() const noexcept; + +private: + /** + * Returns true if we can write at least upcoming_bytes bytes. + * The underlying buffer is reallocated if needed. It is designed + * to be called before writing to the buffer. It should be fast. + */ + simdjson_inline bool capacity_check(size_t upcoming_bytes); + + /** + * Grow the buffer to at least desired_capacity bytes. + * If the allocation fails, is_valid is set to false. We expect + * that this function would not be repeatedly called. + */ + simdjson_inline void grow_buffer(size_t desired_capacity); + + /** + * We use this helper function to make sure that is_valid is kept consistent. + */ + simdjson_inline void set_valid(bool valid) noexcept; + + std::unique_ptr buffer{}; + size_t position{0}; + size_t capacity{0}; + bool is_valid{true}; +}; + + + +} +} +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_STRING_BUILDER_H +/* end file simdjson/generic/ondemand/json_string_builder.h for lasx */ +/* including simdjson/generic/ondemand/json_string_builder-inl.h for lasx: #include "simdjson/generic/ondemand/json_string_builder-inl.h" */ +/* begin file simdjson/generic/ondemand/json_string_builder-inl.h for lasx */ +/** + * This file is part of the builder API. It is temporarily in the ondemand + * directory but we will move it to a builder directory later. + */ +#include +#include +#include +#ifndef SIMDJSON_GENERIC_STRING_BUILDER_INL_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_INL_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +/* + * Empirically, we have found that an inlined optimization is important for + * performance. The following macros are not ideal. We should find a better + * way to inline the code. + */ + +#if defined(__SSE2__) || defined(__x86_64__) || defined(__x86_64) || \ + (defined(_M_AMD64) || defined(_M_X64) || \ + (defined(_M_IX86_FP) && _M_IX86_FP == 2)) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_SSE2 +#define SIMDJSON_EXPERIMENTAL_HAS_SSE2 1 +#endif +#endif + +#if defined(__aarch64__) || defined(_M_ARM64) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_NEON +#define SIMDJSON_EXPERIMENTAL_HAS_NEON 1 +#endif +#endif +#if SIMDJSON_EXPERIMENTAL_HAS_NEON +#include +#endif +#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 +#include +#endif + +namespace simdjson { +namespace lasx { +namespace builder { + +static SIMDJSON_CONSTEXPR_LAMBDA std::array json_quotable_character = { + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + +/** + +A possible SWAR implementation of has_json_escapable_byte. It is not used because +it is slower than the current implementation. It is kept here for reference (to show +that we tried it). + +inline bool has_json_escapable_byte(uint64_t x) { + uint64_t is_ascii = 0x8080808080808080ULL & ~x; + uint64_t xor2 = x ^ 0x0202020202020202ULL; + uint64_t lt32_or_eq34 = xor2 - 0x2121212121212121ULL; + uint64_t sub92 = x ^ 0x5C5C5C5C5C5C5C5CULL; + uint64_t eq92 = (sub92 - 0x0101010101010101ULL); + return ((lt32_or_eq34 | eq92) & is_ascii) != 0; +} + +**/ + +SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline bool +simple_needs_escaping(std::string_view v) { + for (char c : v) { + // a table lookup is faster than a series of comparisons + if(json_quotable_character[static_cast(c)]) { + return true; + } + } + return false; +} + +#if SIMDJSON_EXPERIMENTAL_HAS_NEON +simdjson_inline bool fast_needs_escaping(std::string_view view) { + if (view.size() < 16) { + return simple_needs_escaping(view); + } + size_t i = 0; + uint8x16_t running = vdupq_n_u8(0); + uint8x16_t v34 = vdupq_n_u8(34); + uint8x16_t v92 = vdupq_n_u8(92); + + for (; i + 15 < view.size(); i += 16) { + uint8x16_t word = vld1q_u8((const uint8_t *)view.data() + i); + running = vorrq_u8(running, vceqq_u8(word, v34)); + running = vorrq_u8(running, vceqq_u8(word, v92)); + running = vorrq_u8(running, vcltq_u8(word, vdupq_n_u8(32))); + } + if (i < view.size()) { + uint8x16_t word = + vld1q_u8((const uint8_t *)view.data() + view.length() - 16); + running = vorrq_u8(running, vceqq_u8(word, v34)); + running = vorrq_u8(running, vceqq_u8(word, v92)); + running = vorrq_u8(running, vcltq_u8(word, vdupq_n_u8(32))); + } + return vmaxvq_u32(vreinterpretq_u32_u8(running)) != 0; +} +#elif SIMDJSON_EXPERIMENTAL_HAS_SSE2 +simdjson_inline bool fast_needs_escaping(std::string_view view) { + if (view.size() < 16) { + return simple_needs_escaping(view); + } + size_t i = 0; + __m128i running = _mm_setzero_si128(); + for (; i + 15 < view.size(); i += 16) { + + __m128i word = _mm_loadu_si128(reinterpret_cast(view.data() + i)); + running = _mm_or_si128(running, _mm_cmpeq_epi8(word, _mm_set1_epi8(34))); + running = _mm_or_si128(running, _mm_cmpeq_epi8(word, _mm_set1_epi8(92))); + running = _mm_or_si128( + running, _mm_cmpeq_epi8(_mm_subs_epu8(word, _mm_set1_epi8(31)), + _mm_setzero_si128())); + } + if (i < view.size()) { + __m128i word = + _mm_loadu_si128(reinterpret_cast(view.data() + view.length() - 16)); + running = _mm_or_si128(running, _mm_cmpeq_epi8(word, _mm_set1_epi8(34))); + running = _mm_or_si128(running, _mm_cmpeq_epi8(word, _mm_set1_epi8(92))); + running = _mm_or_si128( + running, _mm_cmpeq_epi8(_mm_subs_epu8(word, _mm_set1_epi8(31)), + _mm_setzero_si128())); + } + return _mm_movemask_epi8(running) != 0; +} +#else +simdjson_inline bool fast_needs_escaping(std::string_view view) { + return simple_needs_escaping(view); +} +#endif + + +SIMDJSON_CONSTEXPR_LAMBDA inline size_t +find_next_json_quotable_character(const std::string_view view, + size_t location) noexcept { + + for (auto pos = view.begin() + location; pos != view.end(); ++pos) { + if (json_quotable_character[static_cast(*pos)]) { + return pos - view.begin(); + } + } + return size_t(view.size()); +} + +SIMDJSON_CONSTEXPR_LAMBDA static std::string_view control_chars[] = { + "\\x0000", "\\x0001", "\\x0002", "\\x0003", "\\x0004", "\\x0005", "\\x0006", + "\\x0007", "\\x0008", "\\t", "\\n", "\\x000b", "\\f", "\\r", + "\\x000e", "\\x000f", "\\x0010", "\\x0011", "\\x0012", "\\x0013", "\\x0014", + "\\x0015", "\\x0016", "\\x0017", "\\x0018", "\\x0019", "\\x001a", "\\x001b", + "\\x001c", "\\x001d", "\\x001e", "\\x001f"}; + +SIMDJSON_CONSTEXPR_LAMBDA void escape_json_char(char c, char *&out) { + if (c == '"') { + memcpy(out, "\\\"", 2); + out += 2; + } else if (c == '\\') { + memcpy(out, "\\\\", 2); + out += 2; + } else { + std::string_view v = control_chars[uint8_t(c)]; + memcpy(out, v.data(), v.size()); + out += v.size(); + } +} + +inline size_t write_string_escaped(const std::string_view input, char *out) { + size_t mysize = input.size(); + if (!fast_needs_escaping(input)) { // fast path! + memcpy(out, input.data(), input.size()); + return input.size(); + } + const char *const initout = out; + size_t location = find_next_json_quotable_character(input, 0); + memcpy(out, input.data(), location); + out += location; + escape_json_char(input[location], out); + location += 1; + while (location < mysize) { + size_t newlocation = find_next_json_quotable_character(input, location); + memcpy(out, input.data() + location, newlocation - location); + out += newlocation - location; + location = newlocation; + if (location == mysize) { + break; + } + escape_json_char(input[location], out); + location += 1; + } + return out - initout; +} + +#if SIMDJSON_CONSTEVAL +// unoptimized, meant for compile-time execution +consteval std::string consteval_to_quoted_escaped(std::string_view input) { + std::string out = "\""; + for (char c : input) { + if (json_quotable_character[uint8_t(c)]) { + if (c == '"') { + out.append("\\\""); + } else if (c == '\\') { + out.append("\\\\"); + } else { + std::string_view v = control_chars[uint8_t(c)]; + out.append(v); + } + } else { + out.push_back(c); + } + } + out.push_back('"'); + return out; +} +#endif // SIMDJSON_CONSTEVAL + +simdjson_inline string_builder::string_builder(size_t initial_capacity) + : buffer(new(std::nothrow) char[initial_capacity]), position(0), + capacity(buffer.get() != nullptr ? initial_capacity : 0), + is_valid(buffer.get() != nullptr) {} + +simdjson_inline bool string_builder::capacity_check(size_t upcoming_bytes) { + // We use the convention that when is_valid is false, then the capacity and + // the position are 0. + // Most of the time, this function will return true. + if (simdjson_likely(upcoming_bytes <= capacity - position)) { + return true; + } + // check for overflow, most of the time there is no overflow + if (simdjson_likely(position + upcoming_bytes < position)) { + return false; + } + // We will rarely get here. + grow_buffer((std::max)(capacity * 2, position + upcoming_bytes)); + // If the buffer allocation failed, we set is_valid to false. + return is_valid; +} + +simdjson_inline void string_builder::grow_buffer(size_t desired_capacity) { + if (!is_valid) { + return; + } + std::unique_ptr new_buffer(new (std::nothrow) char[desired_capacity]); + if (new_buffer.get() == nullptr) { + set_valid(false); + return; + } + std::memcpy(new_buffer.get(), buffer.get(), position); + buffer.swap(new_buffer); + capacity = desired_capacity; +} + +simdjson_inline void string_builder::set_valid(bool valid) noexcept { + if (!valid) { + is_valid = false; + capacity = 0; + position = 0; + buffer.reset(); + } else { + is_valid = true; + } +} + +simdjson_inline size_t string_builder::size() const noexcept { + return position; +} + +simdjson_inline void string_builder::append(char c) noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = c; + } +} + +simdjson_inline void string_builder::append_null() noexcept { + constexpr char null_literal[] = "null"; + constexpr size_t null_len = sizeof(null_literal) - 1; + if (capacity_check(null_len)) { + std::memcpy(buffer.get() + position, null_literal, null_len); + position += null_len; + } +} + +simdjson_inline void string_builder::clear() noexcept { + position = 0; + // if it was invalid, we should try to repair it + if (!is_valid) { + capacity = 0; + buffer.reset(); + is_valid = true; + } +} + +namespace internal { + +// We could specialize further for 32-bit integers. +int int_log2(uint32_t x) { return (63 - leading_zeroes(x | 1)); } + +int fast_digit_count(uint32_t x) { + static uint64_t table[] = { + 4294967296, 8589934582, 8589934582, 8589934582, 12884901788, + 12884901788, 12884901788, 17179868184, 17179868184, 17179868184, + 21474826480, 21474826480, 21474826480, 21474826480, 25769703776, + 25769703776, 25769703776, 30063771072, 30063771072, 30063771072, + 34349738368, 34349738368, 34349738368, 34349738368, 38554705664, + 38554705664, 38554705664, 41949672960, 41949672960, 41949672960, + 42949672960, 42949672960}; + return uint32_t((x + table[int_log2(x)]) >> 32); +} + +int int_log2(uint64_t x) { return 63 - leading_zeroes(x | 1); } + +int fast_digit_count(uint64_t x) { + static uint64_t table[] = {9, + 99, + 999, + 9999, + 99999, + 999999, + 9999999, + 99999999, + 999999999, + 9999999999, + 99999999999, + 999999999999, + 9999999999999, + 99999999999999, + 999999999999999ULL, + 9999999999999999ULL, + 99999999999999999ULL, + 999999999999999999ULL, + 9999999999999999999ULL}; + int y = (19 * int_log2(x) >> 6); + y += x > table[y]; + return y + 1; +} + +template ::value>::type> +simdjson_inline size_t digit_count(number_type v) noexcept { + static_assert(sizeof(number_type) == 8 || sizeof(number_type) == 4 || + sizeof(number_type) == 2 || sizeof(number_type) == 1, + "We only support 8-bit, 16-bit, 32-bit and 64-bit numbers"); + return fast_digit_count(v); +} +static const char decimal_table[200] = { + 0x30, 0x30, 0x30, 0x31, 0x30, 0x32, 0x30, 0x33, 0x30, 0x34, 0x30, 0x35, + 0x30, 0x36, 0x30, 0x37, 0x30, 0x38, 0x30, 0x39, 0x31, 0x30, 0x31, 0x31, + 0x31, 0x32, 0x31, 0x33, 0x31, 0x34, 0x31, 0x35, 0x31, 0x36, 0x31, 0x37, + 0x31, 0x38, 0x31, 0x39, 0x32, 0x30, 0x32, 0x31, 0x32, 0x32, 0x32, 0x33, + 0x32, 0x34, 0x32, 0x35, 0x32, 0x36, 0x32, 0x37, 0x32, 0x38, 0x32, 0x39, + 0x33, 0x30, 0x33, 0x31, 0x33, 0x32, 0x33, 0x33, 0x33, 0x34, 0x33, 0x35, + 0x33, 0x36, 0x33, 0x37, 0x33, 0x38, 0x33, 0x39, 0x34, 0x30, 0x34, 0x31, + 0x34, 0x32, 0x34, 0x33, 0x34, 0x34, 0x34, 0x35, 0x34, 0x36, 0x34, 0x37, + 0x34, 0x38, 0x34, 0x39, 0x35, 0x30, 0x35, 0x31, 0x35, 0x32, 0x35, 0x33, + 0x35, 0x34, 0x35, 0x35, 0x35, 0x36, 0x35, 0x37, 0x35, 0x38, 0x35, 0x39, + 0x36, 0x30, 0x36, 0x31, 0x36, 0x32, 0x36, 0x33, 0x36, 0x34, 0x36, 0x35, + 0x36, 0x36, 0x36, 0x37, 0x36, 0x38, 0x36, 0x39, 0x37, 0x30, 0x37, 0x31, + 0x37, 0x32, 0x37, 0x33, 0x37, 0x34, 0x37, 0x35, 0x37, 0x36, 0x37, 0x37, + 0x37, 0x38, 0x37, 0x39, 0x38, 0x30, 0x38, 0x31, 0x38, 0x32, 0x38, 0x33, + 0x38, 0x34, 0x38, 0x35, 0x38, 0x36, 0x38, 0x37, 0x38, 0x38, 0x38, 0x39, + 0x39, 0x30, 0x39, 0x31, 0x39, 0x32, 0x39, 0x33, 0x39, 0x34, 0x39, 0x35, + 0x39, 0x36, 0x39, 0x37, 0x39, 0x38, 0x39, 0x39, +}; +} // namespace internal + +template +simdjson_inline void string_builder::append(number_type v) noexcept { + static_assert(std::is_same::value || + std::is_integral::value || + std::is_floating_point::value, + "Unsupported number type"); + // If C++17 is available, we can 'if constexpr' here. + SIMDJSON_IF_CONSTEXPR(std::is_same::value) { + if (v) { + constexpr char true_literal[] = "true"; + constexpr size_t true_len = sizeof(true_literal) - 1; + if (capacity_check(true_len)) { + std::memcpy(buffer.get() + position, true_literal, true_len); + position += true_len; + } + } else { + constexpr char false_literal[] = "false"; + constexpr size_t false_len = sizeof(false_literal) - 1; + if (capacity_check(false_len)) { + std::memcpy(buffer.get() + position, false_literal, false_len); + position += false_len; + } + } + } + else SIMDJSON_IF_CONSTEXPR(std::is_unsigned::value) { + constexpr size_t max_number_size = 20; + if (capacity_check(max_number_size)) { + using unsigned_type = typename std::make_unsigned::type; + unsigned_type pv = static_cast(v); + size_t dc = internal::digit_count(pv); + char *write_pointer = buffer.get() + position + dc - 1; + while (pv >= 100) { + memcpy(write_pointer - 1, &internal::decimal_table[(pv % 100)*2], 2); + write_pointer -= 2; + pv /= 100; + } + if (pv >= 10) { + *write_pointer-- = char('0' + (pv % 10)); + pv /= 10; + } + *write_pointer = char('0' + pv); + position += dc; + } + } + else SIMDJSON_IF_CONSTEXPR(std::is_integral::value) { + constexpr size_t max_number_size = 20; + if (capacity_check(max_number_size)) { + using unsigned_type = typename std::make_unsigned::type; + bool negative = v < 0; + unsigned_type pv = static_cast(v); + if (negative) { + pv = 0 - pv; // the 0 is for Microsoft + } + size_t dc = internal::digit_count(pv); + if (negative) { + buffer.get()[position++] = '-'; + } + char *write_pointer = buffer.get() + position + dc - 1; + while (pv >= 100) { + memcpy(write_pointer - 1, &internal::decimal_table[(pv % 100)*2], 2); + write_pointer -= 2; + pv /= 100; + } + if (pv >= 10) { + *write_pointer-- = char('0' + (pv % 10)); + pv /= 10; + } + *write_pointer = char('0' + pv); + position += dc; + } + } + else SIMDJSON_IF_CONSTEXPR(std::is_floating_point::value) { + constexpr size_t max_number_size = 24; + if (capacity_check(max_number_size)) { + // We could specialize for float. + char *end = simdjson::internal::to_chars(buffer.get() + position, nullptr, + double(v)); + position = end - buffer.get(); + } + } +} + +simdjson_inline void +string_builder::escape_and_append(std::string_view input) noexcept { + // escaping might turn a control character into \x00xx so 6 characters. + if (capacity_check(6 * input.size())) { + position += write_string_escaped(input, buffer.get() + position); + } +} + +simdjson_inline void +string_builder::escape_and_append_with_quotes(std::string_view input) noexcept { + // escaping might turn a control character into \x00xx so 6 characters. + if (capacity_check(2 + 6 * input.size())) { + buffer.get()[position++] = '"'; + position += write_string_escaped(input, buffer.get() + position); + buffer.get()[position++] = '"'; + } +} + +simdjson_inline void +string_builder::escape_and_append_with_quotes(char input) noexcept { + // escaping might turn a control character into \x00xx so 6 characters. + if (capacity_check(2 + 6 * 1)) { + buffer.get()[position++] = '"'; + std::string_view cinput(&input, 1); + position += write_string_escaped(cinput, buffer.get() + position); + buffer.get()[position++] = '"'; + } +} + +simdjson_inline void string_builder::escape_and_append_with_quotes(const char* input) noexcept { + std::string_view cinput(input); + escape_and_append_with_quotes(cinput); +} + +simdjson_inline void string_builder::append_raw(const char *c) noexcept { + size_t len = std::strlen(c); + append_raw(c, len); +} + +simdjson_inline void +string_builder::append_raw(std::string_view input) noexcept { + if (capacity_check(input.size())) { + std::memcpy(buffer.get() + position, input.data(), input.size()); + position += input.size(); + } +} + +simdjson_inline void string_builder::append_raw(const char *str, + size_t len) noexcept { + if (capacity_check(len)) { + std::memcpy(buffer.get() + position, str, len); + position += len; + } +} + +#if SIMDJSON_EXCEPTIONS +simdjson_inline string_builder::operator std::string() const noexcept(false) { + return std::string(std::string_view()); +} + +simdjson_inline string_builder::operator std::string_view() const + noexcept(false) { + return view(); +} +#endif + +simdjson_inline simdjson_result +string_builder::view() const noexcept { + if (!is_valid) { + return simdjson::OUT_OF_CAPACITY; + } + return std::string_view(buffer.get(), position); +} + +simdjson_inline simdjson_result string_builder::c_str() noexcept { + if (capacity_check(1)) { + buffer.get()[position] = '\0'; + return buffer.get(); + } + return simdjson::OUT_OF_CAPACITY; +} + +simdjson_inline bool string_builder::validate_unicode() const noexcept { + return simdjson::validate_utf8(buffer.get(), position); +} + +simdjson_inline void string_builder::start_object() noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = '{'; + } +} + + +simdjson_inline void string_builder::end_object() noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = '}'; + } +} + + +simdjson_inline void string_builder::start_array() noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = '['; + } +} + + +simdjson_inline void string_builder::end_array() noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = ']'; + } +} + + +simdjson_inline void string_builder::append_comma() noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = ','; + } +} + + +simdjson_inline void string_builder::append_colon() noexcept { + if (capacity_check(1)) { + buffer.get()[position++] = ':'; + } +} + +template +simdjson_inline void string_builder::append_key_value(key_type key, value_type value) noexcept { + static_assert( + std::is_arithmetic::value || + std::is_same::value || + std::is_same::value || + std::is_convertible::value || + std::is_same::value, + "Unsupported value type"); + static_assert( + std::is_same::value || + std::is_convertible::value, + "Unsupported key type"); + escape_and_append_with_quotes(key); + append_colon(); + SIMDJSON_IF_CONSTEXPR(std::is_same::value) { + append_null(); + } else SIMDJSON_IF_CONSTEXPR(std::is_same::value) { + escape_and_append_with_quotes(value); + } else SIMDJSON_IF_CONSTEXPR(std::is_convertible::value) { + escape_and_append_with_quotes(value); + } else SIMDJSON_IF_CONSTEXPR(std::is_same::value) { + escape_and_append_with_quotes(value); + } else { + append(value); + } +} + +} // namespace builder +} // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_STRING_BUILDER_INL_H +/* end file simdjson/generic/ondemand/json_string_builder-inl.h for lasx */ +/* including simdjson/generic/ondemand/json_builder.h for lasx: #include "simdjson/generic/ondemand/json_builder.h" */ +/* begin file simdjson/generic/ondemand/json_builder.h for lasx */ +/** + * This file is part of the builder API. It is temporarily in the ondemand directory + * but we will move it to a builder directory later. + */ +#ifndef SIMDJSON_GENERIC_BUILDER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/concepts.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ +#if SIMDJSON_STATIC_REFLECTION + +#include +#include +#include +#include +#include +#include +// #include // for std::define_static_string - header not available yet + +namespace simdjson { +namespace lasx { +namespace builder { + +// Concept that checks if a type is a container but not a string (because +// strings handling must be handled differently) +template +concept container_but_not_string = + requires(T a) { + { a.size() } -> std::convertible_to; + { + a[std::declval()] + }; // check if elements are accessible for the subscript operator + } && !std::is_same_v && + !std::is_same_v && !std::is_same_v; + +template + requires(container_but_not_string) +constexpr void atom(string_builder &b, const T &t) { + if (t.size() == 0) { + b.append_raw("[]"); + return; + } + b.append('['); + atom(b, t[0]); + for (size_t i = 1; i < t.size(); ++i) { + b.append(','); + atom(b, t[i]); + } + b.append(']'); +} + +template + requires(std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v) +constexpr void atom(string_builder &b, const T &t) { + b.escape_and_append_with_quotes(t); +} + +template +constexpr void atom(string_builder &b, const T &m) { + if (m.empty()) { + b.append_raw("{}"); + return; + } + b.append('{'); + bool first = true; + for (const auto& [key, value] : m) { + if (!first) { + b.append(','); + } + first = false; + // Keys must be convertible to string_view per the concept + b.escape_and_append_with_quotes(key); + b.append(':'); + atom(b, value); + } + b.append('}'); +} + + +template::value && !std::is_same_v>::type> +constexpr void atom(string_builder &b, const number_type t) { + b.append(t); +} +#if SIMDJSON_CONSTEVAL +consteval std::string consteval_to_quoted_escaped(std::string_view input); +#endif + +template + requires(std::is_class_v && !container_but_not_string && + !concepts::string_view_keyed_map && + !std::is_same_v && + !std::is_same_v) +constexpr void atom(string_builder &b, const T &t) { + int i = 0; + b.append('{'); + [:expand(std::meta::nonstatic_data_members_of(^^T, std::meta::access_context::unchecked())):] >> [&]() { + if (i != 0) + b.append(','); + constexpr auto key = std::define_static_string(consteval_to_quoted_escaped(std::meta::identifier_of(dm))); + b.append_raw(key); + b.append(':'); + atom(b, t.[:dm:]); + i++; + }; + b.append('}'); +} + +// works for struct +template void append(string_builder &b, const Z &z) { + int i = 0; + b.append('{'); + [:expand(std::meta::nonstatic_data_members_of(^^Z, std::meta::access_context::unchecked())):] >> [&]() { + if (i != 0) + b.append(','); + constexpr auto key = std::define_static_string(consteval_to_quoted_escaped(std::meta::identifier_of(dm))); + b.append_raw(key); + b.append(':'); + atom(b, z.[:dm:]); + i++; + }; + b.append('}'); +} + +// works for container +template + requires(container_but_not_string) +void append(string_builder &b, const Z &z) { + if (z.size() == 0) { + b.append_raw("[]"); + return; + } + b.append('['); + atom(b, z[0]); + for (size_t i = 1; i < z.size(); ++i) { + b.append(','); + atom(b, z[i]); + } + b.append(']'); +} + +template +simdjson_result to_json_string(const Z &z) { + string_builder b; + append(b, z); + std::string_view s; + if(auto e = b.view().get(s); e) { return e; } + return std::string(s); +} + +template +simdjson_error to_json(const Z &z, std::string &s) { + string_builder b; + append(b, z); + std::string_view view; + if(auto e = b.view().get(view); e) { return e; } + s.assign(view); + return SUCCESS; +} +} // namespace json_builder +} // namespace lasx +} // namespace simdjson +#endif // SIMDJSON_STATIC_REFLECTION + +#endif +/* end file simdjson/generic/ondemand/json_builder.h for lasx */ /* end file simdjson/generic/ondemand/amalgamated.h for lasx */ /* including simdjson/lasx/end.h: #include "simdjson/lasx/end.h" */ @@ -123526,6 +135111,10 @@ namespace simdjson { * @copydoc simdjson::builtin::ondemand */ namespace ondemand = builtin::ondemand; + /** + * @copydoc simdjson::builtin::builder + */ + namespace builder = builtin::builder; } // namespace simdjson #endif // SIMDJSON_ONDEMAND_H diff --git a/singleheader/singleheader.zip b/singleheader/singleheader.zip index a34d0971d721b338432a033676ae5ed765bbae54..f6c110e28bb9907b1978ed509b48ee7324f255e2 100644 GIT binary patch delta 4764 zcmeI0eRNah9mh?3^U|bEZrY>`=}Vr5CQV|R+(MwVDKsrbEQL~}1%a|QG`DSPnuH{+ zrYPcW)@IKn49 zYRScdIIk7suJLdrlyqN-Mh&wUZB;P;ViVJm<2E-dX$9G}M~Epn9!a?cuh$CepiEr? z`)6B9pwJ>)D;HygnF}`3Pf{^bPp|_0+2Vax0l{Vv3w@=kNs<$eN4*%=WNYP+);fvZ zPO7imla?I}DoSAbZ?+=^Qe9%v2uxqFRj&|S?uw;GNPqvv?w*bf9oyIR_b+f6Oz{8V z^B~t=R&otTg!U2HHXFPcHyR+FU;Mj!?=CI^F0a@E@w*ES)BCIK56i2)UNs?JmkU3M zi>!{t_lRAb2u8+vp&~(;S?o8mZi4adA$&BrE5e7!5Y%USN?`Xn^DvMXii%*W%vcMP zz3q%9nF=*EB~yuTbQosZWCoBm`km?ozk584n}s-FQ)Zh*<^=r{?REKt_`kV9_Vk9l z)f$*BtGCP4u&FQ4pd02>j{h(;(<`gJiXrzZFpHIZm-P}-KeF^{-# zI(o=TxbQ};0R}HNl|uisTU2myyG#$d-v^cXZekTZw=VE(ccLXeiF-brjK`v3%%0?0 z#KZVmy8RPj{#GkgKQLMbB(j!)N34qsaH%`5ytuh%)hdT8H4;ubJ%Wb`Crk~Ml~;L! z$z*sq>X`6^f~lax=_V7NWOy&{a60{J0(!@WQi#9f<6u6$l7)djJCh0DoJYeq=l^I0 z-SZv0R!3W>>`M#b{T`X^(9%*X+@^C?P&VHcErg?aE;~%xJa#($Z2qz~nC*3yz|<+l z3Yh#fUk~$~+z8wVqksXw!T>QH)9{p&c~36QV^0TGLAutZhXJ>1i8x9(Dw#^9G#tOA zZiaXr4vS{kXX;d-t8=lCHZgwqi;gL~cNL?cFX))(vMbm-i+u3WS#1HNr?srKY_d(K zqHz;bot={&4&vrKYd3)9CcPYDX68zm@)ENY!wJG1S4c6;oi(n4xrau*&|zhy1nBb) zrdk{}z`%^z3$Ltm8M7gFyG)`9z&UE@s&fE_zopa(b1dL>SCs<$QlJ#7{0bJ>Muj}H zZAL}6%@o}A%KvT(0~kRWOr21fq2(=$8Y~&L3g)-Dl~7}4DnRU@csZ%oC=Z>|+Nk&b zI@5mO-c+a{&}u3MR-;h@XVMrT+G=Xb;3Z`>g}wW7vcYZP6SJY1%~Lz2@bPA~6h3-a z9lBU3hr@TMH9{Uz7{aa0^`8{d>pv;F@3fT5)wT=Zb}?j5p41{bIC+w7%5l%_WY@x! zQ^mr!`&C}ho+?s8`cx4E@#ESe%Du+&p=?NPtQ9zSl@SKIHmTs)ajiV_s9#SX^%oy` zSqk84dojbwODHEd+;mZNOtFhC+a;^w5}{8SGH66fDH$xgY?SOR*sWqWCU+avFm)lQ zfTP1kjntQiUFHHHttPQAHsLp|*bUc+G4_(!jSO~!23i+Y?9WM^6q|3%N!{}DQWZ>f znW_NyQ3b@#50=1(n@w5>7>sh@3XH`t`5N0S45(oG8Iv8RrdT;V_BvZk+1J_AGKg=& zo84)Hh929cIw1$a0ZpG-^l=YtX$+H@x!AUcqmy`B3@>Wy3F*b ztNg~Ng$n%l11Xka`iG+ow0O-*xa={DWzazR_sr^y=k0RpdAnlApK=nN9II#*g4M&_ z{VEPF{TY{8q7`N-(GQkKv@n0TvLq+RH!CA1I&h})-7Qj^=KoP%nZY(uuiNt0&xP3T zln%$-#|inN2dk8sy7LxVcb>S;=lGpk&eTf>b>MX6N_foIV4&%9)#tKCakxq-M;XlC zQ^i2HeV-iOT)#rv<(d6HE8Xu~d?+IAg#`~+u?T_XZZ*s|lqzV;4;IU|=6F7Kz`ZQz z5#brP7Q$rEjW69ZN*zrrYo=e462pnvmY}f$n0sKc9G*W^mpy4Q86(}Jdw*N=jl3d3 zc6Q$P#0hrmmg`thzgpv6pgSq*PR{j}=oCcoSE4f{3f(UyI*%&o)_>VPl_`84&cj)u z-XSZ4*d<#^=7V=G*R@T*bn!|uMuMO4|~FdLxyr#%KBrrO&xPj))!lbz0) zH>3ycRn)E#oZT)5#9j_K;Wjs_R?deqarTkL1=3~u9eo9KZ?%e-Mu=zBENl$mBUKc{ zqyS~^QUW7yH zktN7d#D^?HmLm*Yb6=_3OBkjl<F$&(gpoY_m%#t`|7Z zPPf>uex2QJm#wvr+3gl>r?)_dDPD&iT&y<+~@}4e$P4`M@jhD>IsxDjDMy^mQov!xr_Te#LVy&A8&Z zm-c2jQp!nLMRj>qb>(bQEr?7QGW5TCEU$3?g29~H zJm_)8G2!DI;)LTHl=sa6%{%ioO9Mf_+q;PbeI(e$6JIFU6AF@!P-iFaCw*OR4^Q}j zlk1^%T)t3;ho8AME9)N@5{XRGPG9Vm_KL?zro66o@UJTB73_3~sE zDNL4HED5-G@OBpVm~DncRL0K-Lw+xOyuz9RLr)sB_FY%M5Dv^%q`{_rO1tn2P`;Z2 zx9w0S!Kq}6s%b*sZ_hID-j8fr_~^G*b^kXSg-jUm+cC>|R>^?7uWMDqCo8OG*n9t+ z94OtPOxQP2-vDp#Q5xaEym-Cv)BBXcRv1ZT%wmDd7M~8fS&RYd8`b4NzGPKUe3-3= z;}&MN@O&KexA>_l>-I0mgQ2mZ?vP&?{K;Gxgwsrw^udwc;>OB^-yrb2h8U0e+!3l^!X z9hXpDy)8GxQTWMeH?ED$T^PptoE|wVFtsqKiMv!5^#%(SiI$Xv zm2`1|c4>RD!``Qj>5Y>9MI=%#m9>jk?X2icI6uT4gcIG3a|T6M;)3Fs2FBHet<`M? zcrK@2n=sY`IJC`T$=@XojHravSZyt8j9Xu1Tyf~2jO9#TGFB7k3pU#8_6I|p$Fp7B zPlJBkR7t=``gr2xyu{<ZY*<><_#udyyf>_Tc`k0Qfz>E<_oo$(L-e6I0ZG=A<^z3FgNy+r3GWS7(lH})yo z6L3Jq?nz$ej#NW0Bj*nZT9eWOf^};m1B1h|gvXTvN5JRxY={13_6++}lOo2|cNLau zh>*-Ixm%HdE*0d0ZYRw9ba917)TWjCBpmwZ;!O2U$@;V~v)|7LLZ0BxNN+^%Be)>o znlbLBkw52>it>n;6%~!G1auT75$OHsqitx8GO5dGqR22)7Bxv`HQ8y)vlSED`t`P) zT^E!Dx&I)Uj5N#`goGcM$ZNavhBZ@}#u zb0st>zpV$WM6I%@Bnpj?y0OKTuJU-v&u>_q2^}!7#Gb}~8Y_y`pLz-Mn{y0&dU+e870DdIU>AKBK(&l!XG6f zcsU}15fQ)qtU>R<`4#m0JnfVs1zMeLB~D+@cDgIzet`pAOlCzE1L`Mu0m@ujBMdE1 zNJ)trXeX3?I?trW8a0|-8hCp7LIVsB=2udpyAX&iK@UfRxeoB@;}eDErUZT?e0eO} zWFFh!yL&u*H}9oRk0hd+QYh2DyXsj8zHwzH^#3{`F`0hK!cRnWua+uEY)|pC3==q> zQxoWaIUE0KRjI<~9!?O>J)C&xXf)djBO?}ucP6ELA+sl$JxL$QW%eYqCuETqMO|g~ zBrz$OJ<-6J%$_cR#L4XGve;8|&CE&|9k%li|PV7M_U6OO-X%!1LSTD345(w4Pd z3Xfu5(q>FVrzmp@>rQ9|-HGILC!(p67^S2^!7v5|5vdp))csJVO)_ngX%o@~Ip`z> z-(=cEKNrhEr$|I!4mwSaeaS(mi4fs;8gxS55@AjM*_NCHJyod@)-Rv?qA zN=@1SqH!fYuv?i0BL|G>FuhNkCJgjyPbgsUFI;5G$=^Yn`5|_BUvvvlGFbB>=fKiKieB3GwzGhhZ51EL`L`)`PG7-D9 zN3U}HN{(NRKTntASIFC=o?%_+Ii?)Hx~%xsz$!x>^jH(j@Io-R6pk+{i-(6eBY~0K z+D!00-=c$gPD9FLFB;i(7f+@#!7;;>9!;j8`2qcO*!^;LI*=m03Wm3o$HU29E(tcC zvS^@m8Y5i`-)S`=(uF&%=2uKp5w8m!Sepi?J>}EHgIVFZwPr+yFt^t7uLUw8lL?tj z$Yeq$6EgWYmV6xRn@)wvk+J_%WDEuhEK6XtU#F7@*|9wp2{7ob(8Ewj%R;l&l256b zX@4(%9bohxrxqOMj68aND}GXZOc=f1;%J30j%BA`a8Z&N(y$ptWGYy~%pX@9x`ml(!f0Ra>3Go9 zWz|n&V8hpE-*fe)#LCu>W*cK@m*d`P*#|ycF8o5MD9V%I~Z4`hMI@% z>EQFQDRAsf)+`KeQNB|oCnJ=T5nebM;iJXY+Rd{ZR2E!Umqc{8_tJ!axnFuE}7>pj;qRaxF zp22TIZ&B)nvyZcUY!hPoL(gHB6;h9|U&f>n9RGf^nDh{6HK@&9mk+nSmZpQzH(3*m z7VE_Xhmpok(|m`H7craY>Bm_e5QAPNU^c|7Vw)9{9!`-fQNVi@Q)vNTd*IY@idBQ} zFl&X;ZIwBK_7(OWCdSlau0&D$A?qw69C?%dAhrsP=Z7m#v4}!M5uz9|6H$UFMK};; zh;l>)q7v}~#4N;Y#2iEwq8d?yn2V@I)FI{}>Jjr13lIwtix3TnD-nwkOAw8Ss}M^O z%Mi;ES0kDb&4?9^@uja21GkzBZ5P8Ae;ynf=6^B zHX*tYZp3E97K8`UjqoCTh#tgNgdY(=1Q8)bFQN~zO{hGzebtGS=q$Fd|L~6D1yTDI r?pYc!->vlP-aVgUzT1Gi+qF61^OkfqFO65xk~sY5$Nz`eo%s8|u-WA3 diff --git a/src/arm64.cpp b/src/arm64.cpp index 7e3a74878..e017ce016 100644 --- a/src/arm64.cpp +++ b/src/arm64.cpp @@ -165,10 +165,6 @@ simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t * return stage2(_doc); } -simdjson_warn_unused size_t implementation::write_string_escaped(const std::string_view input, char *out) const noexcept { - return arm64::stringparsing::write_string_escaped(input, out); -} - } // namespace arm64 } // namespace simdjson diff --git a/src/fallback.cpp b/src/fallback.cpp index 46881a228..dbecb2f32 100644 --- a/src/fallback.cpp +++ b/src/fallback.cpp @@ -403,10 +403,6 @@ simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t * return stage2(_doc); } -simdjson_warn_unused size_t implementation::write_string_escaped(const std::string_view input, char *out) const noexcept { - return fallback::stringparsing::write_string_escaped(input, out); -} - } // namespace fallback } // namespace simdjson diff --git a/src/generic/stage2/stringparsing.h b/src/generic/stage2/stringparsing.h index cd229b0cb..93850a8db 100644 --- a/src/generic/stage2/stringparsing.h +++ b/src/generic/stage2/stringparsing.h @@ -237,104 +237,6 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t } } -///////////// -/// TODO: This function is not used in the codebase. It is not clear if it is needed. -///////////// -simdjson_warn_unused size_t write_string_escaped(const std::string_view input, char *out) noexcept { - // We are making the following assumption: most strings will either be very short or they will not - // need escaping. - size_t i = 0; - size_t pos = 0; - /*if(input.size() >= escaping::BYTES_PROCESSED) { - auto vec_processing = [input,out]() -> size_t { - size_t index = 0; - size_t position = 0; - for(;input.size() - index >= escaping::BYTES_PROCESSED; index += escaping::BYTES_PROCESSED) { - escaping vinput = escaping::copy_and_find(reinterpret_cast(input.data()) + index, reinterpret_cast(out) + position); - if(vinput.has_escape()) { - return index + vinput.escape_index(); // We have a character that needs escaping - } - position += escaping::BYTES_PROCESSED; - } - if(index == input.size()) { return input.size(); } - // We virtually backtrack so we can load a full vector register - index = input.size() - escaping::BYTES_PROCESSED; - position = index; - escaping vinput = escaping::copy_and_find(reinterpret_cast(input.data()) + index, reinterpret_cast(out) + position); - if(vinput.has_escape()) { - return index + vinput.escape_index(); // We have a character that needs escaping - } - return input.size(); - }; - i = vec_processing(); - pos = i; - if(i == input.size()) { return pos; } - // Here we only continue if there was a character that needed escaping. - }*/ - static std::string_view control_chars[] = { - "\\x0000", "\\x0001", "\\x0002", "\\x0003", "\\x0004", "\\x0005", "\\x0006", - "\\x0007", "\\x0008", "\\t", "\\n", "\\x000b", "\\f", "\\r", - "\\x000e", "\\x000f", "\\x0010", "\\x0011", "\\x0012", "\\x0013", "\\x0014", - "\\x0015", "\\x0016", "\\x0017", "\\x0018", "\\x0019", "\\x001a", "\\x001b", - "\\x001c", "\\x001d", "\\x001e", "\\x001f"}; - static std::array json_quotable_character = - []() SIMDJSON_CONSTEXPR_LAMBDA { - std::array result{}; - for (int index = 0; index < 32; index++) { - result[index] = 1; - } - for (int index : {'"', '\\'}) { - result[index] = 1; - } - return result; - }(); - // The rest could possibly be vectorized, but consider that we expect most strings - // to be short or not to require escaping. - for (; i < input.size(); i++) { - uint8_t c = static_cast(input[i]); - if(json_quotable_character[c]) { - switch (c) { - case '"': - out[pos++] = '\\'; - out[pos++] = '"'; - break; - case '\\': - out[pos++] = '\\'; - out[pos++] = '\\'; - break; - case '\b': - out[pos++] = '\\'; - out[pos++] = 'b'; - break; - case '\f': - out[pos++] = '\\'; - out[pos++] = 'f'; - break; - case '\n': - out[pos++] = '\\'; - out[pos++] = 'n'; - break; - case '\r': - out[pos++] = '\\'; - out[pos++] = 'r'; - break; - case '\t': - out[pos++] = '\\'; - out[pos++] = 't'; - break; - default: - control_chars[c].copy(out + pos, 6); - pos += 6; - } - } else { - out[pos++] = c; - } - } - return pos; -} - - - } // namespace stringparsing } // unnamed namespace diff --git a/src/haswell.cpp b/src/haswell.cpp index 1eac5f91d..d369a11fa 100644 --- a/src/haswell.cpp +++ b/src/haswell.cpp @@ -162,10 +162,6 @@ simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t * return stage2(_doc); } -simdjson_warn_unused size_t implementation::write_string_escaped(const std::string_view input, char *out) const noexcept { - return haswell::stringparsing::write_string_escaped(input, out); -} - } // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson diff --git a/src/icelake.cpp b/src/icelake.cpp index 2febab916..c057d4278 100644 --- a/src/icelake.cpp +++ b/src/icelake.cpp @@ -208,10 +208,6 @@ simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t * return stage2(_doc); } -simdjson_warn_unused size_t implementation::write_string_escaped(const std::string_view input, char *out) const noexcept { - return icelake::stringparsing::write_string_escaped(input, out); -} - } // namespace icelake } // namespace simdjson diff --git a/src/implementation.cpp b/src/implementation.cpp index 32d1f43de..3dba4b605 100644 --- a/src/implementation.cpp +++ b/src/implementation.cpp @@ -186,9 +186,6 @@ public: simdjson_warn_unused bool validate_utf8(const char * buf, size_t len) const noexcept final override { return set_best()->validate_utf8(buf, len); } - simdjson_warn_unused size_t write_string_escaped(const std::string_view input, char *out) const noexcept final { - return set_best()->write_string_escaped(input, out); - } simdjson_inline detect_best_supported_implementation_on_first_use() noexcept : implementation("best_supported_detector", "Detects the best supported implementation and sets it", 0) {} private: const implementation *set_best() const noexcept; @@ -239,9 +236,6 @@ public: simdjson_warn_unused error_code minify(const uint8_t *, size_t, uint8_t *, size_t &) const noexcept final override { return UNSUPPORTED_ARCHITECTURE; } - simdjson_warn_unused size_t write_string_escaped(const std::string_view, char *) const noexcept final override { - return 0; // TODO: Evaluate whether this is the right thing to do for unsupported architecture. - } simdjson_warn_unused bool validate_utf8(const char *, size_t) const noexcept final override { return false; // Just refuse to validate. Given that we have a fallback implementation // it seems unlikely that unsupported_implementation will ever be used. If it is used, @@ -325,9 +319,6 @@ simdjson_warn_unused error_code minify(const char *buf, size_t len, char *dst, s simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) noexcept { return get_active_implementation()->validate_utf8(buf, len); } -simdjson_warn_unused size_t write_string_escaped(const std::string_view input, char *out) noexcept { - return get_active_implementation()->write_string_escaped(input, out); -} const implementation * builtin_implementation() { static const implementation * builtin_impl = get_available_implementations()[SIMDJSON_STRINGIFY(SIMDJSON_BUILTIN_IMPLEMENTATION)]; assert(builtin_impl); diff --git a/src/lasx.cpp b/src/lasx.cpp index b77d87694..09b945f59 100644 --- a/src/lasx.cpp +++ b/src/lasx.cpp @@ -125,10 +125,6 @@ simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t * return stage2(_doc); } -simdjson_warn_unused size_t implementation::write_string_escaped(const std::string_view input, char *out) const noexcept { - return lasx::stringparsing::write_string_escaped(input, out); -} - } // namespace lasx } // namespace simdjson diff --git a/src/lsx.cpp b/src/lsx.cpp index e283faf79..e68aada87 100644 --- a/src/lsx.cpp +++ b/src/lsx.cpp @@ -129,10 +129,6 @@ simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t * return stage2(_doc); } -simdjson_warn_unused size_t implementation::write_string_escaped(const std::string_view input, char *out) const noexcept { - return lsx::stringparsing::write_string_escaped(input, out); -} - } // namespace lsx } // namespace simdjson diff --git a/src/ppc64.cpp b/src/ppc64.cpp index a54f0f231..f7dee707e 100644 --- a/src/ppc64.cpp +++ b/src/ppc64.cpp @@ -135,10 +135,6 @@ simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t * return stage2(_doc); } -simdjson_warn_unused size_t implementation::write_string_escaped(const std::string_view input, char *out) const noexcept { - return ppc64::stringparsing::write_string_escaped(input, out); -} - } // namespace ppc64 } // namespace simdjson diff --git a/src/westmere.cpp b/src/westmere.cpp index 2acfaecac..7157ac491 100644 --- a/src/westmere.cpp +++ b/src/westmere.cpp @@ -166,11 +166,6 @@ simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t * if (error) { return error; } return stage2(_doc); } - -simdjson_warn_unused size_t implementation::write_string_escaped(const std::string_view input, char *out) const noexcept { - return westmere::stringparsing::write_string_escaped(input, out); -} - } // namespace westmere } // namespace simdjson From 0cd774097fcf2c60230dd78f556221113d4d0d0d Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Mon, 14 Jul 2025 23:46:30 -0400 Subject: [PATCH 02/11] [skip ci] doc fixes --- doc/basics.md | 85 ++++++++++++++++++++++++++------------------------ doc/builder.md | 18 +++++++++-- doc/dom.md | 9 +++++- 3 files changed, 67 insertions(+), 45 deletions(-) diff --git a/doc/basics.md b/doc/basics.md index 52cab16ba..739d80378 100644 --- a/doc/basics.md +++ b/doc/basics.md @@ -1,47 +1,50 @@ The Basics ========== -An overview of what you need to know to use simdjson, with examples. +An overview of what you need to know to use simdjson to parse JSON documents, with examples. +[Our documentation regarding the generation (serialization) of JSON documents is in a +separate document](https://github.com/simdjson/simdjson/blob/master/doc/builder.md). - [The Basics](#the-basics) - - [Requirements](#requirements) - - [Including simdjson](#including-simdjson) - - [Using simdjson with package managers](#using-simdjson-with-package-managers) - - [Using simdjson as a CMake dependency](#using-simdjson-as-a-cmake-dependency) - - [Versions](#versions) - - [The basics: loading and parsing JSON documents](#the-basics-loading-and-parsing-json-documents) - - [Documents are iterators](#documents-are-iterators) - - [Parser, document and JSON scope](#parser-document-and-json-scope) - - [string_view](#string_view) - - [Avoiding pitfalls: enable development checks](#avoiding-pitfalls-enable-development-checks) - - [Using the parsed JSON](#using-the-parsed-json) - - [Using the parsed JSON: additional examples](#using-the-parsed-json-additional-examples) - - [Adding support for custom types](#adding-support-for-custom-types) - - [1. Specialize `simdjson::ondemand::value::get` to get custom types (pre-C++20)](#1-specialize-simdjsonondemandvalueget-to-get-custom-types-pre-c20) - - [2. Use `tag_invoke` for custom types (C++20)](#2-use-tag_invoke-for-custom-types-c20) - - [Minifying JSON strings without parsing](#minifying-json-strings-without-parsing) - - [UTF-8 validation (alone)](#utf-8-validation-alone) - - [JSON Pointer](#json-pointer) - - [JSONPath](#jsonpath) - - [Error handling](#error-handling) - - [Error handling examples without exceptions](#error-handling-examples-without-exceptions) - - [Disabling exceptions](#disabling-exceptions) - - [Exceptions](#exceptions) - - [Current location in document](#current-location-in-document) - - [Checking for trailing content](#checking-for-trailing-content) - - [Rewinding](#rewinding) - - [Newline-Delimited JSON (ndjson) and JSON lines](#newline-delimited-json-ndjson-and-json-lines) - - [Parsing numbers inside strings](#parsing-numbers-inside-strings) - - [Dynamic Number Types](#dynamic-number-types) - - [Raw strings from keys](#raw-strings-from-keys) - - [General direct access to the raw JSON string](#general-direct-access-to-the-raw-json-string) - - [Storing directly into an existing string instance](#storing-directly-into-an-existing-string-instance) - - [Thread safety](#thread-safety) - - [Standard compliance](#standard-compliance) - - [Backwards compatibility](#backwards-compatibility) - - [Examples](#examples) - - [Performance tips](#performance-tips) - - [Further reading](#further-reading) + * [Requirements](#requirements) + * [Including simdjson](#including-simdjson) + * [Using simdjson with package managers](#using-simdjson-with-package-managers) + * [Using simdjson as a CMake dependency](#using-simdjson-as-a-cmake-dependency) + * [Versions](#versions) + * [The basics: loading and parsing JSON documents](#the-basics--loading-and-parsing-json-documents) + * [Documents are iterators](#documents-are-iterators) + + [Parser, document and JSON scope](#parser--document-and-json-scope) + * [string_view](#string-view) + * [Avoiding pitfalls: enable development checks](#avoiding-pitfalls--enable-development-checks) + * [Using the parsed JSON](#using-the-parsed-json) + + [Using the parsed JSON: additional examples](#using-the-parsed-json--additional-examples) + * [Adding support for custom types](#adding-support-for-custom-types) + + [1. Specialize `simdjson::ondemand::value::get` to get custom types (pre-C++20)](#1-specialize--simdjson--ondemand--value--get--to-get-custom-types--pre-c--20-) + + [2. Use `tag_invoke` for custom types (C++20)](#2-use--tag-invoke--for-custom-types--c--20-) + + [3. Using static reflection (C++26)](#3-using-static-reflection--c--26-) + * [Minifying JSON strings without parsing](#minifying-json-strings-without-parsing) + * [UTF-8 validation (alone)](#utf-8-validation--alone-) + * [JSON Pointer](#json-pointer) + * [JSONPath](#jsonpath) + * [Error handling](#error-handling) + + [Error handling examples without exceptions](#error-handling-examples-without-exceptions) + + [Disabling exceptions](#disabling-exceptions) + + [Exceptions](#exceptions) + + [Current location in document](#current-location-in-document) + + [Checking for trailing content](#checking-for-trailing-content) + * [Rewinding](#rewinding) + * [Newline-Delimited JSON (ndjson) and JSON lines](#newline-delimited-json--ndjson--and-json-lines) + * [Parsing numbers inside strings](#parsing-numbers-inside-strings) + * [Dynamic Number Types](#dynamic-number-types) + * [Raw strings from keys](#raw-strings-from-keys) + * [General direct access to the raw JSON string](#general-direct-access-to-the-raw-json-string) + * [Storing directly into an existing string instance](#storing-directly-into-an-existing-string-instance) + * [Thread safety](#thread-safety) + * [Standard compliance](#standard-compliance) + * [Backwards compatibility](#backwards-compatibility) + * [Examples](#examples) + * [Performance tips](#performance-tips) + * [Further reading](#further-reading) Requirements @@ -826,7 +829,7 @@ There are 3 main ways provided by simdjson to deserialize a value into a custom 1. Specialize `simdjson::ondemand::document::get` for the whole document 2. Specialize `simdjson::ondemand::value::get` for each value 2. Using `tag_invoke` *(the recommended way if your system supports C++20 or better)* -3. Using static reflectioin (requires C++26 or better) +3. Using static reflection (requires C++26 or better) We describe all of them in the following sections. Most users who have systems compatible with C++20 or better should skip ahead to [using `tag_invoke` for custom types (C++20)](#2-use-tag_invoke-for-custom-types-c20) as it is more powerful and simpler. @@ -1140,7 +1143,7 @@ struct Car { }; ``` -Observe how we defined the class to use types that simdjson does not directly support (`float`, `int`). +Observe how we define the class to use types that simdjson does not directly support (`float`, `int`). With C++20 support, the library grabs from the JSON the generic type (`double`, `int`) and then it casts it automatically. diff --git a/doc/builder.md b/doc/builder.md index 739cb9cc0..f9cdc2869 100644 --- a/doc/builder.md +++ b/doc/builder.md @@ -4,8 +4,14 @@ Builder Sometimes you want to generate JSON string outputs efficiently. The simdjson library provides high-performance low-level facilities. When using these low-level functionalities, you are responsible to -define the structure of your JSON document. However, string escaping -and UTF-8 validation is automated. +define the structure of your JSON document. Our more advanced interface +automates the process using C++26 static reflection: you get both high +speed and high convenience. + +- [Builder](#builder) + * [Overview: string_builder](#overview--string-builder) + * [Example: string_builder](#example--string-builder) + * [C++26 static reflection](#c--26-static-reflection) Overview: string_builder --------------------------- @@ -13,7 +19,11 @@ Overview: string_builder The string_builder class is a low-level utility for constructing JSON strings representing documents. It is optimized for performance, potentially leveraging kernel-specific features like SIMD instructions for tasks such as string escaping. This class supports atomic types (e.g., booleans, numbers, strings) but does not handle composed types directly (like arrays or objects). An `string_builder` is created with an initial buffer capacity (e.g., 1kB). The memory -is reallocated when needed. It has the following methods to add content to the string: +is reallocated when needed. +The efficiency of `string_builder` stems from its internal use of a resizable array or buffer. When you append data, it adds the characters to this buffer, resizing it only when necessary, typically in a way that minimizes reallocations. This approach contrasts with regular string concatenation, where each operation creates a new string, copying all previous content, leading to quadratic time complexity for repeated concatenations. + + +It has the following methods to add content to the string: - `append(number_type v)`: Appends a number (including booleans) to the JSON buffer. Booleans are converted to the strings "false" or "true". Numbers are formatted according to the JSON standard, with floating-point numbers using the shortest representation that accurately reflects the value. @@ -93,6 +103,8 @@ Example: string_builder C++26 static reflection ------------------------ +Static reflection (or compile-time reflection) in C++26 introduces a powerful compile-time mechanism that allows a program to inspect and manipulate its own structure, such as types, variables, functions, and other program elements, during compilation. Unlike runtime reflection in languages like Java or Python, C++26’s static reflection operates entirely at compile time, aligning with C++’s emphasis on zero-overhead abstractions and high performance. It means +that you can delegate much of the work to the library. If you have a compiler with support C++26 static reflection, you can compile your code with the `SIMDJSON_STATIC_REFLECTION` macro set: diff --git a/doc/dom.md b/doc/dom.md index 54ad79519..2c25fa404 100644 --- a/doc/dom.md +++ b/doc/dom.md @@ -1,7 +1,10 @@ The Document-Object-Model (DOM) front-end ========== -An overview of what you need to know to use simdjson, with examples. +An overview of what you need to know to use simdjson to parse JSON documents with +our DOM API, with examples. [Our documentation regarding the generation (serialization) of JSON documents is in a +separate document](https://github.com/simdjson/simdjson/blob/master/doc/builder.md). + * [DOM vs On-Demand](#dom-vs-on-demand) * [The Basics: Loading and Parsing JSON Documents](#the-basics-loading-and-parsing-json-documents-using-the-dom-front-end) @@ -28,6 +31,10 @@ a conventional Document-Object-Model (DOM) front-end. In such a scenario, the JS entirely parsed, validated and materialized in memory as the first step. The programmer may then access the parsed data using this in-memory model. +On-Demand is a different model where you parse just what you need, directly into your own +data structure. The On-Demand approach, when well tuned, can provide superior performance. +[We refer you to the On-Demand documentation for further details](https://github.com/simdjson/simdjson/blob/master/doc/basics.md). + The Basics: Loading and Parsing JSON Documents using the DOM front-end ---------------------------------------------- From 1078eb4034a0fd77687ec9b3f8e5c8db2cb5d979 Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Wed, 16 Jul 2025 12:05:09 -0400 Subject: [PATCH 03/11] minor update to the release candidate (#2394) --- doc/basics.md | 2 +- doc/builder.md | 140 +++++++++++----- include/simdjson/common_defs.h | 12 ++ include/simdjson/dom/serialization.h | 3 + .../simdjson/generic/ondemand/document-inl.h | 4 +- include/simdjson/generic/ondemand/document.h | 4 +- .../simdjson/generic/ondemand/json_builder.h | 12 +- .../ondemand/json_string_builder-inl.h | 2 +- .../generic/ondemand/json_string_builder.h | 4 +- include/simdjson/padded_string-inl.h | 4 +- .../builder/builder_string_builder_tests.cpp | 13 ++ .../static_reflection_builder_tests.cpp | 152 +++++++++++------- 12 files changed, 239 insertions(+), 113 deletions(-) diff --git a/doc/basics.md b/doc/basics.md index 739d80378..8376bf183 100644 --- a/doc/basics.md +++ b/doc/basics.md @@ -1362,7 +1362,7 @@ Then you can deserialize a type such as `Car` automatically: std::string json = R"( { "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9 ] } )"; simdjson::ondemand::parser parser; -simdjson::ondemand::document doc = parser.iterate(simdjson::pad(json)).get(doc); +simdjson::ondemand::document doc = parser.iterate(simdjson::pad(json)); Car c = doc.get(); ``` diff --git a/doc/builder.md b/doc/builder.md index f9cdc2869..210ee236c 100644 --- a/doc/builder.md +++ b/doc/builder.md @@ -12,11 +12,14 @@ speed and high convenience. * [Overview: string_builder](#overview--string-builder) * [Example: string_builder](#example--string-builder) * [C++26 static reflection](#c--26-static-reflection) + + [Without `string_buffer` instance](#without--string-buffer--instance) + + [Without `string_buffer` instance but with explicit error handling](#without--string-buffer--instance-but-with-explicit-error-handling) Overview: string_builder --------------------------- The string_builder class is a low-level utility for constructing JSON strings representing documents. It is optimized for performance, potentially leveraging kernel-specific features like SIMD instructions for tasks such as string escaping. This class supports atomic types (e.g., booleans, numbers, strings) but does not handle composed types directly (like arrays or objects). +Note that JSON strings are always encoded as UTF-8. An `string_builder` is created with an initial buffer capacity (e.g., 1kB). The memory is reallocated when needed. @@ -42,6 +45,9 @@ After writting the content, if you have reasons to believe that the content migh - `validate_unicode()`: Checks if the content in the JSON buffer is valid UTF-8. Returns: true if the content is valid UTF-8, false otherwise. +You might need to do unicode validation if you have strings in your data structures containing +malformed UTF-8. + Once you are satisfied, you can recover the string as follows: - `operator std::string()`: Converts the JSON buffer to an std::string. (Might throw if an error occurred.) @@ -54,52 +60,70 @@ Example: string_builder --------------------------- ```C++ +struct Car { + std::string make; + std::string model; + int64_t year; + std::vector tire_pressure; +}; - void serialize_car(const Car& car, simdjson::builder::string_builder& builder) { - // start of JSON - builder.start_object(); +void serialize_car(const Car& car, simdjson::builder::string_builder& builder) { + // start of JSON + builder.start_object(); - // "make" - builder.append_key_value("make", car.make); - builder.append_comma(); + // "make" + builder.append_key_value("make", car.make); + builder.append_comma(); - // "model" - builder.append_key_value("model", car.model); - builder.append_comma(); + // "model" + builder.append_key_value("model", car.model); + builder.append_comma(); - // "year" - builder.append_key_value("year", car.year); - builder.append_comma(); + // "year" + builder.append_key_value("year", car.year); + builder.append_comma(); - // "tire_pressure" - builder.escape_and_append_with_quotes("tire_pressure"); - builder.append_colon(); - builder.start_array(); - // vector tire_pressure - for (size_t i = 0; i < car.tire_pressure.size(); ++i) { - builder.append(car.tire_pressure[i]); - if (i < car.tire_pressure.size() - 1) { - builder.append_comma(); - } + // "tire_pressure" + builder.escape_and_append_with_quotes("tire_pressure"); + builder.append_colon(); + builder.start_array(); + // vector tire_pressure + for (size_t i = 0; i < car.tire_pressure.size(); ++i) { + builder.append(car.tire_pressure[i]); + if (i < car.tire_pressure.size() - 1) { + builder.append_comma(); } - builder.end_array(); - builder.end_object(); } + builder.end_array(); + builder.end_object(); +} - bool car_test() { - simdjson::builder::string_builder sb; - Car c = {"Toyota", "Corolla", 2017, {30.0,30.2,30.513,30.79}}; - serialize_car(c, sb); - std::string_view p; - if(sb.view().get(p)) { - return false; // there was an error - } - // p holds the JSON: - // "{\"make\":\"Toyota\",\"model\":\"Corolla\",\"year\":2017,\"tire_pressure\":[30.0,30.2,30.513,30.79]}" - return true; +bool car_test() { + simdjson::builder::string_builder sb; + Car c = {"Toyota", "Corolla", 2017, {30.0,30.2,30.513,30.79}}; + serialize_car(c, sb); + std::string_view p{sb}; + // p holds the JSON: + // "{\"make\":\"Toyota\",\"model\":\"Corolla\",\"year\":2017,\"tire_pressure\":[30.0,30.2,30.513,30.79]}" + return true; +} +``` + +The `string_builder` constructor takes an optional parameter which specifies the initial +memory allocation in byte. If you know approximately the size of your JSON output, you can +pass this value as a parameter (e.g., `simdjson::builder::string_builder sb{1233213}`). + +The `string_builder` might throw an exception in case of error when you cast it result to `std::string_view`. If you wish to avoid exceptions, you can use the following programming pattern: + +```cpp + std::string_view p; + if(sb.view().get(p)) { + return false; // there was an error } ``` +In all cases, the `std::string_view` instance depends the corresponding `string_builder` instance. + C++26 static reflection ------------------------ @@ -118,21 +142,55 @@ And then you can append your data structures to a `string_builder` instance automatically. In most cases, it should work automatically: ```cpp + struct Car { + std::string make; + std::string model; + int64_t year; + std::vector tire_pressure; + }; + bool car_test() { simdjson::builder::string_builder sb; Car c = {"Toyota", "Corolla", 2017, {30.0,30.2,30.513,30.79}}; - append(sb, c); - std::string_view p; - if(sb.view().get(p)) { - return false; // there was an error - } + sb << c; + std::string_view p{sb}; // p holds the JSON: // "{\"make\":\"Toyota\",\"model\":\"Corolla\",\"year\":2017,\"tire_pressure\":[30.0,30.2,30.513,30.79]}" return true; } ``` -If you prefer, you can also create a string directly: + +### Without `string_buffer` instance + +In some instances, you might want to create a string directly from your own data type. +You can create a string directly, without an explicit `string_builder` instance +with the `simdjson::builder::to_json_string` function. +(Under the hood a `string_builder` instance may still be created.) + +```cpp + struct Car { + std::string make; + std::string model; + int64_t year; + std::vector tire_pressure; + }; + + void f() { + Car c = {"Toyota", "Corolla", 2017, {30.0,30.2,30.513,30.79}}; + std::string json = simdjson::builder::to_json_string(c); + } +``` + +If you know the output size, in bytes, of your JSON string, you may +pass it as a second parameter (e.g., `simdjson::builder::to_json_string(c, 31123)`). + + + +### Without `string_buffer` instance but with explicit error handling + +If prefer a version without exceptions and explicit error handling, you can use the following +pattern: ```cpp std::string json; diff --git a/include/simdjson/common_defs.h b/include/simdjson/common_defs.h index ce3c48616..a151795a6 100644 --- a/include/simdjson/common_defs.h +++ b/include/simdjson/common_defs.h @@ -356,4 +356,16 @@ namespace std { #define SIMDJSON_AVX512_ALLOWED 1 #endif + +#ifndef __has_cpp_attribute +#define simdjson_lifetime_bound +#elif __has_cpp_attribute(msvc::lifetimebound) +#define simdjson_lifetime_bound [[msvc::lifetimebound]] +#elif __has_cpp_attribute(clang::lifetimebound) +#define simdjson_lifetime_bound [[clang::lifetimebound]] +#elif __has_cpp_attribute(lifetimebound) +#define simdjson_lifetime_bound [[lifetimebound]] +#else +#define simdjson_lifetime_bound +#endif #endif // SIMDJSON_COMMON_DEFS_H diff --git a/include/simdjson/dom/serialization.h b/include/simdjson/dom/serialization.h index 87c735bbb..fc92e05bb 100644 --- a/include/simdjson/dom/serialization.h +++ b/include/simdjson/dom/serialization.h @@ -107,6 +107,9 @@ protected: * by a "formatter" which handles the details. Thus * the string_builder template could support both minification * and prettification, and various other tradeoffs. + * + * This is not to be confused with the simdjson::builder::string_builder + * which is a different class. */ template class string_builder { diff --git a/include/simdjson/generic/ondemand/document-inl.h b/include/simdjson/generic/ondemand/document-inl.h index ec889d35b..39a2f6f7b 100644 --- a/include/simdjson/generic/ondemand/document-inl.h +++ b/include/simdjson/generic/ondemand/document-inl.h @@ -195,8 +195,8 @@ simdjson_inline document::operator object() & noexcept(false) { return get_objec simdjson_inline document::operator uint64_t() noexcept(false) { return get_uint64(); } simdjson_inline document::operator int64_t() noexcept(false) { return get_int64(); } simdjson_inline document::operator double() noexcept(false) { return get_double(); } -simdjson_inline document::operator std::string_view() noexcept(false) { return get_string(false); } -simdjson_inline document::operator raw_json_string() noexcept(false) { return get_raw_json_string(); } +simdjson_inline document::operator std::string_view() noexcept(false) simdjson_lifetime_bound { return get_string(false); } +simdjson_inline document::operator raw_json_string() noexcept(false) simdjson_lifetime_bound { return get_raw_json_string(); } simdjson_inline document::operator bool() noexcept(false) { return get_bool(); } simdjson_inline document::operator value() noexcept(false) { return get_value(); } diff --git a/include/simdjson/generic/ondemand/document.h b/include/simdjson/generic/ondemand/document.h index ef174fad8..a642248ea 100644 --- a/include/simdjson/generic/ondemand/document.h +++ b/include/simdjson/generic/ondemand/document.h @@ -322,7 +322,7 @@ public: * time it parses a document or when it is destroyed. * @exception simdjson_error(INCORRECT_TYPE) if the JSON value is not a string. */ - simdjson_inline operator std::string_view() noexcept(false); + simdjson_inline operator std::string_view() noexcept(false) simdjson_lifetime_bound; /** * Cast this JSON value to a raw_json_string. * @@ -331,7 +331,7 @@ public: * @returns A pointer to the raw JSON for the given string. * @exception simdjson_error(INCORRECT_TYPE) if the JSON value is not a string. */ - simdjson_inline operator raw_json_string() noexcept(false); + simdjson_inline operator raw_json_string() noexcept(false) simdjson_lifetime_bound; /** * Cast this JSON value to a bool. * diff --git a/include/simdjson/generic/ondemand/json_builder.h b/include/simdjson/generic/ondemand/json_builder.h index 14fb24699..ac242eca9 100644 --- a/include/simdjson/generic/ondemand/json_builder.h +++ b/include/simdjson/generic/ondemand/json_builder.h @@ -145,8 +145,8 @@ void append(string_builder &b, const Z &z) { } template -simdjson_result to_json_string(const Z &z) { - string_builder b; +simdjson_result to_json_string(const Z &z, size_t initial_capacity = 1024) { + string_builder b(initial_capacity); append(b, z); std::string_view s; if(auto e = b.view().get(s); e) { return e; } @@ -162,7 +162,13 @@ simdjson_error to_json(const Z &z, std::string &s) { s.assign(view); return SUCCESS; } -} // namespace json_builder + +template +string_builder& operator<<(string_builder& b, const Z& z) { + append(b, z); + return b; +} +} // namespace builder } // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson #endif // SIMDJSON_STATIC_REFLECTION diff --git a/include/simdjson/generic/ondemand/json_string_builder-inl.h b/include/simdjson/generic/ondemand/json_string_builder-inl.h index 42e42f7da..94c59bbba 100644 --- a/include/simdjson/generic/ondemand/json_string_builder-inl.h +++ b/include/simdjson/generic/ondemand/json_string_builder-inl.h @@ -516,7 +516,7 @@ simdjson_inline string_builder::operator std::string() const noexcept(false) { } simdjson_inline string_builder::operator std::string_view() const - noexcept(false) { + noexcept(false) simdjson_lifetime_bound { return view(); } #endif diff --git a/include/simdjson/generic/ondemand/json_string_builder.h b/include/simdjson/generic/ondemand/json_string_builder.h index 2d1208abc..a685d8dce 100644 --- a/include/simdjson/generic/ondemand/json_string_builder.h +++ b/include/simdjson/generic/ondemand/json_string_builder.h @@ -19,7 +19,7 @@ namespace builder { * supports atomic types (Booleans, strings), it does not support composed * types (arrays and objects). * - * Ultimately, this class should support kernel-specific optimizations. E.g., + * Ultimately, this class can support kernel-specific optimizations. E.g., * it may make use of SIMD instructions to escape strings faster. */ class string_builder { @@ -146,7 +146,7 @@ public: * The result may not be valid UTF-8 if some of your content was not valid UTF-8. * Use validate_unicode() to check the content if needed. */ - simdjson_inline operator std::string_view() const noexcept(false); + simdjson_inline operator std::string_view() const noexcept(false) simdjson_lifetime_bound; #endif /** diff --git a/include/simdjson/padded_string-inl.h b/include/simdjson/padded_string-inl.h index 34cc76cb9..7c0440ff4 100644 --- a/include/simdjson/padded_string-inl.h +++ b/include/simdjson/padded_string-inl.h @@ -125,9 +125,9 @@ inline const char *padded_string::data() const noexcept { return data_ptr; } inline char *padded_string::data() noexcept { return data_ptr; } -inline padded_string::operator std::string_view() const { return std::string_view(data(), length()); } +inline padded_string::operator std::string_view() const simdjson_lifetime_bound { return std::string_view(data(), length()); } -inline padded_string::operator padded_string_view() const noexcept { +inline padded_string::operator padded_string_view() const noexcept simdjson_lifetime_bound { return padded_string_view(data(), length(), length() + SIMDJSON_PADDING); } diff --git a/tests/builder/builder_string_builder_tests.cpp b/tests/builder/builder_string_builder_tests.cpp index 72524fac0..bafc05cf5 100644 --- a/tests/builder/builder_string_builder_tests.cpp +++ b/tests/builder/builder_string_builder_tests.cpp @@ -270,6 +270,7 @@ namespace builder_tests { } + void serialize_car(const Car& car, simdjson::builder::string_builder& builder) { // start of JSON builder.start_object(); @@ -318,6 +319,17 @@ namespace builder_tests { TEST_SUCCEED(); } + #if SIMDJSON_EXCEPTIONS + bool car_test_exception() { + TEST_START(); + simdjson::builder::string_builder sb; + Car c = {"Toyota", "Corolla", 2017, {30.0,30.2,30.513,30.79}}; + serialize_car_long(c, sb); + std::string_view p{sb}; + ASSERT_EQUAL(p, "{\"make\":\"Toyota\",\"model\":\"Corolla\",\"year\":2017,\"tire_pressure\":[30.0,30.2,30.513,30.79]}"); + TEST_SUCCEED(); + } + #endif bool run() { return various_integers() && @@ -325,6 +337,7 @@ namespace builder_tests { car_test_long() && car_test() && #if SIMDJSON_EXCEPTIONS + car_test_exception() && string_convertion_except() && #endif append_char() && diff --git a/tests/builder/static_reflection_builder_tests.cpp b/tests/builder/static_reflection_builder_tests.cpp index d48086fd2..6f04d3018 100644 --- a/tests/builder/static_reflection_builder_tests.cpp +++ b/tests/builder/static_reflection_builder_tests.cpp @@ -72,69 +72,103 @@ namespace builder_tests { ASSERT_EQUAL(c2.tire_pressure[3], 30.79); TEST_SUCCEED(); } + #if SIMDJSON_EXCEPTIONS + bool car_test_exception() { + TEST_START(); + simdjson::builder::string_builder sb; + Car c = {"Toyota", "Corolla", 2017, {30.0,30.2,30.513,30.79}}; + append(sb, c); + std::string_view p{sb}; + TEST_SUCCEED(); + } + bool car_test_exception2() { + TEST_START(); + simdjson::builder::string_builder sb; + Car c = {"Toyota", "Corolla", 2017, {30.0,30.2,30.513,30.79}}; + sb << c; + std::string_view p{sb}; + TEST_SUCCEED(); + } + void car_test_to_json_exception() { + TEST_START(); + Car c = {"Toyota", "Corolla", 2017, {30.0,30.2,30.513,30.79}}; + std::string json = simdjson::builder::to_json_string(c); + TEST_SUCCEED(); + } + #endif // SIMDJSON_EXCEPTIONS -bool serialize_deserialize_kid() { - TEST_START(); - simdjson::padded_string json_str = - R"({"age": 12, "name": "John", "toys": ["car", "ball"]})"_padded; - simdjson::ondemand::parser parser; - simdjson::ondemand::document doc; - ASSERT_SUCCESS(parser.iterate(json_str).get(doc)); - kid k; - ASSERT_SUCCESS(doc.get().get(k)); - ASSERT_EQUAL(k.age, 12); - ASSERT_EQUAL(k.name, "John"); - ASSERT_EQUAL(k.toys.size(), 2); - ASSERT_EQUAL(k.toys[0], "car"); - ASSERT_EQUAL(k.toys[1], "ball"); - // Now, go the other direction: - std::string json; - ASSERT_SUCCESS(simdjson::builder::to_json_string(k).get(json)); - std::cout << json << std::endl; - // Now we parse it back: - simdjson::ondemand::parser parser2; - simdjson::ondemand::document doc2; - ASSERT_SUCCESS(parser2.iterate(simdjson::pad(json)).get(doc2)); - kid k2; - ASSERT_SUCCESS(doc2.get().get(k2)); - ASSERT_EQUAL(k2.age, 12); - ASSERT_EQUAL(k2.name, "John"); - ASSERT_EQUAL(k2.toys.size(), 2); - ASSERT_EQUAL(k2.toys[0], "car"); - ASSERT_EQUAL(k2.toys[1], "ball"); - TEST_SUCCEED(); -} + bool serialize_deserialize_kid() { + TEST_START(); + simdjson::padded_string json_str = + R"({"age": 12, "name": "John", "toys": ["car", "ball"]})"_padded; + simdjson::ondemand::parser parser; + simdjson::ondemand::document doc; + ASSERT_SUCCESS(parser.iterate(json_str).get(doc)); + kid k; + ASSERT_SUCCESS(doc.get().get(k)); + ASSERT_EQUAL(k.age, 12); + ASSERT_EQUAL(k.name, "John"); + ASSERT_EQUAL(k.toys.size(), 2); + ASSERT_EQUAL(k.toys[0], "car"); + ASSERT_EQUAL(k.toys[1], "ball"); + // Now, go the other direction: + std::string json; + ASSERT_SUCCESS(simdjson::builder::to_json_string(k).get(json)); + std::cout << json << std::endl; + // Now we parse it back: + simdjson::ondemand::parser parser2; + simdjson::ondemand::document doc2; + ASSERT_SUCCESS(parser2.iterate(simdjson::pad(json)).get(doc2)); + kid k2; + ASSERT_SUCCESS(doc2.get().get(k2)); + ASSERT_EQUAL(k2.age, 12); + ASSERT_EQUAL(k2.name, "John"); + ASSERT_EQUAL(k2.toys.size(), 2); + ASSERT_EQUAL(k2.toys[0], "car"); + ASSERT_EQUAL(k2.toys[1], "ball"); + TEST_SUCCEED(); + } -bool serialize_deserialize_x_y_z() { - TEST_START(); - X s1 = {.a = '1', - .b = 10, - .c = 0, - .d = "test string\n\r\"", - .e = {1, 2, 3}, - .f = {"ab", "cd", "fg"}, - .y = {.g = 100, - .h = "test string\n\r\"", - .i = {1, 2, 3}, - .z = {.x = 1000}}}; - std::string pstr; - ASSERT_SUCCESS(simdjson::builder::to_json_string(s1).get(pstr)); - ASSERT_EQUAL( - pstr, - R"({"a":"1","b":10,"c":0,"d":"test string\n\r\"","e":[1,2,3],"f":["ab","cd","fg"],"y":{"g":100,"h":"test string\n\r\"","i":[1,2,3],"z":{"x":1000}}})"); - simdjson::ondemand::parser parser; - simdjson::ondemand::document doc; - ASSERT_SUCCESS(parser.iterate(simdjson::pad(pstr)).get(doc)); - X s2; - ASSERT_SUCCESS(doc.get().get(s2)); - ASSERT_TRUE(s1 == s2); - TEST_SUCCEED(); -} + bool serialize_deserialize_x_y_z() { + TEST_START(); + X s1 = {.a = '1', + .b = 10, + .c = 0, + .d = "test string\n\r\"", + .e = {1, 2, 3}, + .f = {"ab", "cd", "fg"}, + .y = {.g = 100, + .h = "test string\n\r\"", + .i = {1, 2, 3}, + .z = {.x = 1000}}}; + std::string pstr; + ASSERT_SUCCESS(simdjson::builder::to_json_string(s1).get(pstr)); + ASSERT_EQUAL( + pstr, + R"({"a":"1","b":10,"c":0,"d":"test string\n\r\"","e":[1,2,3],"f":["ab","cd","fg"],"y":{"g":100,"h":"test string\n\r\"","i":[1,2,3],"z":{"x":1000}}})"); + simdjson::ondemand::parser parser; + simdjson::ondemand::document doc; + ASSERT_SUCCESS(parser.iterate(simdjson::pad(pstr)).get(doc)); + X s2; + ASSERT_SUCCESS(doc.get().get(s2)); + ASSERT_TRUE(s1 == s2); + TEST_SUCCEED(); + } -bool run() { - return car_test() && serialize_deserialize_kid() && serialize_deserialize_x_y_z() && true; -} + bool run() { + return + + #if SIMDJSON_EXCEPTIONS + car_test_exception() && + car_test_exception2() && + car_test_to_json_exception() && + #endif // SIMDJSON_EXCEPTIONS + car_test() && + serialize_deserialize_kid() && + serialize_deserialize_x_y_z() && + true; + } } // namespace builder_tests From 41ced28821a2e9d6f6c804e3142004fccd7c47d8 Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Wed, 16 Jul 2025 12:07:49 -0400 Subject: [PATCH 04/11] adding a space --- doc/basics.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/basics.md b/doc/basics.md index 739d80378..4aa70296c 100644 --- a/doc/basics.md +++ b/doc/basics.md @@ -1358,6 +1358,7 @@ your code with the `SIMDJSON_STATIC_REFLECTION` macro set: ``` Then you can deserialize a type such as `Car` automatically: + ```cpp std::string json = R"( { "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9 ] } )"; From e6240f18c41da32ebdc71a7223b862ad3f875e21 Mon Sep 17 00:00:00 2001 From: evbse <133204505+evbse@users.noreply.github.com> Date: Wed, 23 Jul 2025 10:20:06 +0300 Subject: [PATCH 05/11] Add version to amalgamated files (#2400) --- singleheader/amalgamate.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/singleheader/amalgamate.py b/singleheader/amalgamate.py index d6590f28a..6b556a783 100755 --- a/singleheader/amalgamate.py +++ b/singleheader/amalgamate.py @@ -296,10 +296,10 @@ class SimdjsonRepository: class Amalgamator: @classmethod - def amalgamate(cls, output_path: str, filename: str, roots: List[RelativeRoot], timestamp: str): + def amalgamate(cls, output_path: str, filename: str, roots: List[RelativeRoot], timestamp: str, version: str): print(f"Creating {output_path}") fid = open(output_path, 'w') - print(f"/* auto-generated on {timestamp}. Do not edit! */", file=fid) + print(f"/* auto-generated on {timestamp}. version {version} Do not edit! */", file=fid) amalgamator = cls(fid, SimdjsonRepository(PROJECTPATH, roots)) file = amalgamator.repository[filename] assert file, f"{filename} not found in {[os.path.join(PROJECTPATH, root) for root in roots]}!" @@ -480,8 +480,13 @@ AMAL_C = os.path.join(AMALGAMATE_OUTPUT_PATH, "simdjson.cpp") DEMOCPP = os.path.join(AMALGAMATE_OUTPUT_PATH, "amalgamate_demo.cpp") README = os.path.join(AMALGAMATE_OUTPUT_PATH, "README.md") -Amalgamator.amalgamate(AMAL_H, "simdjson.h", ['include'], timestamp).validate_all_files_used('include') -Amalgamator.amalgamate(AMAL_C, "simdjson.cpp", ['src', 'include'], timestamp).validate_all_files_used('src') +def read_version(): + with open(os.path.join(PROJECTPATH, 'include/simdjson/simdjson_version.h')) as f: + return re.search(r'\d+\.\d+\.\d+', f.read()).group(0) + +version = read_version() +Amalgamator.amalgamate(AMAL_H, "simdjson.h", ['include'], timestamp, version).validate_all_files_used('include') +Amalgamator.amalgamate(AMAL_C, "simdjson.cpp", ['src', 'include'], timestamp, version).validate_all_files_used('src') # copy the README and DEMOCPP if SCRIPTPATH != AMALGAMATE_OUTPUT_PATH: From 90e4a66c93f7d0d685b2e13c42c49f4992f6f827 Mon Sep 17 00:00:00 2001 From: Francisco Geiman Thiesen Date: Wed, 23 Jul 2025 00:24:05 -0700 Subject: [PATCH 06/11] Bringing a bit more use-cases for reflection based serializations (optional). Also adding string-based enum handling as requested on X. (#2395) * Adding type validation, enhancing optional type support and adding test a few more tests. * Adding support for string-based enum serlalization and deserialization. * Removing unintentional endline. * Removing trailing whitespace. * Adding simpler api as suggested by moisrex. * Removing explicit optiona and optional references and using concepts instead! Credit goes to Lemire for pointing this out and suggesting a concepts based approach here. * Removing tests that are not relevant for this branch. * Removing api related changes. That will be done by moisrex. * Removing unnecessary new endlines. * Removing tests related to api changes and cleaning-up irrelevant tests. * removing broken reference * Removing trailing whitespace --- include/simdjson/concepts.h | 1 - .../simdjson/generic/ondemand/deserialize.h | 2 +- .../simdjson/generic/ondemand/json_builder.h | 130 ++++++++- .../generic/ondemand/serialization-inl.h | 3 + .../simdjson/generic/ondemand/serialization.h | 14 + .../generic/ondemand/std_deserialize.h | 145 +++++++++- include/simdjson/generic/ondemand/value.h | 13 + include/simdjson/ondemand.h | 22 ++ tests/builder/CMakeLists.txt | 3 + .../static_reflection_builder_tests.cpp | 18 +- .../static_reflection_comprehensive_tests.cpp | 231 ++++++++++++++++ .../static_reflection_edge_cases_tests.cpp | 230 ++++++++++++++++ .../builder/static_reflection_enum_tests.cpp | 258 ++++++++++++++++++ 13 files changed, 1045 insertions(+), 25 deletions(-) create mode 100644 tests/builder/static_reflection_comprehensive_tests.cpp create mode 100644 tests/builder/static_reflection_edge_cases_tests.cpp create mode 100644 tests/builder/static_reflection_enum_tests.cpp diff --git a/include/simdjson/concepts.h b/include/simdjson/concepts.h index ffe344c07..371ad223f 100644 --- a/include/simdjson/concepts.h +++ b/include/simdjson/concepts.h @@ -115,7 +115,6 @@ concept optional_type = requires(std::remove_cvref_t obj) { { obj.value() } -> std::same_as::value_type&>; requires requires(typename std::remove_cvref_t::value_type &&val) { obj.emplace(std::move(val)); - obj = std::move(val); { obj.value_or(val) } -> std::convertible_to::value_type>; diff --git a/include/simdjson/generic/ondemand/deserialize.h b/include/simdjson/generic/ondemand/deserialize.h index 8bad5699b..1e310fba6 100644 --- a/include/simdjson/generic/ondemand/deserialize.h +++ b/include/simdjson/generic/ondemand/deserialize.h @@ -77,7 +77,7 @@ template concept custom_deserializable = tag_invocable; template -concept deserializable = custom_deserializable || is_builtin_deserializable_v; +concept deserializable = custom_deserializable || is_builtin_deserializable_v || concepts::optional_type; template concept nothrow_custom_deserializable = nothrow_tag_invocable; diff --git a/include/simdjson/generic/ondemand/json_builder.h b/include/simdjson/generic/ondemand/json_builder.h index 14fb24699..e6a4daf33 100644 --- a/include/simdjson/generic/ondemand/json_builder.h +++ b/include/simdjson/generic/ondemand/json_builder.h @@ -14,6 +14,8 @@ #include #include #include +#include +#include #include #include #include @@ -94,8 +96,13 @@ consteval std::string consteval_to_quoted_escaped(std::string_view input); template requires(std::is_class_v && !container_but_not_string && !concepts::string_view_keyed_map && + !concepts::optional_type && + !concepts::smart_pointer && + !concepts::appendable_containers && !std::is_same_v && - !std::is_same_v) + !std::is_same_v && + !std::is_same_v && + !std::is_same_v) constexpr void atom(string_builder &b, const T &t) { int i = 0; b.append('{'); @@ -111,8 +118,127 @@ constexpr void atom(string_builder &b, const T &t) { b.append('}'); } +// Support for optional types (std::optional, etc.) +template +constexpr void atom(string_builder &b, const T &opt) { + if (opt) { + atom(b, opt.value()); + } else { + b.append_raw("null"); + } +} + +// Support for smart pointers (std::unique_ptr, std::shared_ptr, etc.) +template +constexpr void atom(string_builder &b, const T &ptr) { + if (ptr) { + atom(b, *ptr); + } else { + b.append_raw("null"); + } +} + +// Support for enums - serialize as string representation using expand approach from P2996R12 +template + requires(std::is_enum_v) +void atom(string_builder &b, const T &e) { +#if SIMDJSON_STATIC_REFLECTION + std::string_view result = ""; + [:expand(std::meta::enumerators_of(^^T)):] >> [&]{ + if (e == [:enum_val:]) { + result = std::meta::identifier_of(enum_val); + } + }; + + if (result != "") { + b.append_raw("\""); + b.append_raw(result); + b.append_raw("\""); + } else { + // Fallback to integer if enum value not found + atom(b, static_cast>(e)); + } +#else + // Fallback: serialize as integer if reflection not available + atom(b, static_cast>(e)); +#endif +} + +// Support for appendable containers that don't have operator[] (sets, etc.) +template + requires(!container_but_not_string && !concepts::string_view_keyed_map && + !concepts::optional_type && !concepts::smart_pointer && + !std::is_same_v && + !std::is_same_v && !std::is_same_v) +constexpr void atom(string_builder &b, const T &container) { + if (container.empty()) { + b.append_raw("[]"); + return; + } + b.append('['); + bool first = true; + for (const auto& item : container) { + if (!first) { + b.append(','); + } + first = false; + atom(b, item); + } + b.append(']'); +} + +// append functions that delegate to atom functions for primitive types +template + requires(std::is_arithmetic_v && !std::is_same_v) +void append(string_builder &b, const T &t) { + atom(b, t); +} + +template + requires(std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v) +void append(string_builder &b, const T &t) { + atom(b, t); +} + +template +void append(string_builder &b, const T &t) { + atom(b, t); +} + +template +void append(string_builder &b, const T &t) { + atom(b, t); +} + +template + requires(!container_but_not_string && !concepts::string_view_keyed_map && + !concepts::optional_type && !concepts::smart_pointer && + !std::is_same_v && + !std::is_same_v && !std::is_same_v) +void append(string_builder &b, const T &t) { + atom(b, t); +} + +template +void append(string_builder &b, const T &t) { + atom(b, t); +} + // works for struct -template void append(string_builder &b, const Z &z) { +template + requires(std::is_class_v && !container_but_not_string && + !concepts::string_view_keyed_map && + !concepts::optional_type && + !concepts::smart_pointer && + !concepts::appendable_containers && + !std::is_same_v && + !std::is_same_v && + !std::is_same_v && + !std::is_same_v) +void append(string_builder &b, const Z &z) { int i = 0; b.append('{'); [:expand(std::meta::nonstatic_data_members_of(^^Z, std::meta::access_context::unchecked())):] >> [&]() { diff --git a/include/simdjson/generic/ondemand/serialization-inl.h b/include/simdjson/generic/ondemand/serialization-inl.h index 77be39a11..37a501789 100644 --- a/include/simdjson/generic/ondemand/serialization-inl.h +++ b/include/simdjson/generic/ondemand/serialization-inl.h @@ -9,6 +9,9 @@ #include "simdjson/generic/ondemand/object.h" #include "simdjson/generic/ondemand/serialization.h" #include "simdjson/generic/ondemand/value.h" +#if SIMDJSON_STATIC_REFLECTION +#include "simdjson/generic/ondemand/json_builder.h" +#endif #endif // SIMDJSON_CONDITIONAL_INCLUDE namespace simdjson { diff --git a/include/simdjson/generic/ondemand/serialization.h b/include/simdjson/generic/ondemand/serialization.h index 048c73cda..3d2c8ed85 100644 --- a/include/simdjson/generic/ondemand/serialization.h +++ b/include/simdjson/generic/ondemand/serialization.h @@ -35,6 +35,20 @@ inline simdjson_result to_json_string(simdjson_result to_json_string(simdjson_result x); inline simdjson_result to_json_string(simdjson_result x); inline simdjson_result to_json_string(simdjson_result x); + +#if SIMDJSON_STATIC_REFLECTION +/** + * Create a JSON string from any user-defined type using static reflection. + * Only available when SIMDJSON_STATIC_REFLECTION is enabled. + */ +template + requires(!std::same_as && + !std::same_as && + !std::same_as && + !std::same_as) +inline std::string to_json_string(const T& obj); +#endif + } // namespace simdjson /** diff --git a/include/simdjson/generic/ondemand/std_deserialize.h b/include/simdjson/generic/ondemand/std_deserialize.h index 0e945bc82..7c8f518af 100644 --- a/include/simdjson/generic/ondemand/std_deserialize.h +++ b/include/simdjson/generic/ondemand/std_deserialize.h @@ -248,17 +248,16 @@ error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept(nothrow_deser /** * This CPO (Customization Point Object) will help deserialize into optional types. */ -template +template requires(!require_custom_serialization) -error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept(nothrow_deserializable::value_type, ValT>) { +error_code tag_invoke(deserialize_tag, auto &val, T &out) noexcept(nothrow_deserializable::value_type, decltype(val)>) { using value_type = typename std::remove_cvref_t::value_type; - static_assert( - deserializable, - "The specified type inside the unique_ptr must itself be deserializable"); - static_assert( - std::is_default_constructible_v, - "The specified type inside the unique_ptr must default constructible."); + // Check if the value is null + if (val.is_null()) { + out.reset(); // Set to nullopt + return SUCCESS; + } if (!out) { out.emplace(); @@ -297,7 +296,7 @@ template consteval auto expand(R range) { std::vector args; for (auto r : range) { - args.push_back(reflect_value(r)); + args.push_back(reflect_constant(r)); } return substitute(^^__impl::replicator, args); } @@ -317,17 +316,42 @@ error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept { [:expand(std::meta::nonstatic_data_members_of(^^T, std::meta::access_context::unchecked())):] >> [&]() { if constexpr (!std::meta::is_const(mem) && std::meta::is_public(mem)) { constexpr std::string_view key = std::define_static_string(std::meta::identifier_of(mem)); - static_assert( - deserializable, - "The specified type inside the class must itself be deserializable"); + // Note: removed static assert as optional types are now handled generically // as long we are succesful or the field is not found, we continue if(e == simdjson::SUCCESS || e == simdjson::NO_SUCH_FIELD) { - obj[key].get(out.[:mem:]); + e = obj[key].get(out.[:mem:]); } } }; return e; } + +// Support for enum deserialization - deserialize from string representation using expand approach from P2996R12 +template + requires(std::is_enum_v) +error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept { +#if SIMDJSON_STATIC_REFLECTION + std::string_view str; + SIMDJSON_TRY(val.get_string().get(str)); + + bool found = false; + [:expand(std::meta::enumerators_of(^^T)):] >> [&]{ + if (!found && str == std::meta::identifier_of(enum_val)) { + out = [:enum_val:]; + found = true; + } + }; + + return found ? SUCCESS : INCORRECT_TYPE; +#else + // Fallback: deserialize as integer if reflection not available + std::underlying_type_t int_val; + SIMDJSON_TRY(val.get(int_val)); + out = static_cast(int_val); + return SUCCESS; +#endif +} + template requires(user_defined_type>) error_code tag_invoke(deserialize_tag, simdjson_value &val, std::unique_ptr &out) noexcept { @@ -366,6 +390,10 @@ error_code tag_invoke(deserialize_tag, simdjson_value &val, std::shared_ptr & // Unique pointers //////////////////////////////////////// error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) noexcept { + if (val.is_null()) { + out.reset(); + return SUCCESS; + } if (!out) { out = std::make_unique(); if (!out) { return MEMALLOC; } @@ -375,6 +403,10 @@ error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) no } error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) noexcept { + if (val.is_null()) { + out.reset(); + return SUCCESS; + } if (!out) { out = std::make_unique(); if (!out) { return MEMALLOC; } @@ -384,6 +416,10 @@ error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) } error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) noexcept { + if (val.is_null()) { + out.reset(); + return SUCCESS; + } if (!out) { out = std::make_unique(); if (!out) { return MEMALLOC; } @@ -393,6 +429,10 @@ error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out } error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) noexcept { + if (val.is_null()) { + out.reset(); + return SUCCESS; + } if (!out) { out = std::make_unique(); if (!out) { return MEMALLOC; } @@ -402,6 +442,10 @@ error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) } error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) noexcept { + if (val.is_null()) { + out.reset(); + return SUCCESS; + } if (!out) { out = std::make_unique(); if (!out) { return MEMALLOC; } @@ -415,6 +459,10 @@ error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) noexcept { + if (val.is_null()) { + out.reset(); + return SUCCESS; + } if (!out) { out = std::make_shared(); if (!out) { return MEMALLOC; } @@ -424,6 +472,10 @@ error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) no } error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) noexcept { + if (val.is_null()) { + out.reset(); + return SUCCESS; + } if (!out) { out = std::make_shared(); if (!out) { return MEMALLOC; } @@ -433,6 +485,10 @@ error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) } error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) noexcept { + if (val.is_null()) { + out.reset(); + return SUCCESS; + } if (!out) { out = std::make_shared(); if (!out) { return MEMALLOC; } @@ -442,6 +498,10 @@ error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out } error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) noexcept { + if (val.is_null()) { + out.reset(); + return SUCCESS; + } if (!out) { out = std::make_shared(); if (!out) { return MEMALLOC; } @@ -451,6 +511,10 @@ error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) } error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) noexcept { + if (val.is_null()) { + out.reset(); + return SUCCESS; + } if (!out) { out = std::make_shared(); if (!out) { return MEMALLOC; } @@ -460,6 +524,61 @@ error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) noexcept { + // Check if the value is null + if (val.is_null()) { + out.reset(); // Set to nullptr + return SUCCESS; + } + + if (!out) { + out = std::make_unique(); + } + std::string_view str; + SIMDJSON_TRY(val.get_string().get(str)); + *out = std::string{str}; + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr &out) noexcept { + // Check if the value is null + if (val.is_null()) { + out.reset(); // Set to nullptr + return SUCCESS; + } + + if (!out) { + out = std::make_shared(); + } + std::string_view str; + SIMDJSON_TRY(val.get_string().get(str)); + *out = std::string{str}; + return SUCCESS; +} + +error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr &out) noexcept { + // Check if the value is null + if (val.is_null()) { + out.reset(); // Set to nullptr + return SUCCESS; + } + + if (!out) { + out = std::make_unique(); + } + int64_t temp; + SIMDJSON_TRY(val.get_int64().get(temp)); + *out = static_cast(temp); + return SUCCESS; +} + } // namespace simdjson #endif // SIMDJSON_ONDEMAND_DESERIALIZE_H diff --git a/include/simdjson/generic/ondemand/value.h b/include/simdjson/generic/ondemand/value.h index dc175bab4..6d3c8d279 100644 --- a/include/simdjson/generic/ondemand/value.h +++ b/include/simdjson/generic/ondemand/value.h @@ -75,6 +75,19 @@ public: #if SIMDJSON_SUPPORTS_DESERIALIZATION if constexpr (custom_deserializable) { return deserialize(*this, out); + } else if constexpr (concepts::optional_type) { + using value_type = typename std::remove_cvref_t::value_type; + + // Check if the value is null + if (is_null()) { + out.reset(); // Set to nullopt + return SUCCESS; + } + + if (!out) { + out.emplace(); + } + return get(out.value()); } else { static_assert(!sizeof(T), "The get method with type T is not implemented by the simdjson library. " "And you do not seem to have added support for it. Indeed, we have that " diff --git a/include/simdjson/ondemand.h b/include/simdjson/ondemand.h index 3cfa69ba7..6394fa3fe 100644 --- a/include/simdjson/ondemand.h +++ b/include/simdjson/ondemand.h @@ -12,6 +12,28 @@ namespace simdjson { * @copydoc simdjson::builtin::builder */ namespace builder = builtin::builder; + +#if SIMDJSON_STATIC_REFLECTION + /** + * Create a JSON string from any user-defined type using static reflection. + * Only available when SIMDJSON_STATIC_REFLECTION is enabled. + */ + template + requires(!std::same_as && + !std::same_as && + !std::same_as && + !std::same_as) + inline std::string to_json_string(const T& obj) { + builder::string_builder str_builder; + append(str_builder, obj); + std::string_view view; + if (str_builder.view().get(view) == SUCCESS) { + return std::string(view); + } + return ""; + } +#endif + } // namespace simdjson #endif // SIMDJSON_ONDEMAND_H diff --git a/tests/builder/CMakeLists.txt b/tests/builder/CMakeLists.txt index b8985fa0d..2bf59484d 100644 --- a/tests/builder/CMakeLists.txt +++ b/tests/builder/CMakeLists.txt @@ -3,6 +3,9 @@ include_directories(..) add_cpp_test(builder_string_builder_tests LABELS ondemand acceptance per_implementation) if(SIMDJSON_STATIC_REFLECTION) add_cpp_test(static_reflection_builder_tests LABELS ondemand acceptance per_implementation) + add_cpp_test(static_reflection_comprehensive_tests LABELS ondemand acceptance per_implementation) + add_cpp_test(static_reflection_edge_cases_tests LABELS ondemand acceptance per_implementation) + add_cpp_test(static_reflection_enum_tests LABELS ondemand acceptance per_implementation) endif(SIMDJSON_STATIC_REFLECTION) # Copy the simdjson dll into the tests directory if(MSVC AND BUILD_SHARED_LIBS) diff --git a/tests/builder/static_reflection_builder_tests.cpp b/tests/builder/static_reflection_builder_tests.cpp index d48086fd2..3b0981db2 100644 --- a/tests/builder/static_reflection_builder_tests.cpp +++ b/tests/builder/static_reflection_builder_tests.cpp @@ -48,14 +48,11 @@ namespace builder_tests { bool car_test() { TEST_START(); - simdjson::builder::string_builder sb; +#if SIMDJSON_STATIC_REFLECTION Car c = {"Toyota", "Corolla", 2017, {30.0,30.2,30.513,30.79}}; - append(sb, c); - std::string_view p; - auto result = sb.view().get(p); + auto result = builder::to_json_string(c); ASSERT_SUCCESS(result); - ASSERT_EQUAL(p, "{\"make\":\"Toyota\",\"model\":\"Corolla\",\"year\":2017,\"tire_pressure\":[30.0,30.2,30.513,30.79]}"); - std::string pstr(p.begin(), p.end()); + std::string pstr = result.value(); ASSERT_EQUAL(pstr, "{\"make\":\"Toyota\",\"model\":\"Corolla\",\"year\":2017,\"tire_pressure\":[30.0,30.2,30.513,30.79]}"); simdjson::ondemand::parser parser; simdjson::ondemand::document doc; @@ -70,12 +67,14 @@ namespace builder_tests { ASSERT_EQUAL(c2.tire_pressure[1], 30.2); ASSERT_EQUAL(c2.tire_pressure[2], 30.513); ASSERT_EQUAL(c2.tire_pressure[3], 30.79); +#endif TEST_SUCCEED(); } bool serialize_deserialize_kid() { TEST_START(); +#if SIMDJSON_STATIC_REFLECTION simdjson::padded_string json_str = R"({"age": 12, "name": "John", "toys": ["car", "ball"]})"_padded; simdjson::ondemand::parser parser; @@ -90,7 +89,7 @@ bool serialize_deserialize_kid() { ASSERT_EQUAL(k.toys[1], "ball"); // Now, go the other direction: std::string json; - ASSERT_SUCCESS(simdjson::builder::to_json_string(k).get(json)); + ASSERT_SUCCESS(builder::to_json_string(k).get(json)); std::cout << json << std::endl; // Now we parse it back: simdjson::ondemand::parser parser2; @@ -103,11 +102,13 @@ bool serialize_deserialize_kid() { ASSERT_EQUAL(k2.toys.size(), 2); ASSERT_EQUAL(k2.toys[0], "car"); ASSERT_EQUAL(k2.toys[1], "ball"); +#endif TEST_SUCCEED(); } bool serialize_deserialize_x_y_z() { TEST_START(); +#if SIMDJSON_STATIC_REFLECTION X s1 = {.a = '1', .b = 10, .c = 0, @@ -119,7 +120,7 @@ bool serialize_deserialize_x_y_z() { .i = {1, 2, 3}, .z = {.x = 1000}}}; std::string pstr; - ASSERT_SUCCESS(simdjson::builder::to_json_string(s1).get(pstr)); + ASSERT_SUCCESS(builder::to_json_string(s1).get(pstr)); ASSERT_EQUAL( pstr, R"({"a":"1","b":10,"c":0,"d":"test string\n\r\"","e":[1,2,3],"f":["ab","cd","fg"],"y":{"g":100,"h":"test string\n\r\"","i":[1,2,3],"z":{"x":1000}}})"); @@ -129,6 +130,7 @@ bool serialize_deserialize_x_y_z() { X s2; ASSERT_SUCCESS(doc.get().get(s2)); ASSERT_TRUE(s1 == s2); +#endif TEST_SUCCEED(); } diff --git a/tests/builder/static_reflection_comprehensive_tests.cpp b/tests/builder/static_reflection_comprehensive_tests.cpp new file mode 100644 index 000000000..426dfc5e1 --- /dev/null +++ b/tests/builder/static_reflection_comprehensive_tests.cpp @@ -0,0 +1,231 @@ +#include "simdjson.h" +#include "test_builder.h" +#include +#include +#include +#include +#include +#include +#include + +using namespace simdjson; + +namespace builder_tests { + + bool test_primitive_types() { + TEST_START(); +#if SIMDJSON_STATIC_REFLECTION + struct PrimitiveTypes { + bool bool_val; + char char_val; + int int_val; + double double_val; + float float_val; + }; + + PrimitiveTypes test{true, 'X', 42, 3.14159, 2.71f}; + + auto result = builder::to_json_string(test); + ASSERT_SUCCESS(result); + + std::string json = result.value(); + ASSERT_TRUE(json.find("\"bool_val\":true") != std::string::npos); + ASSERT_TRUE(json.find("\"char_val\":\"X\"") != std::string::npos); + ASSERT_TRUE(json.find("\"int_val\":42") != std::string::npos); + ASSERT_TRUE(json.find("\"double_val\":3.14159") != std::string::npos); + + // Test round-trip + ondemand::parser parser; + auto doc_result = parser.iterate(pad(json)); + ASSERT_SUCCESS(doc_result); + + auto get_result = doc_result.value().get(); + ASSERT_SUCCESS(get_result); + + PrimitiveTypes deserialized = std::move(get_result.value()); + ASSERT_EQUAL(deserialized.bool_val, test.bool_val); + ASSERT_EQUAL(deserialized.char_val, test.char_val); + ASSERT_EQUAL(deserialized.int_val, test.int_val); + ASSERT_EQUAL(deserialized.double_val, test.double_val); +#endif + TEST_SUCCEED(); + } + + bool test_string_types() { + TEST_START(); +#if SIMDJSON_STATIC_REFLECTION + struct StringTypes { + std::string string_val; + std::string_view string_view_val; + }; + + StringTypes test{"hello world", "test_view"}; + + auto result = builder::to_json_string(test); + ASSERT_SUCCESS(result); + + std::string json = result.value(); + ASSERT_TRUE(json.find("\"string_val\":\"hello world\"") != std::string::npos); + ASSERT_TRUE(json.find("\"string_view_val\":\"test_view\"") != std::string::npos); + + // Test round-trip + ondemand::parser parser; + auto doc_result = parser.iterate(pad(json)); + ASSERT_SUCCESS(doc_result); + + auto get_result = doc_result.value().get(); + ASSERT_SUCCESS(get_result); + + StringTypes deserialized = std::move(get_result.value()); + ASSERT_EQUAL(deserialized.string_val, test.string_val); +#endif + TEST_SUCCEED(); + } + + bool test_optional_types() { + TEST_START(); +#if SIMDJSON_STATIC_REFLECTION + struct OptionalTypes { + std::optional opt_int_with_value; + std::optional opt_string_with_value; + std::optional opt_int_null; + std::optional opt_string_null; + }; + + OptionalTypes test; + test.opt_int_with_value = 42; + test.opt_string_with_value = "optional_test"; + test.opt_int_null = std::nullopt; + test.opt_string_null = std::nullopt; + + auto result = builder::to_json_string(test); + ASSERT_SUCCESS(result); + + std::string json = result.value(); + ASSERT_TRUE(json.find("\"opt_int_with_value\":42") != std::string::npos); + ASSERT_TRUE(json.find("\"opt_string_with_value\":\"optional_test\"") != std::string::npos); + ASSERT_TRUE(json.find("\"opt_int_null\":null") != std::string::npos); + ASSERT_TRUE(json.find("\"opt_string_null\":null") != std::string::npos); + + // Test round-trip + ondemand::parser parser; + auto doc_result = parser.iterate(pad(json)); + ASSERT_SUCCESS(doc_result); + + auto get_result = doc_result.value().get(); + ASSERT_SUCCESS(get_result); + + OptionalTypes deserialized = std::move(get_result.value()); + ASSERT_TRUE(deserialized.opt_int_with_value.has_value()); + ASSERT_EQUAL(*deserialized.opt_int_with_value, 42); + ASSERT_TRUE(deserialized.opt_string_with_value.has_value()); + ASSERT_EQUAL(*deserialized.opt_string_with_value, "optional_test"); + ASSERT_FALSE(deserialized.opt_int_null.has_value()); + ASSERT_FALSE(deserialized.opt_string_null.has_value()); +#endif + TEST_SUCCEED(); + } + + bool test_smart_pointer_types() { + TEST_START(); +#if SIMDJSON_STATIC_REFLECTION + struct SmartPointerTypes { + std::unique_ptr unique_int_with_value; + std::shared_ptr shared_string_with_value; + std::unique_ptr unique_bool_with_value; + std::unique_ptr unique_int_null; + std::shared_ptr shared_string_null; + }; + + SmartPointerTypes test; + test.unique_int_with_value = std::make_unique(123); + test.shared_string_with_value = std::make_shared("shared_test"); + test.unique_bool_with_value = std::make_unique(true); + test.unique_int_null = nullptr; + test.shared_string_null = nullptr; + + auto result = builder::to_json_string(test); + ASSERT_SUCCESS(result); + + std::string json = result.value(); + ASSERT_TRUE(json.find("\"unique_int_with_value\":123") != std::string::npos); + ASSERT_TRUE(json.find("\"shared_string_with_value\":\"shared_test\"") != std::string::npos); + ASSERT_TRUE(json.find("\"unique_bool_with_value\":true") != std::string::npos); + ASSERT_TRUE(json.find("\"unique_int_null\":null") != std::string::npos); + ASSERT_TRUE(json.find("\"shared_string_null\":null") != std::string::npos); + + // Test round-trip + ondemand::parser parser; + auto doc_result = parser.iterate(pad(json)); + ASSERT_SUCCESS(doc_result); + + auto get_result = doc_result.value().get(); + ASSERT_SUCCESS(get_result); + + SmartPointerTypes deserialized = std::move(get_result.value()); + ASSERT_TRUE(deserialized.unique_int_with_value != nullptr); + ASSERT_EQUAL(*deserialized.unique_int_with_value, 123); + ASSERT_TRUE(deserialized.shared_string_with_value != nullptr); + ASSERT_EQUAL(*deserialized.shared_string_with_value, "shared_test"); + ASSERT_TRUE(deserialized.unique_bool_with_value != nullptr); + ASSERT_EQUAL(*deserialized.unique_bool_with_value, true); + ASSERT_TRUE(deserialized.unique_int_null == nullptr); + ASSERT_TRUE(deserialized.shared_string_null == nullptr); +#endif + TEST_SUCCEED(); + } + + bool test_container_types() { + TEST_START(); +#if SIMDJSON_STATIC_REFLECTION + struct ContainerTypes { + std::vector int_vector; + std::set string_set; + std::map string_map; + }; + + ContainerTypes test; + test.int_vector = {1, 2, 3, 4, 5}; + test.string_set = {"apple", "banana", "cherry"}; + test.string_map = {{"key1", 10}, {"key2", 20}, {"key3", 30}}; + + auto result = builder::to_json_string(test); + ASSERT_SUCCESS(result); + + std::string json = result.value(); + ASSERT_TRUE(json.find("\"int_vector\":[1,2,3,4,5]") != std::string::npos); + ASSERT_TRUE(json.find("\"string_set\":[") != std::string::npos); + ASSERT_TRUE(json.find("\"string_map\":{") != std::string::npos); + + // Test round-trip + ondemand::parser parser; + auto doc_result = parser.iterate(pad(json)); + ASSERT_SUCCESS(doc_result); + + auto get_result = doc_result.value().get(); + ASSERT_SUCCESS(get_result); + + ContainerTypes deserialized = std::move(get_result.value()); + ASSERT_EQUAL(deserialized.int_vector.size(), 5); + ASSERT_EQUAL(deserialized.string_set.size(), 3); + ASSERT_EQUAL(deserialized.string_map.size(), 3); + ASSERT_EQUAL(deserialized.int_vector[0], 1); + ASSERT_EQUAL(deserialized.int_vector[4], 5); +#endif + TEST_SUCCEED(); + } + + + bool run() { + return test_primitive_types() && + test_string_types() && + test_optional_types() && + test_smart_pointer_types() && + test_container_types(); + } + +} // namespace builder_tests + +int main(int argc, char *argv[]) { + return test_main(argc, argv, builder_tests::run); +} \ No newline at end of file diff --git a/tests/builder/static_reflection_edge_cases_tests.cpp b/tests/builder/static_reflection_edge_cases_tests.cpp new file mode 100644 index 000000000..3037414a4 --- /dev/null +++ b/tests/builder/static_reflection_edge_cases_tests.cpp @@ -0,0 +1,230 @@ +#include "simdjson.h" +#include "test_builder.h" +#include +#include +#include +#include +#include + +using namespace simdjson; + +namespace builder_tests { + + bool test_empty_values() { + TEST_START(); +#if SIMDJSON_STATIC_REFLECTION + struct EmptyValues { + std::string empty_string; + std::vector empty_vector; + std::optional null_optional; + std::unique_ptr null_unique_ptr; + std::shared_ptr null_shared_ptr; + }; + + EmptyValues test; + test.empty_string = ""; + // empty_vector is already empty by default + test.null_optional = std::nullopt; + test.null_unique_ptr = nullptr; + test.null_shared_ptr = nullptr; + + auto result = builder::to_json_string(test); + ASSERT_SUCCESS(result); + + std::string json = result.value(); + ASSERT_TRUE(json.find("\"empty_string\":\"\"") != std::string::npos); + ASSERT_TRUE(json.find("\"empty_vector\":[]") != std::string::npos); + ASSERT_TRUE(json.find("\"null_optional\":null") != std::string::npos); + ASSERT_TRUE(json.find("\"null_unique_ptr\":null") != std::string::npos); + ASSERT_TRUE(json.find("\"null_shared_ptr\":null") != std::string::npos); + + // Test round-trip + ondemand::parser parser; + auto doc_result = parser.iterate(pad(json)); + ASSERT_SUCCESS(doc_result); + + auto get_result = doc_result.value().get(); + ASSERT_SUCCESS(get_result); + + EmptyValues deserialized = std::move(get_result.value()); + ASSERT_EQUAL(deserialized.empty_string, ""); + ASSERT_EQUAL(deserialized.empty_vector.size(), 0); + ASSERT_FALSE(deserialized.null_optional.has_value()); + ASSERT_TRUE(deserialized.null_unique_ptr == nullptr); + ASSERT_TRUE(deserialized.null_shared_ptr == nullptr); +#endif + TEST_SUCCEED(); + } + + bool test_special_characters() { + TEST_START(); +#if SIMDJSON_STATIC_REFLECTION + struct SpecialChars { + std::string quotes; + std::string backslashes; + std::string newlines; + std::string unicode; + char null_char; + }; + + SpecialChars test; + test.quotes = "He said \"Hello\""; + test.backslashes = "Path\\to\\file"; + test.newlines = "Line1\nLine2\tTabbed"; + test.unicode = "Café résumé"; + test.null_char = '\0'; + + auto result = builder::to_json_string(test); + ASSERT_SUCCESS(result); + + std::string json = result.value(); + // Test that quotes are properly escaped + ASSERT_TRUE(json.find("\\\"Hello\\\"") != std::string::npos); + // Test that backslashes are properly escaped + ASSERT_TRUE(json.find("\\\\to\\\\") != std::string::npos); + // Test that newlines are properly escaped + ASSERT_TRUE(json.find("\\n") != std::string::npos); + ASSERT_TRUE(json.find("\\t") != std::string::npos); + + // Test round-trip (excluding null char which has special handling) + struct SpecialCharsNoNull { + std::string quotes; + std::string backslashes; + std::string newlines; + std::string unicode; + }; + + SpecialCharsNoNull test_no_null; + test_no_null.quotes = test.quotes; + test_no_null.backslashes = test.backslashes; + test_no_null.newlines = test.newlines; + test_no_null.unicode = test.unicode; + + auto result_no_null = builder::to_json_string(test_no_null); + ASSERT_SUCCESS(result_no_null); + + ondemand::parser parser; + auto doc_result = parser.iterate(pad(result_no_null.value())); + ASSERT_SUCCESS(doc_result); + + auto get_result = doc_result.value().get(); + ASSERT_SUCCESS(get_result); + + SpecialCharsNoNull deserialized = std::move(get_result.value()); + ASSERT_EQUAL(deserialized.quotes, test.quotes); + ASSERT_EQUAL(deserialized.backslashes, test.backslashes); + ASSERT_EQUAL(deserialized.newlines, test.newlines); + ASSERT_EQUAL(deserialized.unicode, test.unicode); +#endif + TEST_SUCCEED(); + } + + bool test_numeric_limits() { + TEST_START(); +#if SIMDJSON_STATIC_REFLECTION + struct NumericLimits { + int max_int; + int min_int; + double max_double; + double min_double; + bool true_val; + bool false_val; + }; + + NumericLimits test; + test.max_int = std::numeric_limits::max(); + test.min_int = std::numeric_limits::min(); + test.max_double = 1e100; // Large but safe double value + test.min_double = -1e100; // Large negative but safe double value + test.true_val = true; + test.false_val = false; + + auto result = builder::to_json_string(test); + ASSERT_SUCCESS(result); + + std::string json = result.value(); + ASSERT_TRUE(json.find("\"true_val\":true") != std::string::npos); + ASSERT_TRUE(json.find("\"false_val\":false") != std::string::npos); + + // Test round-trip + ondemand::parser parser; + auto doc_result = parser.iterate(pad(json)); + ASSERT_SUCCESS(doc_result); + + auto get_result = doc_result.value().get(); + ASSERT_SUCCESS(get_result); + + NumericLimits deserialized = std::move(get_result.value()); + ASSERT_EQUAL(deserialized.max_int, test.max_int); + ASSERT_EQUAL(deserialized.min_int, test.min_int); + ASSERT_EQUAL(deserialized.true_val, true); + ASSERT_EQUAL(deserialized.false_val, false); +#endif + TEST_SUCCEED(); + } + + bool test_nested_structures() { + TEST_START(); +#if SIMDJSON_STATIC_REFLECTION + struct Inner { + int value; + std::string name; + }; + + struct Outer { + Inner inner_obj; + std::vector inner_vector; + std::optional optional_inner; + std::unique_ptr unique_inner; + }; + + Outer test; + test.inner_obj = {42, "inner"}; + test.inner_vector = {{1, "first"}, {2, "second"}}; + test.optional_inner = Inner{99, "optional"}; + test.unique_inner = std::make_unique(Inner{123, "unique"}); + + auto result = builder::to_json_string(test); + ASSERT_SUCCESS(result); + + std::string json = result.value(); + ASSERT_TRUE(json.find("\"inner_obj\":{") != std::string::npos); + ASSERT_TRUE(json.find("\"inner_vector\":[") != std::string::npos); + ASSERT_TRUE(json.find("\"optional_inner\":{") != std::string::npos); + ASSERT_TRUE(json.find("\"unique_inner\":{") != std::string::npos); + + // Test round-trip + ondemand::parser parser; + auto doc_result = parser.iterate(pad(json)); + ASSERT_SUCCESS(doc_result); + + auto get_result = doc_result.value().get(); + ASSERT_SUCCESS(get_result); + + Outer deserialized = std::move(get_result.value()); + ASSERT_EQUAL(deserialized.inner_obj.value, 42); + ASSERT_EQUAL(deserialized.inner_obj.name, "inner"); + ASSERT_EQUAL(deserialized.inner_vector.size(), 2); + ASSERT_EQUAL(deserialized.inner_vector[0].value, 1); + ASSERT_EQUAL(deserialized.inner_vector[1].name, "second"); + ASSERT_TRUE(deserialized.optional_inner.has_value()); + ASSERT_EQUAL(deserialized.optional_inner->value, 99); + ASSERT_TRUE(deserialized.unique_inner != nullptr); + ASSERT_EQUAL(deserialized.unique_inner->value, 123); +#endif + TEST_SUCCEED(); + } + + + bool run() { + return test_empty_values() && + test_special_characters() && + test_numeric_limits() && + test_nested_structures(); + } + +} // namespace builder_tests + +int main(int argc, char *argv[]) { + return test_main(argc, argv, builder_tests::run); +} \ No newline at end of file diff --git a/tests/builder/static_reflection_enum_tests.cpp b/tests/builder/static_reflection_enum_tests.cpp new file mode 100644 index 000000000..22ceb37c7 --- /dev/null +++ b/tests/builder/static_reflection_enum_tests.cpp @@ -0,0 +1,258 @@ +#include "simdjson.h" +#include "test_builder.h" +#include + +using namespace simdjson; + +namespace builder_tests { + + bool test_enum_serialization() { + TEST_START(); +#if SIMDJSON_STATIC_REFLECTION + enum class Color { + Red, + Green, + Blue + }; + + struct EnumStruct { + Color color; + int value; + }; + + EnumStruct test{Color::Red, 42}; + + auto result = builder::to_json_string(test); + ASSERT_SUCCESS(result); + + std::string json = result.value(); + // Enum should be serialized as string (Red) + ASSERT_TRUE(json.find("\"color\":\"Red\"") != std::string::npos); + ASSERT_TRUE(json.find("\"value\":42") != std::string::npos); + + // Test different enum values + test.color = Color::Green; + auto result2 = builder::to_json_string(test); + ASSERT_SUCCESS(result2); + std::string json2 = result2.value(); + ASSERT_TRUE(json2.find("\"color\":\"Green\"") != std::string::npos); + + test.color = Color::Blue; + auto result3 = builder::to_json_string(test); + ASSERT_SUCCESS(result3); + std::string json3 = result3.value(); + ASSERT_TRUE(json3.find("\"color\":\"Blue\"") != std::string::npos); +#endif + TEST_SUCCEED(); + } + + bool test_enum_deserialization() { + TEST_START(); +#if SIMDJSON_STATIC_REFLECTION + enum class Status { + Active, + Inactive, + Pending + }; + + struct StatusStruct { + Status status; + std::string name; + }; + + // Test deserialization of different enum values with string representation + std::string json1 = "{\"status\":\"Active\",\"name\":\"test1\"}"; + ondemand::parser parser1; + auto doc_result1 = parser1.iterate(pad(json1)); + ASSERT_SUCCESS(doc_result1); + + auto get_result1 = doc_result1.value().get(); + ASSERT_SUCCESS(get_result1); + + StatusStruct deserialized1 = std::move(get_result1.value()); + ASSERT_TRUE(deserialized1.status == Status::Active); + ASSERT_EQUAL(deserialized1.name, "test1"); + + // Test Status::Inactive + std::string json2 = "{\"status\":\"Inactive\",\"name\":\"test2\"}"; + ondemand::parser parser2; + auto doc_result2 = parser2.iterate(pad(json2)); + ASSERT_SUCCESS(doc_result2); + + auto get_result2 = doc_result2.value().get(); + ASSERT_SUCCESS(get_result2); + + StatusStruct deserialized2 = std::move(get_result2.value()); + ASSERT_TRUE(deserialized2.status == Status::Inactive); + ASSERT_EQUAL(deserialized2.name, "test2"); + + // Test Status::Pending + std::string json3 = "{\"status\":\"Pending\",\"name\":\"test3\"}"; + ondemand::parser parser3; + auto doc_result3 = parser3.iterate(pad(json3)); + ASSERT_SUCCESS(doc_result3); + + auto get_result3 = doc_result3.value().get(); + ASSERT_SUCCESS(get_result3); + + StatusStruct deserialized3 = std::move(get_result3.value()); + ASSERT_TRUE(deserialized3.status == Status::Pending); + ASSERT_EQUAL(deserialized3.name, "test3"); +#endif + TEST_SUCCEED(); + } + + bool test_enum_round_trip() { + TEST_START(); +#if SIMDJSON_STATIC_REFLECTION + enum class Priority { + Low, + Medium, + High, + Critical + }; + + struct Task { + Priority priority; + std::string description; + int id; + }; + + Task original{Priority::High, "Important task", 123}; + + // Serialize + auto serialize_result = builder::to_json_string(original); + ASSERT_SUCCESS(serialize_result); + + std::string json = serialize_result.value(); + ASSERT_TRUE(json.find("\"priority\":\"High\"") != std::string::npos); // High as string + ASSERT_TRUE(json.find("\"description\":\"Important task\"") != std::string::npos); + ASSERT_TRUE(json.find("\"id\":123") != std::string::npos); + + // Deserialize + ondemand::parser parser; + auto doc_result = parser.iterate(pad(json)); + ASSERT_SUCCESS(doc_result); + + auto get_result = doc_result.value().get(); + ASSERT_SUCCESS(get_result); + + Task deserialized = std::move(get_result.value()); + ASSERT_TRUE(deserialized.priority == Priority::High); + ASSERT_EQUAL(deserialized.description, "Important task"); + ASSERT_EQUAL(deserialized.id, 123); +#endif + TEST_SUCCEED(); + } + + bool test_enum_with_underlying_type() { + TEST_START(); +#if SIMDJSON_STATIC_REFLECTION + enum class ErrorCode : int { + Success = 0, + NotFound = 404, + ServerError = 500 + }; + + struct Response { + ErrorCode error; + std::string message; + }; + + Response test{ErrorCode::NotFound, "Resource not found"}; + + auto result = builder::to_json_string(test); + ASSERT_SUCCESS(result); + + std::string json = result.value(); + ASSERT_TRUE(json.find("\"error\":\"NotFound\"") != std::string::npos); + ASSERT_TRUE(json.find("\"message\":\"Resource not found\"") != std::string::npos); + + // Test round-trip + ondemand::parser parser; + auto doc_result = parser.iterate(pad(json)); + ASSERT_SUCCESS(doc_result); + + auto get_result = doc_result.value().get(); + ASSERT_SUCCESS(get_result); + + Response deserialized = std::move(get_result.value()); + ASSERT_TRUE(deserialized.error == ErrorCode::NotFound); + ASSERT_EQUAL(deserialized.message, "Resource not found"); +#endif + TEST_SUCCEED(); + } + + bool test_multiple_enums() { + TEST_START(); +#if SIMDJSON_STATIC_REFLECTION + enum class Day { + Monday, + Tuesday, + Wednesday, + Thursday, + Friday, + Saturday, + Sunday + }; + + enum class Month { + January, + February, + March, + April, + May, + June, + July, + August, + September, + October, + November, + December + }; + + struct Date { + Day day; + Month month; + int year; + }; + + Date test{Day::Friday, Month::July, 2024}; + + auto result = builder::to_json_string(test); + ASSERT_SUCCESS(result); + + std::string json = result.value(); + ASSERT_TRUE(json.find("\"day\":\"Friday\"") != std::string::npos); // Friday as string + ASSERT_TRUE(json.find("\"month\":\"July\"") != std::string::npos); // July as string + ASSERT_TRUE(json.find("\"year\":2024") != std::string::npos); + + // Test round-trip + ondemand::parser parser; + auto doc_result = parser.iterate(pad(json)); + ASSERT_SUCCESS(doc_result); + + auto get_result = doc_result.value().get(); + ASSERT_SUCCESS(get_result); + + Date deserialized = std::move(get_result.value()); + ASSERT_TRUE(deserialized.day == Day::Friday); + ASSERT_TRUE(deserialized.month == Month::July); + ASSERT_EQUAL(deserialized.year, 2024); +#endif + TEST_SUCCEED(); + } + + bool run() { + return test_enum_serialization() && + test_enum_deserialization() && + test_enum_round_trip() && + test_enum_with_underlying_type() && + test_multiple_enums(); + } + +} // namespace builder_tests + +int main(int argc, char *argv[]) { + return test_main(argc, argv, builder_tests::run); +} \ No newline at end of file From aa6817d5aa8e1383eede14bb560ccdbe44cf095c Mon Sep 17 00:00:00 2001 From: Brad Bramble Date: Mon, 28 Jul 2025 11:44:32 -0400 Subject: [PATCH 07/11] Fix linker errors for downstream users caused by non-inline symbols in header (#2403) --- .../simdjson/generic/ondemand/json_string_builder-inl.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/include/simdjson/generic/ondemand/json_string_builder-inl.h b/include/simdjson/generic/ondemand/json_string_builder-inl.h index 42e42f7da..39e5d5cdc 100644 --- a/include/simdjson/generic/ondemand/json_string_builder-inl.h +++ b/include/simdjson/generic/ondemand/json_string_builder-inl.h @@ -303,9 +303,9 @@ simdjson_inline void string_builder::clear() noexcept { namespace internal { // We could specialize further for 32-bit integers. -int int_log2(uint32_t x) { return (63 - leading_zeroes(x | 1)); } +inline int int_log2(uint32_t x) { return (63 - leading_zeroes(x | 1)); } -int fast_digit_count(uint32_t x) { +inline int fast_digit_count(uint32_t x) { static uint64_t table[] = { 4294967296, 8589934582, 8589934582, 8589934582, 12884901788, 12884901788, 12884901788, 17179868184, 17179868184, 17179868184, @@ -317,9 +317,9 @@ int fast_digit_count(uint32_t x) { return uint32_t((x + table[int_log2(x)]) >> 32); } -int int_log2(uint64_t x) { return 63 - leading_zeroes(x | 1); } +inline int int_log2(uint64_t x) { return 63 - leading_zeroes(x | 1); } -int fast_digit_count(uint64_t x) { +inline int fast_digit_count(uint64_t x) { static uint64_t table[] = {9, 99, 999, From b9e308a727e52976a2cd504c8bd4d2af0e3c63c8 Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Mon, 28 Jul 2025 11:47:09 -0400 Subject: [PATCH 08/11] marking them as 'really inline' --- .../generic/ondemand/json_string_builder-inl.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/include/simdjson/generic/ondemand/json_string_builder-inl.h b/include/simdjson/generic/ondemand/json_string_builder-inl.h index 39e5d5cdc..30d078f86 100644 --- a/include/simdjson/generic/ondemand/json_string_builder-inl.h +++ b/include/simdjson/generic/ondemand/json_string_builder-inl.h @@ -303,9 +303,9 @@ simdjson_inline void string_builder::clear() noexcept { namespace internal { // We could specialize further for 32-bit integers. -inline int int_log2(uint32_t x) { return (63 - leading_zeroes(x | 1)); } +simdjson_really_inline int int_log2(uint32_t x) { return (63 - leading_zeroes(x | 1)); } -inline int fast_digit_count(uint32_t x) { +simdjson_really_inline int fast_digit_count(uint32_t x) { static uint64_t table[] = { 4294967296, 8589934582, 8589934582, 8589934582, 12884901788, 12884901788, 12884901788, 17179868184, 17179868184, 17179868184, @@ -317,9 +317,9 @@ inline int fast_digit_count(uint32_t x) { return uint32_t((x + table[int_log2(x)]) >> 32); } -inline int int_log2(uint64_t x) { return 63 - leading_zeroes(x | 1); } +simdjson_really_inline int int_log2(uint64_t x) { return 63 - leading_zeroes(x | 1); } -inline int fast_digit_count(uint64_t x) { +simdjson_really_inline int fast_digit_count(uint64_t x) { static uint64_t table[] = {9, 99, 999, @@ -346,7 +346,7 @@ inline int fast_digit_count(uint64_t x) { template ::value>::type> -simdjson_inline size_t digit_count(number_type v) noexcept { +simdjson_really_inline size_t digit_count(number_type v) noexcept { static_assert(sizeof(number_type) == 8 || sizeof(number_type) == 4 || sizeof(number_type) == 2 || sizeof(number_type) == 1, "We only support 8-bit, 16-bit, 32-bit and 64-bit numbers"); From d365cfb4a15027fa378453e139444d66b0f7ad44 Mon Sep 17 00:00:00 2001 From: Borislav Stanimirov Date: Thu, 31 Jul 2025 17:18:28 +0300 Subject: [PATCH 09/11] remove cmake_policy (#2404) * properly gitignore Visual Studio artifacts * remove cmake_policy --- .gitignore | 4 ++-- CMakeLists.txt | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 0608e31b2..3709099fb 100644 --- a/.gitignore +++ b/.gitignore @@ -38,7 +38,7 @@ cmake-build-release/ .history/ # Visual Studio artifacts -/VS/ +/.vs/ # C/C++ build outputs .build/ @@ -106,4 +106,4 @@ objs !.vscode/extensions.json # clangd -.cache \ No newline at end of file +.cache diff --git a/CMakeLists.txt b/CMakeLists.txt index af1e25d97..f26dd4b9e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,4 @@ cmake_minimum_required(VERSION 3.14) -cmake_policy(VERSION 3.5) # For doctest - project( simdjson From e28d63334e8f66d526618e7a10207e3b7121f5fa Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Thu, 31 Jul 2025 13:07:29 -0400 Subject: [PATCH 10/11] fix --- .../static_reflection_builder_tests.cpp | 60 ------------------- 1 file changed, 60 deletions(-) diff --git a/tests/builder/static_reflection_builder_tests.cpp b/tests/builder/static_reflection_builder_tests.cpp index 77462a38a..5034090a4 100644 --- a/tests/builder/static_reflection_builder_tests.cpp +++ b/tests/builder/static_reflection_builder_tests.cpp @@ -96,65 +96,6 @@ namespace builder_tests { #endif // SIMDJSON_EXCEPTIONS -<<<<<<< HEAD - bool serialize_deserialize_kid() { - TEST_START(); - simdjson::padded_string json_str = - R"({"age": 12, "name": "John", "toys": ["car", "ball"]})"_padded; - simdjson::ondemand::parser parser; - simdjson::ondemand::document doc; - ASSERT_SUCCESS(parser.iterate(json_str).get(doc)); - kid k; - ASSERT_SUCCESS(doc.get().get(k)); - ASSERT_EQUAL(k.age, 12); - ASSERT_EQUAL(k.name, "John"); - ASSERT_EQUAL(k.toys.size(), 2); - ASSERT_EQUAL(k.toys[0], "car"); - ASSERT_EQUAL(k.toys[1], "ball"); - // Now, go the other direction: - std::string json; - ASSERT_SUCCESS(simdjson::builder::to_json_string(k).get(json)); - std::cout << json << std::endl; - // Now we parse it back: - simdjson::ondemand::parser parser2; - simdjson::ondemand::document doc2; - ASSERT_SUCCESS(parser2.iterate(simdjson::pad(json)).get(doc2)); - kid k2; - ASSERT_SUCCESS(doc2.get().get(k2)); - ASSERT_EQUAL(k2.age, 12); - ASSERT_EQUAL(k2.name, "John"); - ASSERT_EQUAL(k2.toys.size(), 2); - ASSERT_EQUAL(k2.toys[0], "car"); - ASSERT_EQUAL(k2.toys[1], "ball"); - TEST_SUCCEED(); - } - - bool serialize_deserialize_x_y_z() { - TEST_START(); - X s1 = {.a = '1', - .b = 10, - .c = 0, - .d = "test string\n\r\"", - .e = {1, 2, 3}, - .f = {"ab", "cd", "fg"}, - .y = {.g = 100, - .h = "test string\n\r\"", - .i = {1, 2, 3}, - .z = {.x = 1000}}}; - std::string pstr; - ASSERT_SUCCESS(simdjson::builder::to_json_string(s1).get(pstr)); - ASSERT_EQUAL( - pstr, - R"({"a":"1","b":10,"c":0,"d":"test string\n\r\"","e":[1,2,3],"f":["ab","cd","fg"],"y":{"g":100,"h":"test string\n\r\"","i":[1,2,3],"z":{"x":1000}}})"); - simdjson::ondemand::parser parser; - simdjson::ondemand::document doc; - ASSERT_SUCCESS(parser.iterate(simdjson::pad(pstr)).get(doc)); - X s2; - ASSERT_SUCCESS(doc.get().get(s2)); - ASSERT_TRUE(s1 == s2); - TEST_SUCCEED(); - } -======= bool serialize_deserialize_kid() { TEST_START(); #if SIMDJSON_STATIC_REFLECTION @@ -216,7 +157,6 @@ bool serialize_deserialize_x_y_z() { #endif TEST_SUCCEED(); } ->>>>>>> master bool run() { return From dd4d02617f39cd9dd32ee64a12539bd1302719ee Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Thu, 31 Jul 2025 18:40:54 -0400 Subject: [PATCH 11/11] removing unnecessary hack --- include/simdjson/common_defs.h | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/include/simdjson/common_defs.h b/include/simdjson/common_defs.h index a151795a6..2cd772e88 100644 --- a/include/simdjson/common_defs.h +++ b/include/simdjson/common_defs.h @@ -199,17 +199,6 @@ double from_chars(const char *first, const char* end) noexcept; // We assume by default static linkage #define SIMDJSON_DLLIMPORTEXPORT #endif - -/** - * Workaround for the vcpkg package manager. Only vcpkg should - * ever touch the next line. The SIMDJSON_USING_LIBRARY macro is otherwise unused. - */ -#if SIMDJSON_USING_LIBRARY -#define SIMDJSON_DLLIMPORTEXPORT __declspec(dllimport) -#endif -/** - * End of workaround for the vcpkg package manager. - */ #else #define SIMDJSON_DLLIMPORTEXPORT #endif