From 1f369ef210062c31e20c81959a558abaa1ab9e6b Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Mon, 15 Sep 2025 22:22:25 -0600 Subject: [PATCH] =?UTF-8?q?minor=20patch=20which=20allows=20us=20to=20pass?= =?UTF-8?q?=20mutable=20strings=20to=20simdjson::from=E2=80=A6=20(#2448)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * minor patch which allows us to pass mutable strings to simdjson::from and fix an issue with ambiguous integrals * compatibility patch. --- CMakeLists.txt | 2 +- Doxyfile | 2 +- benchmark/static_reflect/CMakeLists.txt | 64 ++++--- include/simdjson/convert-inl.h | 10 + include/simdjson/convert.h | 5 + .../ondemand/json_string_builder-inl.h | 17 +- include/simdjson/simdjson_version.h | 4 +- singleheader/simdjson.cpp | 2 +- singleheader/simdjson.h | 173 ++++++++++++------ singleheader/singleheader.zip | Bin 8238963 -> 8241047 bytes tests/ondemand/ondemand_convert_tests.cpp | 53 +++++- 11 files changed, 231 insertions(+), 101 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c8c027c7b..8f828194a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.14) project( simdjson # The version number is modified by tools/release.py - VERSION 4.0.2 + VERSION 4.0.3 DESCRIPTION "Parsing gigabytes of JSON per second" HOMEPAGE_URL "https://simdjson.org/" LANGUAGES CXX C diff --git a/Doxyfile b/Doxyfile index 02ffbef4a..0cceef438 100644 --- a/Doxyfile +++ b/Doxyfile @@ -38,7 +38,7 @@ PROJECT_NAME = simdjson # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = "4.0.2" +PROJECT_NUMBER = "4.0.3" # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/benchmark/static_reflect/CMakeLists.txt b/benchmark/static_reflect/CMakeLists.txt index 66196351c..28c3be8ea 100644 --- a/benchmark/static_reflect/CMakeLists.txt +++ b/benchmark/static_reflect/CMakeLists.txt @@ -6,38 +6,42 @@ CPMAddPackage( EXCLUDE_FROM_ALL YES ) +option(SIMDJSON_USE_RUST "Build the static_reflect benchmark" OFF) - -if(NOT WIN32) -# We want the check whether Rust is available before trying to build a crate. -CPMAddPackage( - NAME corrosion - GITHUB_REPOSITORY corrosion-rs/corrosion - VERSION 0.4.4 - DOWNLOAD_ONLY ON - OPTIONS "Rust_FIND_QUIETLY OFF" -) -include("${corrosion_SOURCE_DIR}/cmake/FindRust.cmake") -endif() - -if(RUST_FOUND) - message(STATUS "Rust found: " ${Rust_VERSION} ) - add_subdirectory("${corrosion_SOURCE_DIR}" "${PROJECT_BINARY_DIR}/_deps/corrosion" EXCLUDE_FROM_ALL) - # Important: we want to build in release mode! - corrosion_import_crate(MANIFEST_PATH "serde-benchmark/Cargo.toml" NO_LINKER_OVERRIDE PROFILE release) -else() - message(STATUS "Rust/Cargo is unavailable." ) - message(STATUS "We will not benchmark serde-benchmark." ) - if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") - message(STATUS "Under macOS, you may be able to install rust with") - message(STATUS "curl https://sh.rustup.rs -sSf | sh") - elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux") - message(STATUS "Under Linux, you may be able to install rust with a command such as") - message(STATUS "apt-get install cargo" ) - message(STATUS "or" ) - message(STATUS "curl https://sh.rustup.rs -sSf | sh") +if(SIMDJSON_USER_RUST) + if(NOT WIN32) + # We want the check whether Rust is available before trying to build a crate. + CPMAddPackage( + NAME corrosion + GITHUB_REPOSITORY corrosion-rs/corrosion + VERSION 0.4.4 + DOWNLOAD_ONLY ON + OPTIONS "Rust_FIND_QUIETLY OFF" + ) + include("${corrosion_SOURCE_DIR}/cmake/FindRust.cmake") endif() -endif() + + if(RUST_FOUND) + message(STATUS "Rust found: " ${Rust_VERSION} ) + add_subdirectory("${corrosion_SOURCE_DIR}" "${PROJECT_BINARY_DIR}/_deps/corrosion" EXCLUDE_FROM_ALL) + # Important: we want to build in release mode! + corrosion_import_crate(MANIFEST_PATH "serde-benchmark/Cargo.toml" NO_LINKER_OVERRIDE PROFILE release) + else() + message(STATUS "Rust/Cargo is unavailable." ) + message(STATUS "We will not benchmark serde-benchmark." ) + if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") + message(STATUS "Under macOS, you may be able to install rust with") + message(STATUS "curl https://sh.rustup.rs -sSf | sh") + elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux") + message(STATUS "Under Linux, you may be able to install rust with a command such as") + message(STATUS "apt-get install cargo" ) + message(STATUS "or" ) + message(STATUS "curl https://sh.rustup.rs -sSf | sh") + endif() + endif() +else(SIMDJSON_USER_RUST) + message(STATUS "We will not benchmark serde-benchmark." ) +endif(SIMDJSON_USER_RUST) # Add the benchmark executable targets add_subdirectory(twitter_benchmark) diff --git a/include/simdjson/convert-inl.h b/include/simdjson/convert-inl.h index 5bd05e521..0fd60e75f 100644 --- a/include/simdjson/convert-inl.h +++ b/include/simdjson/convert-inl.h @@ -116,6 +116,16 @@ template inline auto to_adaptor::operator()(ondemand::parser &parser, padded_string_view const str) const noexcept { return auto_parser{parser, str}; } + +template +inline auto to_adaptor::operator()(std::string str) const noexcept { + return auto_parser{pad_with_reserve(str)}; +} + +template +inline auto to_adaptor::operator()(ondemand::parser &parser, std::string str) const noexcept { + return auto_parser{parser, pad_with_reserve(str)}; +} } // namespace internal } // namespace convert } // namespace simdjson diff --git a/include/simdjson/convert.h b/include/simdjson/convert.h index fa6246be0..e6d613fde 100644 --- a/include/simdjson/convert.h +++ b/include/simdjson/convert.h @@ -72,6 +72,11 @@ struct to_adaptor { T operator()(simdjson_result &val) const noexcept; auto operator()(padded_string_view const str) const noexcept; auto operator()(ondemand::parser &parser, padded_string_view const str) const noexcept; + // The std::string is padded with reserve to ensure there is enough space for padding. + // Some sanitizers may not like this, so you can use simdjson::pad instead. + // simdjson::from(simdjson::pad(str)) + auto operator()(std::string str) const noexcept; + auto operator()(ondemand::parser &parser, std::string str) const noexcept; }; } // namespace internal } // namespace convert diff --git a/include/simdjson/generic/ondemand/json_string_builder-inl.h b/include/simdjson/generic/ondemand/json_string_builder-inl.h index ed74bbdd5..ebfe8a925 100644 --- a/include/simdjson/generic/ondemand/json_string_builder-inl.h +++ b/include/simdjson/generic/ondemand/json_string_builder-inl.h @@ -284,10 +284,12 @@ simdjson_inline void string_builder::clear() noexcept { namespace internal { -// We could specialize further for 32-bit integers. -simdjson_really_inline int int_log2(uint32_t x) { return (63 - leading_zeroes(x | 1)); } -simdjson_really_inline int fast_digit_count(uint32_t x) { +template ::value>::type> +simdjson_really_inline int int_log2(number_type x) { return 63 - leading_zeroes(uint64_t(x) | 1); } + +simdjson_really_inline int fast_digit_count_32(uint32_t x) { static uint64_t table[] = { 4294967296, 8589934582, 8589934582, 8589934582, 12884901788, 12884901788, 12884901788, 17179868184, 17179868184, 17179868184, @@ -299,9 +301,8 @@ simdjson_really_inline int fast_digit_count(uint32_t x) { return uint32_t((x + table[int_log2(x)]) >> 32); } -simdjson_really_inline int int_log2(uint64_t x) { return 63 - leading_zeroes(x | 1); } -simdjson_really_inline int fast_digit_count(uint64_t x) { +simdjson_really_inline int fast_digit_count_64(uint64_t x) { static uint64_t table[] = {9, 99, 999, @@ -332,7 +333,11 @@ simdjson_really_inline size_t digit_count(number_type v) noexcept { static_assert(sizeof(number_type) == 8 || sizeof(number_type) == 4 || sizeof(number_type) == 2 || sizeof(number_type) == 1, "We only support 8-bit, 16-bit, 32-bit and 64-bit numbers"); - return fast_digit_count(v); + SIMDJSON_IF_CONSTEXPR(sizeof(number_type) <= 4) { + return fast_digit_count_32(static_cast(v)); + } else { + return fast_digit_count_64(static_cast(v)); + } } static const char decimal_table[200] = { 0x30, 0x30, 0x30, 0x31, 0x30, 0x32, 0x30, 0x33, 0x30, 0x34, 0x30, 0x35, diff --git a/include/simdjson/simdjson_version.h b/include/simdjson/simdjson_version.h index 4b6004b5f..0972eecfb 100644 --- a/include/simdjson/simdjson_version.h +++ b/include/simdjson/simdjson_version.h @@ -4,7 +4,7 @@ #define SIMDJSON_SIMDJSON_VERSION_H /** The version of simdjson being used (major.minor.revision) */ -#define SIMDJSON_VERSION "4.0.2" +#define SIMDJSON_VERSION "4.0.3" namespace simdjson { enum { @@ -19,7 +19,7 @@ enum { /** * The revision (major.minor.REVISION) of simdjson being used. */ - SIMDJSON_VERSION_REVISION = 2 + SIMDJSON_VERSION_REVISION = 3 }; } // namespace simdjson diff --git a/singleheader/simdjson.cpp b/singleheader/simdjson.cpp index d1fc6b9b8..f4a488e2b 100644 --- a/singleheader/simdjson.cpp +++ b/singleheader/simdjson.cpp @@ -1,4 +1,4 @@ -/* auto-generated on 2025-09-14 09:01:41 -0600. version 4.0.2 Do not edit! */ +/* auto-generated on 2025-09-15 19:47:29 -0600. version 4.0.3 Do not edit! */ /* including simdjson.cpp: */ /* begin file simdjson.cpp */ #define SIMDJSON_SRC_SIMDJSON_CPP diff --git a/singleheader/simdjson.h b/singleheader/simdjson.h index bf27830da..69f6826fb 100644 --- a/singleheader/simdjson.h +++ b/singleheader/simdjson.h @@ -1,4 +1,4 @@ -/* auto-generated on 2025-09-14 09:01:41 -0600. version 4.0.2 Do not edit! */ +/* auto-generated on 2025-09-15 19:47:29 -0600. version 4.0.3 Do not edit! */ /* including simdjson.h: */ /* begin file simdjson.h */ #ifndef SIMDJSON_H @@ -2508,7 +2508,7 @@ namespace std { #define SIMDJSON_SIMDJSON_VERSION_H /** The version of simdjson being used (major.minor.revision) */ -#define SIMDJSON_VERSION "4.0.2" +#define SIMDJSON_VERSION "4.0.3" namespace simdjson { enum { @@ -2523,7 +2523,7 @@ enum { /** * The revision (major.minor.REVISION) of simdjson being used. */ - SIMDJSON_VERSION_REVISION = 2 + SIMDJSON_VERSION_REVISION = 3 }; } // namespace simdjson @@ -38968,7 +38968,7 @@ error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept { } error_code e = simdjson::SUCCESS; template for (constexpr auto mem : std::define_static_array(std::meta::nonstatic_data_members_of(^^T, std::meta::access_context::unchecked()))) { - if constexpr (!std::meta::is_const(mem) && std::meta::is_public(mem)) { + if constexpr (!std::meta::is_const(mem) && std::meta::is_public(mem)) { constexpr std::string_view key = std::define_static_string(std::meta::identifier_of(mem)); // Note: removed static assert as optional types are now handled generically // as long we are succesful or the field is not found, we continue @@ -45345,10 +45345,12 @@ simdjson_inline void string_builder::clear() noexcept { namespace internal { -// We could specialize further for 32-bit integers. -simdjson_really_inline int int_log2(uint32_t x) { return (63 - leading_zeroes(x | 1)); } -simdjson_really_inline int fast_digit_count(uint32_t x) { +template ::value>::type> +simdjson_really_inline int int_log2(number_type x) { return 63 - leading_zeroes(uint64_t(x) | 1); } + +simdjson_really_inline int fast_digit_count_32(uint32_t x) { static uint64_t table[] = { 4294967296, 8589934582, 8589934582, 8589934582, 12884901788, 12884901788, 12884901788, 17179868184, 17179868184, 17179868184, @@ -45360,9 +45362,8 @@ simdjson_really_inline int fast_digit_count(uint32_t x) { return uint32_t((x + table[int_log2(x)]) >> 32); } -simdjson_really_inline int int_log2(uint64_t x) { return 63 - leading_zeroes(x | 1); } -simdjson_really_inline int fast_digit_count(uint64_t x) { +simdjson_really_inline int fast_digit_count_64(uint64_t x) { static uint64_t table[] = {9, 99, 999, @@ -45393,7 +45394,11 @@ simdjson_really_inline size_t digit_count(number_type v) noexcept { static_assert(sizeof(number_type) == 8 || sizeof(number_type) == 4 || sizeof(number_type) == 2 || sizeof(number_type) == 1, "We only support 8-bit, 16-bit, 32-bit and 64-bit numbers"); - return fast_digit_count(v); + SIMDJSON_IF_CONSTEXPR(sizeof(number_type) <= 4) { + return fast_digit_count_32(static_cast(v)); + } else { + return fast_digit_count_64(static_cast(v)); + } } static const char decimal_table[200] = { 0x30, 0x30, 0x30, 0x31, 0x30, 0x32, 0x30, 0x33, 0x30, 0x34, 0x30, 0x35, @@ -51927,7 +51932,7 @@ error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept { } error_code e = simdjson::SUCCESS; template for (constexpr auto mem : std::define_static_array(std::meta::nonstatic_data_members_of(^^T, std::meta::access_context::unchecked()))) { - if constexpr (!std::meta::is_const(mem) && std::meta::is_public(mem)) { + if constexpr (!std::meta::is_const(mem) && std::meta::is_public(mem)) { constexpr std::string_view key = std::define_static_string(std::meta::identifier_of(mem)); // Note: removed static assert as optional types are now handled generically // as long we are succesful or the field is not found, we continue @@ -58304,10 +58309,12 @@ simdjson_inline void string_builder::clear() noexcept { namespace internal { -// We could specialize further for 32-bit integers. -simdjson_really_inline int int_log2(uint32_t x) { return (63 - leading_zeroes(x | 1)); } -simdjson_really_inline int fast_digit_count(uint32_t x) { +template ::value>::type> +simdjson_really_inline int int_log2(number_type x) { return 63 - leading_zeroes(uint64_t(x) | 1); } + +simdjson_really_inline int fast_digit_count_32(uint32_t x) { static uint64_t table[] = { 4294967296, 8589934582, 8589934582, 8589934582, 12884901788, 12884901788, 12884901788, 17179868184, 17179868184, 17179868184, @@ -58319,9 +58326,8 @@ simdjson_really_inline int fast_digit_count(uint32_t x) { return uint32_t((x + table[int_log2(x)]) >> 32); } -simdjson_really_inline int int_log2(uint64_t x) { return 63 - leading_zeroes(x | 1); } -simdjson_really_inline int fast_digit_count(uint64_t x) { +simdjson_really_inline int fast_digit_count_64(uint64_t x) { static uint64_t table[] = {9, 99, 999, @@ -58352,7 +58358,11 @@ simdjson_really_inline size_t digit_count(number_type v) noexcept { static_assert(sizeof(number_type) == 8 || sizeof(number_type) == 4 || sizeof(number_type) == 2 || sizeof(number_type) == 1, "We only support 8-bit, 16-bit, 32-bit and 64-bit numbers"); - return fast_digit_count(v); + SIMDJSON_IF_CONSTEXPR(sizeof(number_type) <= 4) { + return fast_digit_count_32(static_cast(v)); + } else { + return fast_digit_count_64(static_cast(v)); + } } static const char decimal_table[200] = { 0x30, 0x30, 0x30, 0x31, 0x30, 0x32, 0x30, 0x33, 0x30, 0x34, 0x30, 0x35, @@ -65385,7 +65395,7 @@ error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept { } error_code e = simdjson::SUCCESS; template for (constexpr auto mem : std::define_static_array(std::meta::nonstatic_data_members_of(^^T, std::meta::access_context::unchecked()))) { - if constexpr (!std::meta::is_const(mem) && std::meta::is_public(mem)) { + if constexpr (!std::meta::is_const(mem) && std::meta::is_public(mem)) { constexpr std::string_view key = std::define_static_string(std::meta::identifier_of(mem)); // Note: removed static assert as optional types are now handled generically // as long we are succesful or the field is not found, we continue @@ -71762,10 +71772,12 @@ simdjson_inline void string_builder::clear() noexcept { namespace internal { -// We could specialize further for 32-bit integers. -simdjson_really_inline int int_log2(uint32_t x) { return (63 - leading_zeroes(x | 1)); } -simdjson_really_inline int fast_digit_count(uint32_t x) { +template ::value>::type> +simdjson_really_inline int int_log2(number_type x) { return 63 - leading_zeroes(uint64_t(x) | 1); } + +simdjson_really_inline int fast_digit_count_32(uint32_t x) { static uint64_t table[] = { 4294967296, 8589934582, 8589934582, 8589934582, 12884901788, 12884901788, 12884901788, 17179868184, 17179868184, 17179868184, @@ -71777,9 +71789,8 @@ simdjson_really_inline int fast_digit_count(uint32_t x) { return uint32_t((x + table[int_log2(x)]) >> 32); } -simdjson_really_inline int int_log2(uint64_t x) { return 63 - leading_zeroes(x | 1); } -simdjson_really_inline int fast_digit_count(uint64_t x) { +simdjson_really_inline int fast_digit_count_64(uint64_t x) { static uint64_t table[] = {9, 99, 999, @@ -71810,7 +71821,11 @@ simdjson_really_inline size_t digit_count(number_type v) noexcept { static_assert(sizeof(number_type) == 8 || sizeof(number_type) == 4 || sizeof(number_type) == 2 || sizeof(number_type) == 1, "We only support 8-bit, 16-bit, 32-bit and 64-bit numbers"); - return fast_digit_count(v); + SIMDJSON_IF_CONSTEXPR(sizeof(number_type) <= 4) { + return fast_digit_count_32(static_cast(v)); + } else { + return fast_digit_count_64(static_cast(v)); + } } static const char decimal_table[200] = { 0x30, 0x30, 0x30, 0x31, 0x30, 0x32, 0x30, 0x33, 0x30, 0x34, 0x30, 0x35, @@ -78843,7 +78858,7 @@ error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept { } error_code e = simdjson::SUCCESS; template for (constexpr auto mem : std::define_static_array(std::meta::nonstatic_data_members_of(^^T, std::meta::access_context::unchecked()))) { - if constexpr (!std::meta::is_const(mem) && std::meta::is_public(mem)) { + if constexpr (!std::meta::is_const(mem) && std::meta::is_public(mem)) { constexpr std::string_view key = std::define_static_string(std::meta::identifier_of(mem)); // Note: removed static assert as optional types are now handled generically // as long we are succesful or the field is not found, we continue @@ -85220,10 +85235,12 @@ simdjson_inline void string_builder::clear() noexcept { namespace internal { -// We could specialize further for 32-bit integers. -simdjson_really_inline int int_log2(uint32_t x) { return (63 - leading_zeroes(x | 1)); } -simdjson_really_inline int fast_digit_count(uint32_t x) { +template ::value>::type> +simdjson_really_inline int int_log2(number_type x) { return 63 - leading_zeroes(uint64_t(x) | 1); } + +simdjson_really_inline int fast_digit_count_32(uint32_t x) { static uint64_t table[] = { 4294967296, 8589934582, 8589934582, 8589934582, 12884901788, 12884901788, 12884901788, 17179868184, 17179868184, 17179868184, @@ -85235,9 +85252,8 @@ simdjson_really_inline int fast_digit_count(uint32_t x) { return uint32_t((x + table[int_log2(x)]) >> 32); } -simdjson_really_inline int int_log2(uint64_t x) { return 63 - leading_zeroes(x | 1); } -simdjson_really_inline int fast_digit_count(uint64_t x) { +simdjson_really_inline int fast_digit_count_64(uint64_t x) { static uint64_t table[] = {9, 99, 999, @@ -85268,7 +85284,11 @@ simdjson_really_inline size_t digit_count(number_type v) noexcept { static_assert(sizeof(number_type) == 8 || sizeof(number_type) == 4 || sizeof(number_type) == 2 || sizeof(number_type) == 1, "We only support 8-bit, 16-bit, 32-bit and 64-bit numbers"); - return fast_digit_count(v); + SIMDJSON_IF_CONSTEXPR(sizeof(number_type) <= 4) { + return fast_digit_count_32(static_cast(v)); + } else { + return fast_digit_count_64(static_cast(v)); + } } static const char decimal_table[200] = { 0x30, 0x30, 0x30, 0x31, 0x30, 0x32, 0x30, 0x33, 0x30, 0x34, 0x30, 0x35, @@ -92416,7 +92436,7 @@ error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept { } error_code e = simdjson::SUCCESS; template for (constexpr auto mem : std::define_static_array(std::meta::nonstatic_data_members_of(^^T, std::meta::access_context::unchecked()))) { - if constexpr (!std::meta::is_const(mem) && std::meta::is_public(mem)) { + if constexpr (!std::meta::is_const(mem) && std::meta::is_public(mem)) { constexpr std::string_view key = std::define_static_string(std::meta::identifier_of(mem)); // Note: removed static assert as optional types are now handled generically // as long we are succesful or the field is not found, we continue @@ -98793,10 +98813,12 @@ simdjson_inline void string_builder::clear() noexcept { namespace internal { -// We could specialize further for 32-bit integers. -simdjson_really_inline int int_log2(uint32_t x) { return (63 - leading_zeroes(x | 1)); } -simdjson_really_inline int fast_digit_count(uint32_t x) { +template ::value>::type> +simdjson_really_inline int int_log2(number_type x) { return 63 - leading_zeroes(uint64_t(x) | 1); } + +simdjson_really_inline int fast_digit_count_32(uint32_t x) { static uint64_t table[] = { 4294967296, 8589934582, 8589934582, 8589934582, 12884901788, 12884901788, 12884901788, 17179868184, 17179868184, 17179868184, @@ -98808,9 +98830,8 @@ simdjson_really_inline int fast_digit_count(uint32_t x) { return uint32_t((x + table[int_log2(x)]) >> 32); } -simdjson_really_inline int int_log2(uint64_t x) { return 63 - leading_zeroes(x | 1); } -simdjson_really_inline int fast_digit_count(uint64_t x) { +simdjson_really_inline int fast_digit_count_64(uint64_t x) { static uint64_t table[] = {9, 99, 999, @@ -98841,7 +98862,11 @@ simdjson_really_inline size_t digit_count(number_type v) noexcept { static_assert(sizeof(number_type) == 8 || sizeof(number_type) == 4 || sizeof(number_type) == 2 || sizeof(number_type) == 1, "We only support 8-bit, 16-bit, 32-bit and 64-bit numbers"); - return fast_digit_count(v); + SIMDJSON_IF_CONSTEXPR(sizeof(number_type) <= 4) { + return fast_digit_count_32(static_cast(v)); + } else { + return fast_digit_count_64(static_cast(v)); + } } static const char decimal_table[200] = { 0x30, 0x30, 0x30, 0x31, 0x30, 0x32, 0x30, 0x33, 0x30, 0x34, 0x30, 0x35, @@ -106305,7 +106330,7 @@ error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept { } error_code e = simdjson::SUCCESS; template for (constexpr auto mem : std::define_static_array(std::meta::nonstatic_data_members_of(^^T, std::meta::access_context::unchecked()))) { - if constexpr (!std::meta::is_const(mem) && std::meta::is_public(mem)) { + if constexpr (!std::meta::is_const(mem) && std::meta::is_public(mem)) { constexpr std::string_view key = std::define_static_string(std::meta::identifier_of(mem)); // Note: removed static assert as optional types are now handled generically // as long we are succesful or the field is not found, we continue @@ -112682,10 +112707,12 @@ simdjson_inline void string_builder::clear() noexcept { namespace internal { -// We could specialize further for 32-bit integers. -simdjson_really_inline int int_log2(uint32_t x) { return (63 - leading_zeroes(x | 1)); } -simdjson_really_inline int fast_digit_count(uint32_t x) { +template ::value>::type> +simdjson_really_inline int int_log2(number_type x) { return 63 - leading_zeroes(uint64_t(x) | 1); } + +simdjson_really_inline int fast_digit_count_32(uint32_t x) { static uint64_t table[] = { 4294967296, 8589934582, 8589934582, 8589934582, 12884901788, 12884901788, 12884901788, 17179868184, 17179868184, 17179868184, @@ -112697,9 +112724,8 @@ simdjson_really_inline int fast_digit_count(uint32_t x) { return uint32_t((x + table[int_log2(x)]) >> 32); } -simdjson_really_inline int int_log2(uint64_t x) { return 63 - leading_zeroes(x | 1); } -simdjson_really_inline int fast_digit_count(uint64_t x) { +simdjson_really_inline int fast_digit_count_64(uint64_t x) { static uint64_t table[] = {9, 99, 999, @@ -112730,7 +112756,11 @@ simdjson_really_inline size_t digit_count(number_type v) noexcept { static_assert(sizeof(number_type) == 8 || sizeof(number_type) == 4 || sizeof(number_type) == 2 || sizeof(number_type) == 1, "We only support 8-bit, 16-bit, 32-bit and 64-bit numbers"); - return fast_digit_count(v); + SIMDJSON_IF_CONSTEXPR(sizeof(number_type) <= 4) { + return fast_digit_count_32(static_cast(v)); + } else { + return fast_digit_count_64(static_cast(v)); + } } static const char decimal_table[200] = { 0x30, 0x30, 0x30, 0x31, 0x30, 0x32, 0x30, 0x33, 0x30, 0x34, 0x30, 0x35, @@ -119671,7 +119701,7 @@ error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept { } error_code e = simdjson::SUCCESS; template for (constexpr auto mem : std::define_static_array(std::meta::nonstatic_data_members_of(^^T, std::meta::access_context::unchecked()))) { - if constexpr (!std::meta::is_const(mem) && std::meta::is_public(mem)) { + if constexpr (!std::meta::is_const(mem) && std::meta::is_public(mem)) { constexpr std::string_view key = std::define_static_string(std::meta::identifier_of(mem)); // Note: removed static assert as optional types are now handled generically // as long we are succesful or the field is not found, we continue @@ -126048,10 +126078,12 @@ simdjson_inline void string_builder::clear() noexcept { namespace internal { -// We could specialize further for 32-bit integers. -simdjson_really_inline int int_log2(uint32_t x) { return (63 - leading_zeroes(x | 1)); } -simdjson_really_inline int fast_digit_count(uint32_t x) { +template ::value>::type> +simdjson_really_inline int int_log2(number_type x) { return 63 - leading_zeroes(uint64_t(x) | 1); } + +simdjson_really_inline int fast_digit_count_32(uint32_t x) { static uint64_t table[] = { 4294967296, 8589934582, 8589934582, 8589934582, 12884901788, 12884901788, 12884901788, 17179868184, 17179868184, 17179868184, @@ -126063,9 +126095,8 @@ simdjson_really_inline int fast_digit_count(uint32_t x) { return uint32_t((x + table[int_log2(x)]) >> 32); } -simdjson_really_inline int int_log2(uint64_t x) { return 63 - leading_zeroes(x | 1); } -simdjson_really_inline int fast_digit_count(uint64_t x) { +simdjson_really_inline int fast_digit_count_64(uint64_t x) { static uint64_t table[] = {9, 99, 999, @@ -126096,7 +126127,11 @@ simdjson_really_inline size_t digit_count(number_type v) noexcept { static_assert(sizeof(number_type) == 8 || sizeof(number_type) == 4 || sizeof(number_type) == 2 || sizeof(number_type) == 1, "We only support 8-bit, 16-bit, 32-bit and 64-bit numbers"); - return fast_digit_count(v); + SIMDJSON_IF_CONSTEXPR(sizeof(number_type) <= 4) { + return fast_digit_count_32(static_cast(v)); + } else { + return fast_digit_count_64(static_cast(v)); + } } static const char decimal_table[200] = { 0x30, 0x30, 0x30, 0x31, 0x30, 0x32, 0x30, 0x33, 0x30, 0x34, 0x30, 0x35, @@ -133050,7 +133085,7 @@ error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept { } error_code e = simdjson::SUCCESS; template for (constexpr auto mem : std::define_static_array(std::meta::nonstatic_data_members_of(^^T, std::meta::access_context::unchecked()))) { - if constexpr (!std::meta::is_const(mem) && std::meta::is_public(mem)) { + if constexpr (!std::meta::is_const(mem) && std::meta::is_public(mem)) { constexpr std::string_view key = std::define_static_string(std::meta::identifier_of(mem)); // Note: removed static assert as optional types are now handled generically // as long we are succesful or the field is not found, we continue @@ -139427,10 +139462,12 @@ simdjson_inline void string_builder::clear() noexcept { namespace internal { -// We could specialize further for 32-bit integers. -simdjson_really_inline int int_log2(uint32_t x) { return (63 - leading_zeroes(x | 1)); } -simdjson_really_inline int fast_digit_count(uint32_t x) { +template ::value>::type> +simdjson_really_inline int int_log2(number_type x) { return 63 - leading_zeroes(uint64_t(x) | 1); } + +simdjson_really_inline int fast_digit_count_32(uint32_t x) { static uint64_t table[] = { 4294967296, 8589934582, 8589934582, 8589934582, 12884901788, 12884901788, 12884901788, 17179868184, 17179868184, 17179868184, @@ -139442,9 +139479,8 @@ simdjson_really_inline int fast_digit_count(uint32_t x) { return uint32_t((x + table[int_log2(x)]) >> 32); } -simdjson_really_inline int int_log2(uint64_t x) { return 63 - leading_zeroes(x | 1); } -simdjson_really_inline int fast_digit_count(uint64_t x) { +simdjson_really_inline int fast_digit_count_64(uint64_t x) { static uint64_t table[] = {9, 99, 999, @@ -139475,7 +139511,11 @@ simdjson_really_inline size_t digit_count(number_type v) noexcept { static_assert(sizeof(number_type) == 8 || sizeof(number_type) == 4 || sizeof(number_type) == 2 || sizeof(number_type) == 1, "We only support 8-bit, 16-bit, 32-bit and 64-bit numbers"); - return fast_digit_count(v); + SIMDJSON_IF_CONSTEXPR(sizeof(number_type) <= 4) { + return fast_digit_count_32(static_cast(v)); + } else { + return fast_digit_count_64(static_cast(v)); + } } static const char decimal_table[200] = { 0x30, 0x30, 0x30, 0x31, 0x30, 0x32, 0x30, 0x33, 0x30, 0x34, 0x30, 0x35, @@ -140244,6 +140284,11 @@ struct to_adaptor { T operator()(simdjson_result &val) const noexcept; auto operator()(padded_string_view const str) const noexcept; auto operator()(ondemand::parser &parser, padded_string_view const str) const noexcept; + // The std::string is padded with reserve to ensure there is enough space for padding. + // Some sanitizers may not like this, so you can use simdjson::pad instead. + // simdjson::from(simdjson::pad(str)) + auto operator()(std::string str) const noexcept; + auto operator()(ondemand::parser &parser, std::string str) const noexcept; }; } // namespace internal } // namespace convert @@ -140402,6 +140447,16 @@ template inline auto to_adaptor::operator()(ondemand::parser &parser, padded_string_view const str) const noexcept { return auto_parser{parser, str}; } + +template +inline auto to_adaptor::operator()(std::string str) const noexcept { + return auto_parser{pad_with_reserve(str)}; +} + +template +inline auto to_adaptor::operator()(ondemand::parser &parser, std::string str) const noexcept { + return auto_parser{parser, pad_with_reserve(str)}; +} } // namespace internal } // namespace convert } // namespace simdjson diff --git a/singleheader/singleheader.zip b/singleheader/singleheader.zip index e2bbc04ec76d08bb32e4348388abcea7eeed7a06..f1d9f0f419d6b43de10ba5c36a2b60b6532f8326 100644 GIT binary patch delta 2522 zcmd7UUrbw790%}0VU+TxKpB4r943s`cKt(vK?m|rRKy7s*70Xj3cb*>wzu@&vN3FA z$xg05c!-+$iy^_GJeZ(Um%x26>eR$V$C5p{U_zM2_~JC0B|K=h`74`3)Wz2}`84;> z`JMc}_vW6Q+rMln;NF%ZtE)qiki94Rv%hH?ymYSmr&qFSN~(#a&Q^8UR#`_5nQP2u zBbnktf|n1FDx=wGh2^EPinENeILhKFQ&5&bSt4b7DNCYENm(*wD#}tQOQkH0vUJMS zlx0wsNm&+U8p^UM+ecXrWx15)QI=0x0cC}h?We4WvSP|gC?k}WQg(o{M^;$b=WnNX zV}mR6=Z-EG^zT_LfT5)_+iuA~U4k05lAv3a|N@PVAV3Asad( zFRa(MR)DU=nJ$|z1X@3S!BeNl~X$15gZ(mCoR5X zTDaV|w^C}dG@};OiXNAnENw4qb|sP2?l$xUdQw)qpDO$@wnC7rh6X7((~h1-9q1W3 zICHchxI4s}>RD+`)rq>~HC4Crhu8{Xo=>g^4ABpFC%Tg0W^bn+t`GRqVbju_2*wMk zNpSg{ZavI;M%$yODN;kJ9rd7IbWCn2br`00CthB4NXx6oQJ=iL+8=9)GhRx2ka$)Y zNP-JLJ9Z~F8J$;H=(OcT3P&pDjZoMRZd!mSor`Y zTt$!{1rU#`O7Oke?e-;H1kof4;esgK`aQP3x0_o_YfiI z8s*3sAKI?NDU7PSL zB78Su)4hM|>+?q4MzYl~ogC6bam zgAF*NPs;h=No{u%0ar&`UbM9JVWrxfhg5dgCg`J0Tv7hDJ%f9zk;;GWg&O6Eaj*K* I2yWy41a?@KEC2ui delta 2153 zcmd7UQA|^36u@yyOGl+EtfJuR>@C88&6e9si-=mOQ?^i=&=K(E%n*d(`+ze<1lmKmjm4I7jvFheGrtAhkkIamu%{^<< zW(rJIIsR{>DsNO=o~qv1P@q(P>~xlkg?rT<8jYgL_p8~9tcKOHI`$HKnQdiu=3w%a}>~>hZY0H-j_@i*Lro!>x;7M8(sZL`oizCX-3IvkGMr&o3Fl* zZXWiP>uN{pc8HcvUso;7b?X%>4H=4PsnS;H(0m0b+spNW5(y-RLAFg*;}kn zjL!!q!nqi=y{DfIu(w5f&(Vhu@^dJYU-wyQV8&!w@$PuSgQ_Py#DZ*Cc*4i%S}u(~ zJ3VAnt*ysdh`l4Mt>OH8cb*;&mj>Od4+l>mqIv=+Sd_(tClJ?n=K|I9cjN3`mJsLf zp1l5Re$I}@|420D=0<$x?Nu)>CRvJ&uv6mV;^;zeu8rp2OX<|`7NoZ__sC_m1@T<*?Vo1>(gt+g;@IDHfouOnyK^y sO@;X7Qo68=x|(z~)cIZd`=Y0R3$*1%T3P*R9=wru=>NKs=D&)60U%=_=>Px# diff --git a/tests/ondemand/ondemand_convert_tests.cpp b/tests/ondemand/ondemand_convert_tests.cpp index ab3bcb4d6..3a5a0fea1 100644 --- a/tests/ondemand/ondemand_convert_tests.cpp +++ b/tests/ondemand/ondemand_convert_tests.cpp @@ -2,6 +2,7 @@ #include "simdjson/convert.h" #include "test_ondemand.h" +#include #include #include #include @@ -21,6 +22,24 @@ private: std::string date_str; }; + +struct Meeting { + std::string title; + std::vector attendees; + std::optional location; + bool is_recurring; +}; + + +struct MeetingTime { + std::string title; + std::chrono::system_clock::time_point start_time; + std::vector attendees; + std::optional location; + bool is_recurring; +}; + + namespace simdjson { template auto tag_invoke(deserialize_tag, simdjson_value &val, MyDate& date) { @@ -160,6 +179,38 @@ bool simple() { ASSERT_SUCCESS(doc.get(p)); TEST_SUCCEED(); } + + + bool meeting_test() { + TEST_START(); + std::string json = simdjson::to_json(Meeting{ + .title = "CppCon Planning", + .attendees = {"Alice", "Bob", "Charlie"}, + .location = "Denver", + .is_recurring = true + }); + Meeting m2 = simdjson::from(json); + std::cout << m2.title << std::endl; + TEST_SUCCEED(); + } + bool meeting_time_test() { + TEST_START(); + auto start_time = std::chrono::system_clock::now(); + std::string json = simdjson::to_json(MeetingTime{ + .title = "CppCon Planning", + .start_time = start_time, + .attendees = {"Alice", "Bob", "Charlie"}, + .location = "Denver", + .is_recurring = true + }); + std::cout << json << std::endl; + MeetingTime m2 = simdjson::from(json); + //ASSERT_EQUAL(m2.start_time, start_time); + std::cout << m2.title << std::endl; + TEST_SUCCEED(); + } + + #endif bool broken() { TEST_START(); @@ -415,7 +466,7 @@ bool test_to_vs_from_equivalence() { bool run() { return #if SIMDJSON_STATIC_REFLECTION - bad_player() && good_player() && complicated_weather_test() && + meeting_time_test() && meeting_test() && bad_player() && good_player() && complicated_weather_test() && #endif #if SIMDJSON_EXCEPTIONS && SIMDJSON_SUPPORTS_CONCEPTS broken() && simple() && simple_optional() && with_parser() && to_array() &&