From 9370abb17bcd4043e020a4f9ca20c0dc0a4d64aa Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Thu, 6 Mar 2025 16:18:10 -0500 Subject: [PATCH] tuning --- include/simdjson/arm64/stringparsing_defs.h | 36 ++ include/simdjson/compiler_check.h | 8 + .../simdjson/fallback/stringparsing_defs.h | 14 + .../simdjson/generic/ondemand/json_builder.h | 9 +- .../ondemand/json_string_builder-inl.h | 326 ++++++++++++++---- include/simdjson/haswell/stringparsing_defs.h | 36 ++ include/simdjson/icelake/stringparsing_defs.h | 37 ++ include/simdjson/lasx/stringparsing_defs.h | 36 ++ include/simdjson/lsx/stringparsing_defs.h | 37 ++ include/simdjson/ppc64/stringparsing_defs.h | 37 ++ .../simdjson/westmere/stringparsing_defs.h | 37 ++ src/generic/stage2/stringparsing.h | 8 +- 12 files changed, 558 insertions(+), 63 deletions(-) diff --git a/include/simdjson/arm64/stringparsing_defs.h b/include/simdjson/arm64/stringparsing_defs.h index 72b8c8f08..868a74ef6 100644 --- a/include/simdjson/arm64/stringparsing_defs.h +++ b/include/simdjson/arm64/stringparsing_defs.h @@ -18,6 +18,10 @@ struct backslash_and_quote { public: static constexpr uint32_t BYTES_PROCESSED = 32; simdjson_inline static backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); + ///////////// + /// TODO: This function is not used in the codebase. It is not clear if it is needed. + ///////////// + simdjson_inline static bool requires_escaping(const uint8_t *src, size_t len); simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } simdjson_inline bool has_backslash() { return bs_bits != 0; } @@ -28,6 +32,38 @@ public: uint32_t quote_bits; }; // struct backslash_and_quote +simdjson_inline bool backslash_and_quote::requires_escaping(const uint8_t *src, size_t len) { + // For short strings, we use a scalar approach + if(len < BYTES_PROCESSED) { + bool requires_escaping = false; + for(size_t i = 0; i < len; i++) { + uint8_t c = src[i]; + requires_escaping |= (c == '\\' || c == '"' || c < 32); + } + return requires_escaping; + } + // We use SIMD: + simd8 requires_escaping{}; + size_t j = 0; + for(; j + BYTES_PROCESSED <= len; j += BYTES_PROCESSED) { + simd8 v(src + j); + simd8 is_quote = (v == '"'); + simd8 is_backslash = (v == '\\'); + simd8 is_control = (v < 32); + requires_escaping |= is_backslash | is_quote | is_control; + } + if(j < len) { + // We virtually backtrack so we can load a full vector register + j = lenn - BYTES_PROCESSED; + simd8 v(src + j); + simd8 is_quote = (v == '"'); + simd8 is_backslash = (v == '\\'); + simd8 is_control = (v < 32); + requires_escaping |= is_backslash | is_quote | is_control; + } + return requires_escaping.any(); +} + simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { // this can read up to 31 bytes beyond the buffer size, but we require // SIMDJSON_PADDING of padding diff --git a/include/simdjson/compiler_check.h b/include/simdjson/compiler_check.h index 4f46948d9..8ce7ab697 100644 --- a/include/simdjson/compiler_check.h +++ b/include/simdjson/compiler_check.h @@ -93,4 +93,12 @@ #define SIMDJSON_SUPPORTS_DESERIALIZATION 0 #endif // defined(__cpp_concepts) && !defined(SIMDJSON_CONCEPT_DISABLED) +#if !defined(SIMDJSON_CONSTEVAL) +#if defined(__cpp_consteval) && __cpp_consteval >= 201811L +#define SIMDJSON_CONSTEVAL 1 +#else +#define SIMDJSON_CONSTEVAL 0 +#endif // defined(__cpp_consteval) && __cpp_consteval >= 201811L +#endif // !defined(SIMDJSON_CONSTEVAL) + #endif // SIMDJSON_COMPILER_CHECK_H diff --git a/include/simdjson/fallback/stringparsing_defs.h b/include/simdjson/fallback/stringparsing_defs.h index a7dedeb02..732abe0a9 100644 --- a/include/simdjson/fallback/stringparsing_defs.h +++ b/include/simdjson/fallback/stringparsing_defs.h @@ -14,6 +14,10 @@ struct backslash_and_quote { public: static constexpr uint32_t BYTES_PROCESSED = 1; simdjson_inline static backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); + ///////////// + /// TODO: This function is not used in the codebase. It is not clear if it is needed. + ///////////// + simdjson_inline static bool requires_escaping(const uint8_t *src, size_t len); simdjson_inline bool has_quote_first() { return c == '"'; } simdjson_inline bool has_backslash() { return c == '\\'; } @@ -23,6 +27,16 @@ public: uint8_t c; }; // struct backslash_and_quote + +simdjson_inline bool backslash_and_quote::requires_escaping(const uint8_t *src, size_t len) { + bool requires_escaping = false; + for(size_t i = 0; i < len; i++) { + uint8_t c = src[i]; + requires_escaping |= (c == '\\' || c == '"' || c < 32); + } + return requires_escaping; +} + simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { // store to dest unconditionally - we can overwrite the bits we don't like later dst[0] = src[0]; diff --git a/include/simdjson/generic/ondemand/json_builder.h b/include/simdjson/generic/ondemand/json_builder.h index 994ae15e2..a50291c1c 100644 --- a/include/simdjson/generic/ondemand/json_builder.h +++ b/include/simdjson/generic/ondemand/json_builder.h @@ -64,6 +64,9 @@ template requires(std::is_class_v && !container_but_not_string && @@ -75,7 +78,8 @@ constexpr void atom(string_builder &b, const T &t) { [:expand(std::meta::nonstatic_data_members_of(^T)):] >> [&] { if (i != 0) b.append(','); - b.escape_and_append_with_quotes(std::meta::identifier_of(dm)); + constexpr auto key = consteval_to_quoted_escaped(std::meta::identifier_of(dm)); + b.append_raw(key); b.append(':'); atom(b, t.[:dm:]); i++; @@ -90,7 +94,8 @@ template void append(string_builder &b, const Z &z) { [:expand(std::meta::nonstatic_data_members_of(^Z)):] >> [&] { if (i != 0) b.append(','); - b.escape_and_append_with_quotes(std::meta::identifier_of(dm)); + constexpr auto key = consteval_to_quoted_escaped(std::meta::identifier_of(dm)); + b.append_raw(key); b.append(':'); atom(b, z.[:dm:]); i++; diff --git a/include/simdjson/generic/ondemand/json_string_builder-inl.h b/include/simdjson/generic/ondemand/json_string_builder-inl.h index 40ffe2ade..3f3dd7708 100644 --- a/include/simdjson/generic/ondemand/json_string_builder-inl.h +++ b/include/simdjson/generic/ondemand/json_string_builder-inl.h @@ -1,7 +1,8 @@ /** - * This file is part of the builder API. It is temporarily in the ondemand directory - * but we will move it to a builder directory later. + * This file is part of the builder API. It is temporarily in the ondemand + * directory but we will move it to a builder directory later. */ +#include #include #ifndef SIMDJSON_GENERIC_STRING_BUILDER_INL_H @@ -10,22 +11,215 @@ #include "simdjson/generic/builder/json_string_builder.h" #endif // SIMDJSON_CONDITIONAL_INCLUDE +/* + * Empirically, we have found that an inlined optimization is important for + * performance. The following macros are not ideal. We should find a better + * way to inline the code. + */ + +#if defined(__SSE2__) || defined(__x86_64__) || defined(__x86_64) || \ + (defined(_M_AMD64) || defined(_M_X64) || \ + (defined(_M_IX86_FP) && _M_IX86_FP == 2)) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_SSE2 +#define SIMDJSON_EXPERIMENTAL_HAS_SSE2 1 +#endif +#endif + +#if defined(__aarch64__) || defined(_M_ARM64) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_NEON +#define SIMDJSON_EXPERIMENTAL_HAS_NEON 1 +#endif +#endif +#if SIMDJSON_EXPERIMENTAL_HAS_NEON +#include +#endif +#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 +#include +#endif + namespace simdjson { namespace SIMDJSON_IMPLEMENTATION { namespace builder { -simdjson_inline string_builder::string_builder(size_t initial_capacity) : - buffer(new (std::nothrow) char[initial_capacity]), - position(0), capacity(buffer.get() != nullptr ? initial_capacity : 0), - is_valid(buffer.get() != nullptr) {} +SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline bool +simple_needs_escaping(std::string_view v) { + for (char c : v) { + if ((uint8_t(c) < 32) | (c == '"') | (c == '\\')) { + return true; + } + } + return false; +} + +#if SIMDJSON_EXPERIMENTAL_HAS_NEON +simdjson_inline bool fast_needs_escaping(std::string_view view) { + if (view.size() < 16) { + return simple_needs_escaping(view); + } + size_t i = 0; + uint8x16_t running = vdupq_n_u8(0); + uint8x16_t v34 = vdupq_n_u8(34); + uint8x16_t v92 = vdupq_n_u8(92); + + for (; i + 15 < view.size(); i += 16) { + uint8x16_t word = vld1q_u8((const uint8_t *)view.data() + i); + running = vorrq_u8(running, vceqq_u8(word, v34)); + running = vorrq_u8(running, vceqq_u8(word, v92)); + running = vorrq_u8(running, vcltq_u8(word, vdupq_n_u8(32))); + } + if (i < view.size()) { + uint8x16_t word = + vld1q_u8((const uint8_t *)view.data() + view.length() - 16); + running = vorrq_u8(running, vceqq_u8(word, v34)); + running = vorrq_u8(running, vceqq_u8(word, v92)); + running = vorrq_u8(running, vcltq_u8(word, vdupq_n_u8(32))); + } + return vmaxvq_u32(vreinterpretq_u32_u8(running)) != 0; +} +#elif SIMDJSON_EXPERIMENTAL_HAS_SSE2 +simdjson_inline bool fast_needs_escaping(std::string_view view) { + if (view.size() < 16) { + return simple_needs_escaping(view); + } + size_t i = 0; + __m128i running = _mm_setzero_si128(); + for (; i + 15 < view.size(); i += 16) { + __m128i word = _mm_loadu_si128((const __m128i *)(view.data() + i)); + running = _mm_or_si128(running, _mm_cmpeq_epi8(word, _mm_set1_epi8(34))); + running = _mm_or_si128(running, _mm_cmpeq_epi8(word, _mm_set1_epi8(92))); + running = _mm_or_si128( + running, _mm_cmpeq_epi8(_mm_subs_epu8(word, _mm_set1_epi8(31)), + _mm_setzero_si128())); + } + if (i < view.size()) { + __m128i word = + _mm_loadu_si128((const __m128i *)(view.data() + view.length() - 16)); + running = _mm_or_si128(running, _mm_cmpeq_epi8(word, _mm_set1_epi8(34))); + running = _mm_or_si128(running, _mm_cmpeq_epi8(word, _mm_set1_epi8(92))); + running = _mm_or_si128( + running, _mm_cmpeq_epi8(_mm_subs_epu8(word, _mm_set1_epi8(31)), + _mm_setzero_si128())); + } + return _mm_movemask_epi8(running) != 0; +} +#else +simdjson_inline bool fast_needs_escaping(std::string_view view) { + return simple_needs_escaping(view); +} +#endif + +static constexpr std::array json_quotable_character = { + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + +SIMDJSON_CONSTEXPR_LAMBDA inline size_t +find_next_json_quotable_character(const std::string_view view, + size_t location) noexcept { + + for (auto pos = view.begin() + location; pos != view.end(); ++pos) { + if (json_quotable_character[(uint8_t)*pos]) { + return pos - view.begin(); + } + } + return size_t(view.size()); +} + +SIMDJSON_CONSTEXPR_LAMBDA static std::string_view control_chars[] = { + "\\x0000", "\\x0001", "\\x0002", "\\x0003", "\\x0004", "\\x0005", "\\x0006", + "\\x0007", "\\x0008", "\\t", "\\n", "\\x000b", "\\f", "\\r", + "\\x000e", "\\x000f", "\\x0010", "\\x0011", "\\x0012", "\\x0013", "\\x0014", + "\\x0015", "\\x0016", "\\x0017", "\\x0018", "\\x0019", "\\x001a", "\\x001b", + "\\x001c", "\\x001d", "\\x001e", "\\x001f"}; + +SIMDJSON_CONSTEXPR_LAMBDA void escape_json_char(char c, char *&out) { + if (c == '"') { + memcpy(out, "\\\"", 2); + out += 2; + } else if (c == '\\') { + memcpy(out, "\\\\", 2); + out += 2; + } else { + std::string_view v = control_chars[uint8_t(c)]; + memcpy(out, v.data(), v.size()); + out += v.size(); + } +} + +SIMDJSON_CONSTEXPR_LAMBDA inline size_t +write_string_escaped(const std::string_view input, char *out) { + size_t mysize = input.size(); + if (!fast_needs_escaping(input)) { // fast path! + memcpy(out, input.data(), input.size()); + return input.size(); + } + const char *const initout = out; + size_t location = find_next_json_quotable_character(input, 0); + memcpy(out, input.data(), location); + out += location; + escape_json_char(input[location], out); + location += 1; + while (location < mysize) { + size_t newlocation = find_next_json_quotable_character(input, location); + memcpy(out, input.data() + location, newlocation - location); + out += newlocation - location; + location = newlocation; + if (location == mysize) { + break; + } + escape_json_char(input[location], out); + location += 1; + } + return out - initout; +} + +#if SIMDJSON_CONSTEVAL +// unoptimized, meant for compile-time execution +consteval std::string consteval_to_quoted_escaped(std::string_view input) { + std::string out = "\""; + for (char c : input) { + if (json_quotable_character[uint8_t(c)]) { + if (c == '"') { + out.append("\\\""); + } else if (c == '\\') { + out.append("\\\\"); + } else { + std::string_view v = control_chars[uint8_t(c)]; + out.append(v); + } + } else { + out.push_back(c); + } + } + out.push_back('"'); + return out; +} +#endif // SIMDJSON_CONSTEVAL + +simdjson_inline string_builder::string_builder(size_t initial_capacity) + : buffer(new(std::nothrow) char[initial_capacity]), position(0), + capacity(buffer.get() != nullptr ? initial_capacity : 0), + is_valid(buffer.get() != nullptr) {} simdjson_inline bool string_builder::capacity_check(size_t upcoming_bytes) { // We use the convention that when is_valid is false, then the capacity and // the position are 0. // Most of the time, this function will return true. - if (simdjson_likely(upcoming_bytes <= capacity - position)) { return true; } + if (simdjson_likely(upcoming_bytes <= capacity - position)) { + return true; + } // check for overflow, most of the time there is no overflow - if (simdjson_likely(position + upcoming_bytes < position)) { return false; } + if (simdjson_likely(position + upcoming_bytes < position)) { + return false; + } // We will rarely get here. grow_buffer((std::max)(capacity * 2, position + upcoming_bytes)); // If the buffer allocation failed, we set is_valid to false. @@ -33,7 +227,9 @@ simdjson_inline bool string_builder::capacity_check(size_t upcoming_bytes) { } simdjson_inline void string_builder::grow_buffer(size_t desired_capacity) { - if (!is_valid) { return; } + if (!is_valid) { + return; + } std::unique_ptr new_buffer(new (std::nothrow) char[desired_capacity]); if (new_buffer.get() == nullptr) { set_valid(false); @@ -45,7 +241,7 @@ simdjson_inline void string_builder::grow_buffer(size_t desired_capacity) { } simdjson_inline void string_builder::set_valid(bool valid) noexcept { - if(!valid) { + if (!valid) { is_valid = false; capacity = 0; position = 0; @@ -59,27 +255,25 @@ simdjson_inline size_t string_builder::size() const noexcept { return position; } - simdjson_inline void string_builder::append(char c) noexcept { - if(capacity_check(1)) { - buffer.get()[position++] = c; - } + if (capacity_check(1)) { + buffer.get()[position++] = c; + } } - simdjson_inline void string_builder::append_null() noexcept { constexpr char null_literal[] = "null"; constexpr size_t null_len = sizeof(null_literal) - 1; - if(capacity_check(null_len)) { + if (capacity_check(null_len)) { std::memcpy(buffer.get() + position, null_literal, null_len); position += null_len; } } -simdjson_inline void string_builder::clear() noexcept { +simdjson_inline void string_builder::clear() noexcept { position = 0; // if it was invalid, we should try to repair it - if(!is_valid) { + if (!is_valid) { capacity = 0; buffer.reset(); is_valid = true; @@ -130,123 +324,133 @@ int fast_digit_count(uint64_t x) { return y + 1; } -template::value>::type> +template ::value>::type> simdjson_inline size_t digit_count(number_type v) noexcept { - static_assert(sizeof(number_type) == 8 - || sizeof(number_type) == 4 - || sizeof(number_type) == 2 - || sizeof(number_type) == 1, "We only support 8-bit, 16-bit, 32-bit and 64-bit numbers"); + static_assert(sizeof(number_type) == 8 || sizeof(number_type) == 4 || + sizeof(number_type) == 2 || sizeof(number_type) == 1, + "We only support 8-bit, 16-bit, 32-bit and 64-bit numbers"); return fast_digit_count(v); } -} // internal +} // namespace internal -template +template simdjson_inline void string_builder::append(number_type v) noexcept { - static_assert(std::is_same::value - || std::is_integral::value || std::is_floating_point::value, "Unsupported number type"); + static_assert(std::is_same::value || + std::is_integral::value || + std::is_floating_point::value, + "Unsupported number type"); // If C++17 is available, we can 'if constexpr' here. - SIMDJSON_IF_CONSTEXPR (std::is_same::value) { + SIMDJSON_IF_CONSTEXPR(std::is_same::value) { if (v) { constexpr char true_literal[] = "true"; constexpr size_t true_len = sizeof(true_literal) - 1; - if(capacity_check(true_len)) { + if (capacity_check(true_len)) { std::memcpy(buffer.get() + position, true_literal, true_len); position += true_len; } } else { constexpr char false_literal[] = "false"; constexpr size_t false_len = sizeof(false_literal) - 1; - if(capacity_check(false_len)) { + if (capacity_check(false_len)) { std::memcpy(buffer.get() + position, false_literal, false_len); position += false_len; } } - } else SIMDJSON_IF_CONSTEXPR (std::is_unsigned::value) { + } + else SIMDJSON_IF_CONSTEXPR(std::is_unsigned::value) { constexpr size_t max_number_size = 20; - if(capacity_check(max_number_size)) { + if (capacity_check(max_number_size)) { using unsigned_type = typename std::make_unsigned::type; unsigned_type pv = static_cast(v); size_t dc = internal::digit_count(pv); char *write_pointer = buffer.get() + position + dc - 1; // optimization opportunity: if v is large, we can do better. - while(pv >= 10) { + while (pv >= 10) { *write_pointer-- = char('0' + (pv % 10)); pv /= 10; } *write_pointer = char('0' + pv); position += dc; } - } else SIMDJSON_IF_CONSTEXPR (std::is_integral::value) { + } + else SIMDJSON_IF_CONSTEXPR(std::is_integral::value) { constexpr size_t max_number_size = 20; - if(capacity_check(max_number_size)) { + if (capacity_check(max_number_size)) { using unsigned_type = typename std::make_unsigned::type; bool negative = v < 0; unsigned_type pv = static_cast(negative ? -v : v); size_t dc = internal::digit_count(pv); - if(negative) { + if (negative) { buffer.get()[position++] = '-'; } char *write_pointer = buffer.get() + position + dc - 1; // optimization opportunity: if v is large, we can do better. - while(pv >= 10) { + while (pv >= 10) { *write_pointer-- = char('0' + (pv % 10)); pv /= 10; } *write_pointer = char('0' + pv); position += dc; } - } else SIMDJSON_IF_CONSTEXPR (std::is_floating_point::value) { + } + else SIMDJSON_IF_CONSTEXPR(std::is_floating_point::value) { constexpr size_t max_number_size = 24; - if(capacity_check(max_number_size)) { + if (capacity_check(max_number_size)) { // We could specialize for float. - char *end = simdjson::internal::to_chars(buffer.get() + position, nullptr, double(v)); + char *end = simdjson::internal::to_chars(buffer.get() + position, nullptr, + double(v)); position = end - buffer.get(); } } } -simdjson_inline void string_builder::escape_and_append(std::string_view input) noexcept { +simdjson_inline void +string_builder::escape_and_append(std::string_view input) noexcept { // escaping might turn a control character into \x00xx so 6 characters. - if(capacity_check(6 * input.size())) { - position += simdjson::write_string_escaped(input, buffer.get() + position); + if (capacity_check(6 * input.size())) { + position += write_string_escaped(input, buffer.get() + position); } } -simdjson_inline void string_builder::escape_and_append_with_quotes(std::string_view input) noexcept { +simdjson_inline void +string_builder::escape_and_append_with_quotes(std::string_view input) noexcept { // escaping might turn a control character into \x00xx so 6 characters. - if(capacity_check(2 + 6 * input.size())) { + if (capacity_check(2 + 6 * input.size())) { buffer.get()[position++] = '"'; - position += simdjson::write_string_escaped(input, buffer.get() + position); + position += write_string_escaped(input, buffer.get() + position); buffer.get()[position++] = '"'; } } -simdjson_inline void string_builder::escape_and_append_with_quotes(char input) noexcept { +simdjson_inline void +string_builder::escape_and_append_with_quotes(char input) noexcept { // escaping might turn a control character into \x00xx so 6 characters. - if(capacity_check(2 + 6 * 1)) { + if (capacity_check(2 + 6 * 1)) { buffer.get()[position++] = '"'; std::string_view cinput(&input, 1); - position += simdjson::write_string_escaped(cinput, buffer.get() + position); + position += write_string_escaped(cinput, buffer.get() + position); buffer.get()[position++] = '"'; } } -simdjson_inline void string_builder::append_raw(const char *c) noexcept { +simdjson_inline void string_builder::append_raw(const char *c) noexcept { size_t len = std::strlen(c); append_raw(c, len); } -simdjson_inline void string_builder::append_raw(std::string_view input) noexcept { - if(capacity_check(input.size())) { +simdjson_inline void +string_builder::append_raw(std::string_view input) noexcept { + if (capacity_check(input.size())) { std::memcpy(buffer.get() + position, input.data(), input.size()); position += input.size(); } } -simdjson_inline void string_builder::append_raw(const char *str, size_t len) noexcept { - if(capacity_check(len)) { +simdjson_inline void string_builder::append_raw(const char *str, + size_t len) noexcept { + if (capacity_check(len)) { std::memcpy(buffer.get() + position, str, len); position += len; } @@ -257,18 +461,22 @@ simdjson_inline string_builder::operator std::string() const noexcept(false) { return std::string(std::string_view()); } -simdjson_inline string_builder::operator std::string_view() const noexcept(false) { +simdjson_inline string_builder::operator std::string_view() const + noexcept(false) { return view(); } #endif -simdjson_inline simdjson_result string_builder::view() const noexcept { - if (!is_valid) { return simdjson::OUT_OF_CAPACITY; } +simdjson_inline simdjson_result +string_builder::view() const noexcept { + if (!is_valid) { + return simdjson::OUT_OF_CAPACITY; + } return std::string_view(buffer.get(), position); } simdjson_inline simdjson_result string_builder::c_str() noexcept { - if(capacity_check(1)) { + if (capacity_check(1)) { buffer.get()[position] = '\0'; return buffer.get(); } diff --git a/include/simdjson/haswell/stringparsing_defs.h b/include/simdjson/haswell/stringparsing_defs.h index e8ce6e9fe..7c3d25e76 100644 --- a/include/simdjson/haswell/stringparsing_defs.h +++ b/include/simdjson/haswell/stringparsing_defs.h @@ -18,6 +18,10 @@ struct backslash_and_quote { public: static constexpr uint32_t BYTES_PROCESSED = 32; simdjson_inline static backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); + ///////////// + /// TODO: This function is not used in the codebase. It is not clear if it is needed. + ///////////// + simdjson_inline static bool requires_escaping(const uint8_t *src, size_t len); simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } simdjson_inline bool has_backslash() { return ((quote_bits - 1) & bs_bits) != 0; } @@ -28,6 +32,38 @@ public: uint32_t quote_bits; }; // struct backslash_and_quote +simdjson_inline bool backslash_and_quote::requires_escaping(const uint8_t *src, size_t len) { + // For short strings, we use a scalar approach + if(len < BYTES_PROCESSED) { + bool requires_escaping = false; + for(size_t i = 0; i < len; i++) { + uint8_t c = src[i]; + requires_escaping |= (c == '\\' || c == '"' || c < 32); + } + return requires_escaping; + } + // We use SIMD: + simd8 requires_escaping{}; + size_t j = 0; + for(; j + BYTES_PROCESSED <= len; j += BYTES_PROCESSED) { + simd8 v(src + j); + simd8 is_quote = (v == '"'); + simd8 is_backslash = (v == '\\'); + simd8 is_control = (v < 32); + requires_escaping |= is_backslash | is_quote | is_control; + } + if(j < len) { + // We virtually backtrack so we can load a full vector register + j = len - BYTES_PROCESSED; + simd8 v(src + j); + simd8 is_quote = (v == '"'); + simd8 is_backslash = (v == '\\'); + simd8 is_control = (v < 32); + requires_escaping |= is_backslash | is_quote | is_control; + } + return requires_escaping.any(); +} + simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { // this can read up to 15 bytes beyond the buffer size, but we require // SIMDJSON_PADDING of padding diff --git a/include/simdjson/icelake/stringparsing_defs.h b/include/simdjson/icelake/stringparsing_defs.h index 13e4fef25..6077e5f0e 100644 --- a/include/simdjson/icelake/stringparsing_defs.h +++ b/include/simdjson/icelake/stringparsing_defs.h @@ -18,6 +18,10 @@ struct backslash_and_quote { public: static constexpr uint32_t BYTES_PROCESSED = 64; simdjson_inline static backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); + ///////////// + /// TODO: This function is not used in the codebase. It is not clear if it is needed. + ///////////// + simdjson_inline static bool requires_escaping(const uint8_t *src, size_t len); simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } simdjson_inline bool has_backslash() { return ((quote_bits - 1) & bs_bits) != 0; } @@ -28,6 +32,39 @@ public: uint64_t quote_bits; }; // struct backslash_and_quote + +simdjson_inline bool backslash_and_quote::requires_escaping(const uint8_t *src, size_t len) { + // For short strings, we use a scalar approach + if(len < BYTES_PROCESSED) { + bool requires_escaping = false; + for(size_t i = 0; i < len; i++) { + uint8_t c = src[i]; + requires_escaping |= (c == '\\' || c == '"' || c < 32); + } + return requires_escaping; + } + // We use SIMD: + simd8 requires_escaping{}; + size_t j = 0; + for(; j + BYTES_PROCESSED <= len; j += BYTES_PROCESSED) { + simd8 v(src + j); + simd8 is_quote = (v == '"'); + simd8 is_backslash = (v == '\\'); + simd8 is_control = (v < 32); + requires_escaping |= is_backslash | is_quote | is_control; + } + if(j < len) { + // We virtually backtrack so we can load a full vector register + j = len - BYTES_PROCESSED; + simd8 v(src + j); + simd8 is_quote = (v == '"'); + simd8 is_backslash = (v == '\\'); + simd8 is_control = (v < 32); + requires_escaping |= is_backslash | is_quote | is_control; + } + return requires_escaping.any(); +} + simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { // this can read up to 15 bytes beyond the buffer size, but we require // SIMDJSON_PADDING of padding diff --git a/include/simdjson/lasx/stringparsing_defs.h b/include/simdjson/lasx/stringparsing_defs.h index 97d109a04..c8bc1f9b6 100644 --- a/include/simdjson/lasx/stringparsing_defs.h +++ b/include/simdjson/lasx/stringparsing_defs.h @@ -18,6 +18,10 @@ struct backslash_and_quote { public: static constexpr uint32_t BYTES_PROCESSED = 32; simdjson_inline static backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); + ///////////// + /// TODO: This function is not used in the codebase. It is not clear if it is needed. + ///////////// + simdjson_inline static bool requires_escaping(const uint8_t *src, size_t len); simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } simdjson_inline bool has_backslash() { return bs_bits != 0; } @@ -28,6 +32,38 @@ public: uint32_t quote_bits; }; // struct backslash_and_quote +simdjson_inline bool backslash_and_quote::requires_escaping(const uint8_t *src, size_t len) { + // For short strings, we use a scalar approach + if(len < BYTES_PROCESSED) { + bool requires_escaping = false; + for(size_t i = 0; i < len; i++) { + uint8_t c = src[i]; + requires_escaping |= (c == '\\' || c == '"' || c < 32); + } + return requires_escaping; + } + // We use SIMD: + simd8 requires_escaping{}; + size_t j = 0; + for(; j + BYTES_PROCESSED <= len; j += BYTES_PROCESSED) { + simd8 v(src + j); + simd8 is_quote = (v == '"'); + simd8 is_backslash = (v == '\\'); + simd8 is_control = (v < 32); + requires_escaping |= is_backslash | is_quote | is_control; + } + if(j < len) { + // We virtually backtrack so we can load a full vector register + j = len - BYTES_PROCESSED; + simd8 v(src + j); + simd8 is_quote = (v == '"'); + simd8 is_backslash = (v == '\\'); + simd8 is_control = (v < 32); + requires_escaping |= is_backslash | is_quote | is_control; + } + return requires_escaping.any(); +} + simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { // this can read up to 31 bytes beyond the buffer size, but we require // SIMDJSON_PADDING of padding diff --git a/include/simdjson/lsx/stringparsing_defs.h b/include/simdjson/lsx/stringparsing_defs.h index df297332a..59edc7647 100644 --- a/include/simdjson/lsx/stringparsing_defs.h +++ b/include/simdjson/lsx/stringparsing_defs.h @@ -18,6 +18,10 @@ struct backslash_and_quote { public: static constexpr uint32_t BYTES_PROCESSED = 32; simdjson_inline static backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); + ///////////// + /// TODO: This function is not used in the codebase. It is not clear if it is needed. + ///////////// + simdjson_inline static bool requires_escaping(const uint8_t *src, size_t len); simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } simdjson_inline bool has_backslash() { return bs_bits != 0; } @@ -28,6 +32,39 @@ public: uint32_t quote_bits; }; // struct backslash_and_quote + +simdjson_inline bool backslash_and_quote::requires_escaping(const uint8_t *src, size_t len) { + // For short strings, we use a scalar approach + if(len < BYTES_PROCESSED) { + bool requires_escaping = false; + for(size_t i = 0; i < len; i++) { + uint8_t c = src[i]; + requires_escaping |= (c == '\\' || c == '"' || c < 32); + } + return requires_escaping; + } + // We use SIMD: + simd8 requires_escaping{}; + size_t j = 0; + for(; j + BYTES_PROCESSED <= len; j += BYTES_PROCESSED) { + simd8 v(src + j); + simd8 is_quote = (v == '"'); + simd8 is_backslash = (v == '\\'); + simd8 is_control = (v < 32); + requires_escaping |= is_backslash | is_quote | is_control; + } + if(j < len) { + // We virtually backtrack so we can load a full vector register + j = len - BYTES_PROCESSED; + simd8 v(src + j); + simd8 is_quote = (v == '"'); + simd8 is_backslash = (v == '\\'); + simd8 is_control = (v < 32); + requires_escaping |= is_backslash | is_quote | is_control; + } + return requires_escaping.any(); +} + simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { // this can read up to 31 bytes beyond the buffer size, but we require // SIMDJSON_PADDING of padding diff --git a/include/simdjson/ppc64/stringparsing_defs.h b/include/simdjson/ppc64/stringparsing_defs.h index 8e5b6438b..e11910395 100644 --- a/include/simdjson/ppc64/stringparsing_defs.h +++ b/include/simdjson/ppc64/stringparsing_defs.h @@ -19,6 +19,10 @@ public: static constexpr uint32_t BYTES_PROCESSED = 32; simdjson_inline static backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); + ///////////// + /// TODO: This function is not used in the codebase. It is not clear if it is needed. + ///////////// + simdjson_inline static bool requires_escaping(const uint8_t *src, size_t len); simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; @@ -35,6 +39,39 @@ public: uint32_t quote_bits; }; // struct backslash_and_quote + +simdjson_inline bool backslash_and_quote::requires_escaping(const uint8_t *src, size_t len) { + // For short strings, we use a scalar approach + if(len < BYTES_PROCESSED) { + bool requires_escaping = false; + for(size_t i = 0; i < len; i++) { + uint8_t c = src[i]; + requires_escaping |= (c == '\\' || c == '"' || c < 32); + } + return requires_escaping; + } + // We use SIMD: + simd8 requires_escaping{}; + size_t j = 0; + for(; j + BYTES_PROCESSED <= len; j += BYTES_PROCESSED) { + simd8 v(src + j); + simd8 is_quote = (v == '"'); + simd8 is_backslash = (v == '\\'); + simd8 is_control = (v < 32); + requires_escaping |= is_backslash | is_quote | is_control; + } + if(j < len) { + // We virtually backtrack so we can load a full vector register + j = len - BYTES_PROCESSED; + simd8 v(src + j); + simd8 is_quote = (v == '"'); + simd8 is_backslash = (v == '\\'); + simd8 is_control = (v < 32); + requires_escaping |= is_backslash | is_quote | is_control; + } + return requires_escaping.any(); +} + simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { // this can read up to 31 bytes beyond the buffer size, but we require diff --git a/include/simdjson/westmere/stringparsing_defs.h b/include/simdjson/westmere/stringparsing_defs.h index 1b576e7a5..3c2dffdf7 100644 --- a/include/simdjson/westmere/stringparsing_defs.h +++ b/include/simdjson/westmere/stringparsing_defs.h @@ -15,6 +15,10 @@ struct backslash_and_quote { public: static constexpr uint32_t BYTES_PROCESSED = 32; simdjson_inline static backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); + ///////////// + /// TODO: This function is not used in the codebase. It is not clear if it is needed. + ///////////// + simdjson_inline static bool requires_escaping(const uint8_t *src, size_t len); simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } simdjson_inline bool has_backslash() { return bs_bits != 0; } @@ -25,6 +29,39 @@ public: uint32_t quote_bits; }; // struct backslash_and_quote + +simdjson_inline bool backslash_and_quote::requires_escaping(const uint8_t *src, size_t len) { + // For short strings, we use a scalar approach + if(len < BYTES_PROCESSED) { + bool requires_escaping = false; + for(size_t i = 0; i < len; i++) { + uint8_t c = src[i]; + requires_escaping |= (c == '\\' || c == '"' || c < 32); + } + return requires_escaping; + } + // We use SIMD: + simd8 requires_escaping{}; + size_t j = 0; + for(; j + BYTES_PROCESSED <= len; j += BYTES_PROCESSED) { + simd8 v(src + j); + simd8 is_quote = (v == '"'); + simd8 is_backslash = (v == '\\'); + simd8 is_control = (v < 32); + requires_escaping |= is_backslash | is_quote | is_control; + } + if(j < len) { + // We virtually backtrack so we can load a full vector register + j = len - BYTES_PROCESSED; + simd8 v(src + j); + simd8 is_quote = (v == '"'); + simd8 is_backslash = (v == '\\'); + simd8 is_control = (v < 32); + requires_escaping |= is_backslash | is_quote | is_control; + } + return requires_escaping.any(); +} + simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { // this can read up to 31 bytes beyond the buffer size, but we require // SIMDJSON_PADDING of padding diff --git a/src/generic/stage2/stringparsing.h b/src/generic/stage2/stringparsing.h index 5b3ce93fd..cd229b0cb 100644 --- a/src/generic/stage2/stringparsing.h +++ b/src/generic/stage2/stringparsing.h @@ -1,3 +1,4 @@ +#include #ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H #ifndef SIMDJSON_CONDITIONAL_INCLUDE @@ -236,12 +237,15 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t } } +///////////// +/// TODO: This function is not used in the codebase. It is not clear if it is needed. +///////////// simdjson_warn_unused size_t write_string_escaped(const std::string_view input, char *out) noexcept { // We are making the following assumption: most strings will either be very short or they will not // need escaping. size_t i = 0; size_t pos = 0; - if(input.size() >= escaping::BYTES_PROCESSED) { + /*if(input.size() >= escaping::BYTES_PROCESSED) { auto vec_processing = [input,out]() -> size_t { size_t index = 0; size_t position = 0; @@ -266,7 +270,7 @@ simdjson_warn_unused size_t write_string_escaped(const std::string_view input, c pos = i; if(i == input.size()) { return pos; } // Here we only continue if there was a character that needed escaping. - } + }*/ static std::string_view control_chars[] = { "\\x0000", "\\x0001", "\\x0002", "\\x0003", "\\x0004", "\\x0005", "\\x0006", "\\x0007", "\\x0008", "\\t", "\\n", "\\x000b", "\\f", "\\r",