From db5323a9727201420aa5ee2560ad4acdacb0451d Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Mon, 25 Nov 2024 21:10:25 -0500 Subject: [PATCH] tweaking the approach --- include/simdjson/arm64/simd.h | 6 ++++ include/simdjson/arm64/stringparsing_defs.h | 25 ++++++++++++++++ .../simdjson/fallback/stringparsing_defs.h | 18 ++++++++++++ include/simdjson/haswell/stringparsing_defs.h | 25 ++++++++++++++++ include/simdjson/icelake/simd.h | 1 - include/simdjson/icelake/stringparsing_defs.h | 29 +++++++++++++++++++ include/simdjson/lasx/stringparsing_defs.h | 25 ++++++++++++++++ include/simdjson/lsx/stringparsing_defs.h | 24 +++++++++++++++ include/simdjson/ppc64/stringparsing_defs.h | 25 ++++++++++++++++ .../simdjson/westmere/stringparsing_defs.h | 25 ++++++++++++++++ src/generic/stage2/stringparsing.h | 27 ++++++++--------- 11 files changed, 214 insertions(+), 16 deletions(-) diff --git a/include/simdjson/arm64/simd.h b/include/simdjson/arm64/simd.h index 34c5117ad..945544d79 100644 --- a/include/simdjson/arm64/simd.h +++ b/include/simdjson/arm64/simd.h @@ -134,6 +134,12 @@ namespace { tmp = vpaddq_u8(tmp, tmp); return vgetq_lane_u16(vreinterpretq_u16_u8(tmp), 0); } + // Returns 4-bit out of each byte, alternating between the high 4 bits and low + // bits result it is 64 bit. + simdjson_inline uint64_t to_bitmask64() const { + return vget_lane_u64( + vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(*this), 4)), 0); + } simdjson_inline bool any() const { return vmaxvq_u32(vreinterpretq_u32_u8(*this)) != 0; } }; diff --git a/include/simdjson/arm64/stringparsing_defs.h b/include/simdjson/arm64/stringparsing_defs.h index 30d02faff..deea34444 100644 --- a/include/simdjson/arm64/stringparsing_defs.h +++ b/include/simdjson/arm64/stringparsing_defs.h @@ -46,6 +46,31 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin }; } +struct escaping { + static constexpr uint32_t BYTES_PROCESSED = 16; + simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst); + + simdjson_inline bool has_escape() { return escape_bits != 0; } + simdjson_inline int escape_index() { return trailing_zeroes(escape_bits) / 4; } + + uint64_t escape_bits; +}; // struct escaping + + + +simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) { + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v(src); + simd8 is_quote = (v == '"'); + simd8 is_backslash = (v == '\\'); + simd8 is_control = (v < 32); + return { + (is_backslash | is_quote | is_control).to_bitmask64() + }; +} + + + } // unnamed namespace } // namespace arm64 } // namespace simdjson diff --git a/include/simdjson/fallback/stringparsing_defs.h b/include/simdjson/fallback/stringparsing_defs.h index 64f23c4b0..a7dedeb02 100644 --- a/include/simdjson/fallback/stringparsing_defs.h +++ b/include/simdjson/fallback/stringparsing_defs.h @@ -29,6 +29,24 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin return { src[0] }; } + +struct escaping { + static constexpr uint32_t BYTES_PROCESSED = 1; + simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst); + + simdjson_inline bool has_escape() { return escape_bits; } + simdjson_inline int escape_index() { return 0; } + + bool escape_bits; +}; // struct escaping + + + +simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) { + dst[0] = src[0]; + return { (src[0] == '\\') || (src[0] == '"') || (src[0] < 32) }; +} + } // unnamed namespace } // namespace fallback } // namespace simdjson diff --git a/include/simdjson/haswell/stringparsing_defs.h b/include/simdjson/haswell/stringparsing_defs.h index f896a10e2..e8ce6e9fe 100644 --- a/include/simdjson/haswell/stringparsing_defs.h +++ b/include/simdjson/haswell/stringparsing_defs.h @@ -41,6 +41,31 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin }; } + +struct escaping { + static constexpr uint32_t BYTES_PROCESSED = 32; + simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst); + + simdjson_inline bool has_escape() { return escape_bits != 0; } + simdjson_inline int escape_index() { return trailing_zeroes(escape_bits); } + + uint64_t escape_bits; +}; // struct escaping + + + +simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) { + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v(src); + v.store(dst); + simd8 is_quote = (v == '"'); + simd8 is_backslash = (v == '\\'); + simd8 is_control = (v < 32); + return { + uint64_t((is_backslash | is_quote | is_control).to_bitmask()) + }; +} + } // unnamed namespace } // namespace haswell } // namespace simdjson diff --git a/include/simdjson/icelake/simd.h b/include/simdjson/icelake/simd.h index 04203f4b9..27390a907 100644 --- a/include/simdjson/icelake/simd.h +++ b/include/simdjson/icelake/simd.h @@ -77,7 +77,6 @@ namespace simd { friend simdjson_really_inline uint64_t operator==(const simd8 lhs, const simd8 rhs) { return _mm512_cmpeq_epi8_mask(lhs, rhs); } - static const int SIZE = sizeof(base::value); template diff --git a/include/simdjson/icelake/stringparsing_defs.h b/include/simdjson/icelake/stringparsing_defs.h index 4cc582737..13e4fef25 100644 --- a/include/simdjson/icelake/stringparsing_defs.h +++ b/include/simdjson/icelake/stringparsing_defs.h @@ -41,6 +41,35 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin }; } + + +struct escaping { + static constexpr uint32_t BYTES_PROCESSED = 64; + simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst); + + simdjson_inline bool has_escape() { return escape_bits != 0; } + simdjson_inline int escape_index() { return trailing_zeroes(uint64_t(escape_bits)); } + + __mmask64 escape_bits; +}; // struct escaping + + + +simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) { + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v(src); + v.store(dst); + __mmask64 is_quote = _mm512_cmpeq_epi8_mask(v, _mm512_set1_epi8('"')); + __mmask64 is_backslash = _mm512_cmpeq_epi8_mask(v, _mm512_set1_epi8('\\')); + __mmask64 is_control = _mm512_cmplt_epi8_mask(v, _mm512_set1_epi8(32)); + return { + (is_backslash | is_quote | is_control) + }; +} + + + + } // unnamed namespace } // namespace icelake } // namespace simdjson diff --git a/include/simdjson/lasx/stringparsing_defs.h b/include/simdjson/lasx/stringparsing_defs.h index fe7a7430e..97d109a04 100644 --- a/include/simdjson/lasx/stringparsing_defs.h +++ b/include/simdjson/lasx/stringparsing_defs.h @@ -40,6 +40,31 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin }; } + +struct escaping { + static constexpr uint32_t BYTES_PROCESSED = 16; + simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst); + + simdjson_inline bool has_escape() { return escape_bits != 0; } + simdjson_inline int escape_index() { return trailing_zeroes(escape_bits); } + + uint64_t escape_bits; +}; // struct escaping + + + +simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) { + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v(src); + v.store(dst); + simd8 is_quote = (v == '"'); + simd8 is_backslash = (v == '\\'); + simd8 is_control = (v < 32); + return { + (is_backslash | is_quote | is_control).to_bitmask() + }; +} + } // unnamed namespace } // namespace lasx } // namespace simdjson diff --git a/include/simdjson/lsx/stringparsing_defs.h b/include/simdjson/lsx/stringparsing_defs.h index af493dc55..a7157335e 100644 --- a/include/simdjson/lsx/stringparsing_defs.h +++ b/include/simdjson/lsx/stringparsing_defs.h @@ -46,6 +46,30 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin }; } + +struct escaping { + static constexpr uint32_t BYTES_PROCESSED = 16; + simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst); + + simdjson_inline bool has_escape() { return escape_bits != 0; } + simdjson_inline int escape_index() { return trailing_zeroes(escape_bits); } + + uint64_t escape_bits; +}; // struct escaping + + + +simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) { + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v(src); + simd8 is_quote = (v == '"'); + simd8 is_backslash = (v == '\\'); + simd8 is_control = (v < 32); + return { + (is_backslash | is_quote | is_control).to_bitmask() + }; +} + } // unnamed namespace } // namespace lsx } // namespace simdjson diff --git a/include/simdjson/ppc64/stringparsing_defs.h b/include/simdjson/ppc64/stringparsing_defs.h index 82e442435..8e5b6438b 100644 --- a/include/simdjson/ppc64/stringparsing_defs.h +++ b/include/simdjson/ppc64/stringparsing_defs.h @@ -58,6 +58,31 @@ backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { }; } + +struct escaping { + static constexpr uint32_t BYTES_PROCESSED = 16; + simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst); + + simdjson_inline bool has_escape() { return escape_bits != 0; } + simdjson_inline int escape_index() { return trailing_zeroes(escape_bits); } + + uint64_t escape_bits; +}; // struct escaping + + + +simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) { + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v(src); + v.store(dst); + simd8 is_quote = (v == '"'); + simd8 is_backslash = (v == '\\'); + simd8 is_control = (v < 32); + return { + (is_backslash | is_quote | is_control).to_bitmask() + }; +} + } // unnamed namespace } // namespace ppc64 } // namespace simdjson diff --git a/include/simdjson/westmere/stringparsing_defs.h b/include/simdjson/westmere/stringparsing_defs.h index 439f19cbc..1b576e7a5 100644 --- a/include/simdjson/westmere/stringparsing_defs.h +++ b/include/simdjson/westmere/stringparsing_defs.h @@ -40,6 +40,31 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin }; } + +struct escaping { + static constexpr uint32_t BYTES_PROCESSED = 16; + simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst); + + simdjson_inline bool has_escape() { return escape_bits != 0; } + simdjson_inline int escape_index() { return trailing_zeroes(escape_bits); } + + uint64_t escape_bits; +}; // struct escaping + + + +simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) { + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v(src); + v.store(dst); + simd8 is_quote = (v == '"'); + simd8 is_backslash = (v == '\\'); + simd8 is_control = (v < 32); + return { + uint64_t((is_backslash | is_quote | is_control).to_bitmask()) + }; +} + } // unnamed namespace } // namespace westmere } // namespace simdjson diff --git a/src/generic/stage2/stringparsing.h b/src/generic/stage2/stringparsing.h index b944c964d..cf21b482a 100644 --- a/src/generic/stage2/stringparsing.h +++ b/src/generic/stage2/stringparsing.h @@ -241,28 +241,25 @@ simdjson_warn_unused size_t write_string_escaped(const std::string_view input, c // need escaping. size_t i = 0; size_t pos = 0; - if(input.size() >= sizeof(simd8)) { + if(input.size() >= escaping::BYTES_PROCESSED) { auto vec_processing = [input,out]() -> size_t { size_t i = 0; size_t pos = 0; - for(;input.size() - i >= sizeof(simd8); i += sizeof(simd8)) { - simd8 vinput(reinterpret_cast(input.data()) + i); - // instead of doing it register by register, we could regroup, but consider - // that we expect most strings to be short. - if(((vinput <= 31) | (vinput == '\\') | (vinput == '"')).any()) { - return i; // We have a character that needs escaping - // We could be more carefully and identify the character that needs escaping. + for(;input.size() - i >= escaping::BYTES_PROCESSED; i += escaping::BYTES_PROCESSED) { + escaping vinput = escaping::copy_and_find(reinterpret_cast(input.data()) + i, reinterpret_cast(out) + pos); + if(vinput.has_escape()) { + return i + vinput.escape_index(); // We have a character that needs escaping } - vinput.store(reinterpret_cast(out) + pos); - pos += sizeof(simd8); + pos += escaping::BYTES_PROCESSED; } if(i == input.size()) { return input.size(); } - simd8 vinput(reinterpret_cast(input.data()) + input.size() - sizeof(simd8)); - if(((vinput <= 31) | (vinput == '\\') | (vinput == '"')).any()) { - return i; // We have a character that needs escaping - // We could be more carefully and identify the character that needs escaping. + // We virtually backtrack so we can load a full vector register + i = input.size() - escaping::BYTES_PROCESSED; + pos = i; + escaping vinput = escaping::copy_and_find(reinterpret_cast(input.data()) + i, reinterpret_cast(out) + pos); + if(vinput.has_escape()) { + return i + vinput.escape_index(); // We have a character that needs escaping } - vinput.store(reinterpret_cast(out) + input.size() - sizeof(simd8)); return input.size(); }; i = vec_processing();