From c6e43a631cace1ea1341fed89b7952ffc236896b Mon Sep 17 00:00:00 2001 From: John Keiser Date: Fri, 4 Aug 2023 16:11:52 -0700 Subject: [PATCH] Add byte_classifier abstraction to make lookup tables readable. --- include/simdjson/arm64.h | 1 + include/simdjson/arm64/base.h | 2 - include/simdjson/arm64/simd.h | 383 ++++++++++--------- include/simdjson/common_defs.h | 26 +- include/simdjson/generic/lookup_table.h | 296 +++++++++++++++ include/simdjson/haswell.h | 1 + include/simdjson/haswell/base.h | 2 - include/simdjson/haswell/simd.h | 337 ++++++++--------- include/simdjson/icelake.h | 1 + include/simdjson/icelake/base.h | 2 - include/simdjson/icelake/simd.h | 316 ++++++++-------- include/simdjson/ppc64.h | 1 + include/simdjson/ppc64/base.h | 2 - include/simdjson/ppc64/simd.h | 386 +++++++++++--------- include/simdjson/westmere.h | 1 + include/simdjson/westmere/base.h | 2 - include/simdjson/westmere/simd.h | 367 +++++++++++-------- src/arm64.cpp | 46 ++- src/generic/stage1/utf8_lookup4_algorithm.h | 157 ++++---- src/haswell.cpp | 25 +- src/icelake.cpp | 29 +- src/ppc64.cpp | 68 +++- src/westmere.cpp | 29 +- tests/unicode_tests.cpp | 6 +- 24 files changed, 1505 insertions(+), 981 deletions(-) create mode 100644 include/simdjson/generic/lookup_table.h diff --git a/include/simdjson/arm64.h b/include/simdjson/arm64.h index 1493c3562..da67e775e 100644 --- a/include/simdjson/arm64.h +++ b/include/simdjson/arm64.h @@ -3,6 +3,7 @@ #include "simdjson/arm64/begin.h" #include "simdjson/generic/amalgamated.h" +#include "simdjson/generic/lookup_table.h" #include "simdjson/arm64/end.h" #endif // SIMDJSON_ARM64_H \ No newline at end of file diff --git a/include/simdjson/arm64/base.h b/include/simdjson/arm64/base.h index d5fc65285..1c3fb9f2e 100644 --- a/include/simdjson/arm64/base.h +++ b/include/simdjson/arm64/base.h @@ -13,7 +13,6 @@ namespace arm64 { class implementation; -namespace { namespace simd { template struct simd8; @@ -22,7 +21,6 @@ template <> struct simd8; template struct simd8x64; } // namespace simd -} // unnamed namespace } // namespace arm64 } // namespace simdjson diff --git a/include/simdjson/arm64/simd.h b/include/simdjson/arm64/simd.h index 462039c87..98f9e5667 100644 --- a/include/simdjson/arm64/simd.h +++ b/include/simdjson/arm64/simd.h @@ -9,7 +9,6 @@ namespace simdjson { namespace arm64 { -namespace { namespace simd { #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO @@ -28,7 +27,7 @@ namespace { * You should not use this function except for compile-time constants: * it is not efficient. */ -simdjson_constexpr simd_t make_uint8x16_t(uint8_t x1, uint8_t x2, uint8_t x3, uint8_t x4, +simdjson_inline simd_t make_uint8x16_t(uint8_t x1, uint8_t x2, uint8_t x3, uint8_t x4, uint8_t x5, uint8_t x6, uint8_t x7, uint8_t x8, uint8_t x9, uint8_t x10, uint8_t x11, uint8_t x12, uint8_t x13, uint8_t x14, uint8_t x15, uint8_t x16) { @@ -57,7 +56,7 @@ simdjson_constexpr simd_t make_uint8x16_t(uint8_t x1, uint8_t x2, uint8_t x3, return x; } -simdjson_constexpr uint8x8_t make_uint8x8_t(uint8_t x1, uint8_t x2, uint8_t x3, uint8_t x4, +simdjson_inline uint8x8_t make_uint8x8_t(uint8_t x1, uint8_t x2, uint8_t x3, uint8_t x4, uint8_t x5, uint8_t x6, uint8_t x7, uint8_t x8) { uint8x8_t x{}; x = vset_lane_u8(x1, x, 0); @@ -72,7 +71,7 @@ simdjson_constexpr uint8x8_t make_uint8x8_t(uint8_t x1, uint8_t x2, uint8_t x3 } // We have to do the same work for make_int8x16_t -simdjson_constexpr int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, int8_t x4, +simdjson_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, int8_t x4, int8_t x5, int8_t x6, int8_t x7, int8_t x8, int8_t x9, int8_t x10, int8_t x11, int8_t x12, int8_t x13, int8_t x14, int8_t x15, int8_t x16) { @@ -109,6 +108,11 @@ simdjson_constexpr int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, template struct simd8; +#if !SIMDJSON_IS_ARM && !defined(SIMDJSON_CONDITIONAL_INCLUDE) + // Make errors a bit more manageable when editing on non-ARM + struct uint8x16_t { uint8_t x[16]; }; +#endif + // // Base class of simd8 and simd8, both of which use uint8x16_t internally. // @@ -118,30 +122,30 @@ simdjson_constexpr int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, using simd_t = uint8x16_t; static constexpr const int LANES = sizeof(simd_t); using bitmask_t = uint16_t; - static_assert(sizeof(bitmask_t)*8 == LANES); + static_assert(sizeof(bitmask_t)*8 == LANES, "Bitmask type's bits must equal the simd type's bytes"); simd_t value; // Conversion from/to SIMD register - simdjson_constexpr base_u8(const simd_t _value) : value(_value) {} - simdjson_constexpr operator const simd_t&() const { return this->value; } - simdjson_constexpr operator simd_t&() { return this->value; } + simdjson_inline base_u8(const simd_t _value) : value(_value) {} + simdjson_inline operator const simd_t&() const { return this->value; } + simdjson_inline operator simd_t&() { return this->value; } // Bit operations - simdjson_constexpr simd8 operator|(const simd8 other) const { return vorrq_u8(*this, other); } - simdjson_constexpr simd8 operator&(const simd8 other) const { return vandq_u8(*this, other); } - simdjson_constexpr simd8 operator^(const simd8 other) const { return veorq_u8(*this, other); } - simdjson_constexpr simd8 bit_andnot(const simd8 other) const { return vbicq_u8(*this, other); } - simdjson_constexpr simd8 operator~() const { return *this ^ 0xFFu; } - simdjson_constexpr simd8& operator|=(const simd8 other) { auto this_cast = static_cast*>(this); *this_cast = *this_cast | other; return *this_cast; } - simdjson_constexpr simd8& operator&=(const simd8 other) { auto this_cast = static_cast*>(this); *this_cast = *this_cast & other; return *this_cast; } - simdjson_constexpr simd8& operator^=(const simd8 other) { auto this_cast = static_cast*>(this); *this_cast = *this_cast ^ other; return *this_cast; } + simdjson_inline simd8 operator|(const simd8 other) const { return vorrq_u8(*this, other); } + simdjson_inline simd8 operator&(const simd8 other) const { return vandq_u8(*this, other); } + simdjson_inline simd8 operator^(const simd8 other) const { return veorq_u8(*this, other); } + simdjson_inline simd8 bit_andnot(const simd8 other) const { return vbicq_u8(*this, other); } + simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } + simdjson_inline simd8& operator|=(const simd8 other) { auto this_cast = static_cast*>(this); *this_cast = *this_cast | other; return *this_cast; } + simdjson_inline simd8& operator&=(const simd8 other) { auto this_cast = static_cast*>(this); *this_cast = *this_cast & other; return *this_cast; } + simdjson_inline simd8& operator^=(const simd8 other) { auto this_cast = static_cast*>(this); *this_cast = *this_cast ^ other; return *this_cast; } - simdjson_constexpr Mask eq(const simd8 rhs) const { return vceqq_u8(*this, rhs); } - friend simdjson_constexpr Mask operator==(const simd8 lhs, const simd8 rhs) { return lhs.eq(rhs); } + simdjson_inline Mask eq(const simd8 rhs) const { return vceqq_u8(*this, rhs); } + friend simdjson_inline Mask operator==(const simd8 lhs, const simd8 rhs) { return lhs.eq(rhs); } template - simdjson_constexpr simd8 prev(const simd8 prev_chunk) const { + simdjson_inline simd8 prev(const simd8 prev_chunk) const { return vextq_u8(prev_chunk, *this, 16 - N); } }; @@ -152,17 +156,17 @@ simdjson_constexpr int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, typedef uint16_t bitmask_t; typedef uint32_t bitmask2_t; - static simdjson_constexpr simd8 splat(bool _value) { return vmovq_n_u8(uint8_t(-(!!_value))); } + static simdjson_inline simd8 splat(bool _value) { return vmovq_n_u8(uint8_t(-(!!_value))); } - simdjson_constexpr simd8(const simd_t _value) : base_u8(_value) {} + simdjson_inline simd8(const simd_t _value) : base_u8(_value) {} // False constructor - simdjson_constexpr simd8() : simd8(vdupq_n_u8(0)) {} + simdjson_inline simd8() : simd8(vdupq_n_u8(0)) {} // Splat constructor - simdjson_constexpr simd8(bool _value) : simd8(splat(_value)) {} + simdjson_inline simd8(bool _value) : simd8(splat(_value)) {} // We return uint32_t instead of uint16_t because that seems to be more efficient for most // purposes (cutting it down to uint16_t costs performance in some compilers). - simdjson_constexpr uint32_t to_bitmask() const { + simdjson_inline uint32_t to_bitmask() const { #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO const simd_t bit_mask = make_simd_t(0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80); @@ -176,7 +180,7 @@ simdjson_constexpr int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, tmp = vpaddq_u8(tmp, tmp); return vgetq_lane_u16(vreinterpretq_u16_u8(tmp), 0); } - simdjson_constexpr bool any() const { return vmaxvq_u8(*this) != 0; } + simdjson_inline bool any() const { return vmaxvq_u8(*this) != 0; } }; // Unsigned bytes @@ -185,20 +189,20 @@ simdjson_constexpr int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, using typename base_u8::simd_t; using base_u8::LANES; - static simdjson_constexpr simd_t splat(uint8_t _value) { return vmovq_n_u8(_value); } - static simdjson_constexpr simd_t zero() { return vdupq_n_u8(0); } - static simdjson_constexpr simd_t load(const uint8_t* values) { return vld1q_u8(values); } + static simdjson_inline simd_t splat(uint8_t _value) { return vmovq_n_u8(_value); } + static simdjson_inline simd_t zero() { return vdupq_n_u8(0); } + static simdjson_inline simd_t load(const uint8_t* values) { return vld1q_u8(values); } - simdjson_constexpr simd8(const simd_t _value) : base_u8(_value) {} + simdjson_inline simd8(const simd_t _value) : base_u8(_value) {} // Zero constructor - simdjson_constexpr simd8() : simd8(zero()) {} + simdjson_inline simd8() : simd8(zero()) {} // Array constructor - simdjson_constexpr simd8(const uint8_t values[16]) : simd8(load(values)) {} + simdjson_inline simd8(const uint8_t values[16]) : simd8(load(values)) {} // Splat constructor - simdjson_constexpr simd8(uint8_t _value) : simd8(splat(_value)) {} + simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} // Member-by-member initialization #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO - simdjson_constexpr simd8( + simdjson_inline simd8( uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 ) : simd8(make_uint8x16_t( @@ -206,7 +210,7 @@ simdjson_constexpr int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, v8, v9, v10,v11,v12,v13,v14,v15 )) {} #else - simdjson_constexpr simd8( + simdjson_inline simd8( uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 ) : simd8(simd_t{ @@ -216,7 +220,7 @@ simdjson_constexpr int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, #endif // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_constexpr static simd8 repeat_16( + simdjson_inline static simd8 repeat_16( uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 ) { @@ -227,44 +231,43 @@ simdjson_constexpr int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, } // Store to array - simdjson_constexpr void store(uint8_t dst[16]) const { return vst1q_u8(dst, *this); } + simdjson_inline void store(uint8_t dst[16]) const { return vst1q_u8(dst, *this); } // Saturated math - simdjson_constexpr simd8 saturating_add(const simd8 other) const { return vqaddq_u8(*this, other); } - simdjson_constexpr simd8 saturating_sub(const simd8 other) const { return vqsubq_u8(*this, other); } + simdjson_inline simd8 saturating_add(const simd8 other) const { return vqaddq_u8(*this, other); } + simdjson_inline simd8 saturating_sub(const simd8 other) const { return vqsubq_u8(*this, other); } // Addition/subtraction are the same for signed and unsigned - simdjson_constexpr simd8 operator+(const simd8 other) const { return vaddq_u8(*this, other); } - simdjson_constexpr simd8 operator-(const simd8 other) const { return vsubq_u8(*this, other); } - simdjson_constexpr simd8& operator+=(const simd8 other) { *this = *this + other; return *this; } - simdjson_constexpr simd8& operator-=(const simd8 other) { *this = *this - other; return *this; } + simdjson_inline simd8 operator+(const simd8 other) const { return vaddq_u8(*this, other); } + simdjson_inline simd8 operator-(const simd8 other) const { return vsubq_u8(*this, other); } + simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *this; } + simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *this; } // Order-specific operations - simdjson_constexpr uint8_t max_val() const { return vmaxvq_u8(*this); } - simdjson_constexpr uint8_t min_val() const { return vminvq_u8(*this); } - simdjson_constexpr simd8 max_val(const simd8 other) const { return vmaxq_u8(*this, other); } - simdjson_constexpr simd8 min_val(const simd8 other) const { return vminq_u8(*this, other); } - simdjson_constexpr simd8 operator<=(const simd8 other) const { return vcleq_u8(*this, other); } - simdjson_constexpr simd8 operator>=(const simd8 other) const { return vcgeq_u8(*this, other); } - simdjson_constexpr simd8 operator<(const simd8 other) const { return vcltq_u8(*this, other); } - simdjson_constexpr simd8 operator>(const simd8 other) const { return vcgtq_u8(*this, other); } + simdjson_inline uint8_t max_val() const { return vmaxvq_u8(*this); } + simdjson_inline uint8_t min_val() const { return vminvq_u8(*this); } + simdjson_inline simd8 max_val(const simd8 other) const { return vmaxq_u8(*this, other); } + simdjson_inline simd8 min_val(const simd8 other) const { return vminq_u8(*this, other); } + simdjson_inline simd8 operator<=(const simd8 other) const { return vcleq_u8(*this, other); } + simdjson_inline simd8 operator>=(const simd8 other) const { return vcgeq_u8(*this, other); } + simdjson_inline simd8 operator<(const simd8 other) const { return vcltq_u8(*this, other); } + simdjson_inline simd8 operator>(const simd8 other) const { return vcgtq_u8(*this, other); } // Same as >, but instead of guaranteeing all 1's == true, false = 0 and true = nonzero. For ARM, returns all 1's. - simdjson_constexpr simd8 gt_bits(const simd8 other) const { return simd8(*this > other); } + simdjson_inline simd8 gt_bits(const simd8 other) const { return simd8(*this > other); } // Same as <, but instead of guaranteeing all 1's == true, false = 0 and true = nonzero. For ARM, returns all 1's. - simdjson_constexpr simd8 lt_bits(const simd8 other) const { return simd8(*this < other); } + simdjson_inline simd8 lt_bits(const simd8 other) const { return simd8(*this < other); } // Bit-specific operations - simdjson_constexpr simd8 any_bits_set(simd8 bits) const { return vtstq_u8(*this, bits); } - simdjson_constexpr bool any_bits_set_anywhere() const { return this->max_val() != 0; } - simdjson_constexpr bool any_bits_set_anywhere(simd8 bits) const { return (*this & bits).any_bits_set_anywhere(); } + simdjson_inline simd8 any_bits_set(simd8 bits) const { return vtstq_u8(*this, bits); } + simdjson_inline bool any_bits_set_anywhere() const { return this->max_val() != 0; } + simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return (*this & bits).any_bits_set_anywhere(); } template - simdjson_constexpr simd8 shr() const { return vshrq_n_u8(*this, N); } + simdjson_inline simd8 shr() const { return vshrq_n_u8(*this, N); } template - simdjson_constexpr simd8 shl() const { return vshlq_n_u8(*this, N); } + simdjson_inline simd8 shl() const { return vshlq_n_u8(*this, N); } // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) - template - simdjson_constexpr simd8 lookup_16(simd8 lookup_table) const { + simdjson_inline simd8 lookup_16(simd8 lookup_table) const { return lookup_table.apply_lookup_16_to(*this); } @@ -277,7 +280,7 @@ simdjson_constexpr int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, // signature simd8 compress(uint16_t mask) would be // sensible, but the AVX ISA makes this kind of approach difficult. template - simdjson_constexpr void compress(uint16_t mask, L * output) const { + simdjson_inline void compress(uint16_t mask, L * output) const { using internal::thintable_epi8; using internal::BitsSetTable256mul2; using internal::pshufb_combine_table; @@ -314,7 +317,7 @@ simdjson_constexpr int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, // Copies all bytes corresponding to a 0 in the low half of the mask (interpreted as a // bitset) to output1, then those corresponding to a 0 in the high half to output2. template - simdjson_constexpr void compress_halves(uint16_t mask, L *output1, L *output2) const { + simdjson_inline void compress_halves(uint16_t mask, L *output1, L *output2) const { using internal::thintable_epi8; uint8_t mask1 = uint8_t(mask); // least significant 8 bits uint8_t mask2 = uint8_t(mask >> 8); // most significant 8 bits @@ -332,68 +335,8 @@ simdjson_constexpr int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, vst1_u8((uint8_t*)output2, vqtbl1_u8(*this, compactmask2)); } - struct lane_with_value { int lane; T value; }; - - /** - * Initialize a simd8 by filling in only specific lanes. - * - * @param entries A set of index/value pairs, like {{1, 'a'}, {2, 'b'}, ...} - * @param default_value The value to use for other lanes. - */ - static simdjson_constexpr simd8 create_sparse( - std::initializer_list entries, - T default_value = {} - ) noexcept { - bool filled[LANES] = {0}; - uint8_t table[LANES] = {default_value}; - for (auto [lane, value] : entries) { - assert(lane < LANES); - assert(!filled[lane]); - filled[lane] = true; - table[lane] = value; - } - return table; - } - - static simdjson_constexpr simd8 create_eq_lookup_16_table(std::initializer_list values) { - bool filled[16] = {0}; - - // Set the defaults to 0, except at 0 itself (which we set to 1 so it won't accidentally match 0). - uint8_t table[LANES] = {0}; - for (int lane = 0; lane < 16; lane += 16) { table[lane] = 1; } - - for (T value : values) { - int lane = value & 0x0F; - assert(!filled[lane]); - filled[lane] = true; - // Repeat the value at the same position in each 16-byte section of lanes. - for (; lane < LANES; lane += 16) { table[lane] = value; } - } - return table; - } - - template - simdjson_inline simd8 eq_any(V ...values) const { - static constexpr const simd8 LOOKUP_TABLE = create_eq_lookup_16_table({values...}); - return eq(lookup_16(LOOKUP_TABLE)); - } - - template - simdjson_constexpr simd8 lookup_16( - L replace0, L replace1, L replace2, L replace3, - L replace4, L replace5, L replace6, L replace7, - L replace8, L replace9, L replace10, L replace11, - L replace12, L replace13, L replace14, L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, - replace4, replace5, replace6, replace7, - replace8, replace9, replace10, replace11, - replace12, replace13, replace14, replace15 - )); - } - template - simdjson_constexpr simd8 apply_lookup_16_to(const simd8 original) { + simdjson_inline simd8 apply_lookup_16_to(const simd8 original) { return vqtbl1q_u8(*this, simd8(original)); } }; @@ -403,24 +346,24 @@ simdjson_constexpr int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, struct simd8 { int8x16_t value; - static simdjson_constexpr simd8 splat(int8_t _value) { return vmovq_n_s8(_value); } - static simdjson_constexpr simd8 zero() { return vdupq_n_s8(0); } - static simdjson_constexpr simd8 load(const int8_t values[16]) { return vld1q_s8(values); } + static simdjson_inline simd8 splat(int8_t _value) { return vmovq_n_s8(_value); } + static simdjson_inline simd8 zero() { return vdupq_n_s8(0); } + static simdjson_inline simd8 load(const int8_t values[16]) { return vld1q_s8(values); } // Conversion from/to SIMD register - simdjson_constexpr simd8(const int8x16_t _value) : value{_value} {} - simdjson_constexpr operator const int8x16_t&() const { return this->value; } - simdjson_constexpr operator int8x16_t&() { return this->value; } + simdjson_inline simd8(const int8x16_t _value) : value{_value} {} + simdjson_inline operator const int8x16_t&() const { return this->value; } + simdjson_inline operator int8x16_t&() { return this->value; } // Zero constructor - simdjson_constexpr simd8() : simd8(zero()) {} + simdjson_inline simd8() : simd8(zero()) {} // Splat constructor - simdjson_constexpr simd8(int8_t _value) : simd8(splat(_value)) {} + simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} // Array constructor - simdjson_constexpr simd8(const int8_t* values) : simd8(load(values)) {} + simdjson_inline simd8(const int8_t* values) : simd8(load(values)) {} // Member-by-member initialization #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO - simdjson_constexpr simd8( + simdjson_inline simd8( int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 ) : simd8(make_int8x16_t( @@ -428,7 +371,7 @@ simdjson_constexpr int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, v8, v9, v10,v11,v12,v13,v14,v15 )) {} #else - simdjson_constexpr simd8( + simdjson_inline simd8( int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 ) : simd8(int8x16_t{ @@ -437,7 +380,7 @@ simdjson_constexpr int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, }) {} #endif // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_constexpr static simd8 repeat_16( + simdjson_inline static simd8 repeat_16( int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 ) { @@ -448,7 +391,7 @@ simdjson_constexpr int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, } // Store to array - simdjson_constexpr void store(int8_t dst[16]) const { return vst1q_s8(dst, *this); } + simdjson_inline void store(int8_t dst[16]) const { return vst1q_s8(dst, *this); } // Explicit conversion to/from unsigned // @@ -456,35 +399,34 @@ simdjson_constexpr int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, // In theory, we could check this occurrence with std::same_as and std::enabled_if but it is C++14 // and relatively ugly and hard to read. #ifndef SIMDJSON_REGULAR_VISUAL_STUDIO - simdjson_constexpr explicit simd8(const simd_t other): simd8(vreinterpretq_s8_u8(other)) {} + simdjson_inline explicit simd8(const simd_t other): simd8(vreinterpretq_s8_u8(other)) {} #endif - simdjson_constexpr explicit operator simd8() const { return vreinterpretq_u8_s8(this->value); } + simdjson_inline explicit operator simd8() const { return vreinterpretq_u8_s8(this->value); } // Math - simdjson_constexpr simd8 operator+(const simd8 other) const { return vaddq_s8(*this, other); } - simdjson_constexpr simd8 operator-(const simd8 other) const { return vsubq_s8(*this, other); } - simdjson_constexpr simd8& operator+=(const simd8 other) { *this = *this + other; return *this; } - simdjson_constexpr simd8& operator-=(const simd8 other) { *this = *this - other; return *this; } + simdjson_inline simd8 operator+(const simd8 other) const { return vaddq_s8(*this, other); } + simdjson_inline simd8 operator-(const simd8 other) const { return vsubq_s8(*this, other); } + simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *this; } + simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *this; } // Order-sensitive comparisons - simdjson_constexpr simd8 max_val(const simd8 other) const { return vmaxq_s8(*this, other); } - simdjson_constexpr simd8 min_val(const simd8 other) const { return vminq_s8(*this, other); } - simdjson_constexpr simd8 operator>(const simd8 other) const { return vcgtq_s8(*this, other); } - simdjson_constexpr simd8 operator<(const simd8 other) const { return vcltq_s8(*this, other); } - simdjson_constexpr simd8 operator==(const simd8 other) const { return vceqq_s8(*this, other); } + simdjson_inline simd8 max_val(const simd8 other) const { return vmaxq_s8(*this, other); } + simdjson_inline simd8 min_val(const simd8 other) const { return vminq_s8(*this, other); } + simdjson_inline simd8 operator>(const simd8 other) const { return vcgtq_s8(*this, other); } + simdjson_inline simd8 operator<(const simd8 other) const { return vcltq_s8(*this, other); } + simdjson_inline simd8 operator==(const simd8 other) const { return vceqq_s8(*this, other); } template - simdjson_constexpr simd8 prev(const simd8 prev_chunk) const { + simdjson_inline simd8 prev(const simd8 prev_chunk) const { return vextq_s8(prev_chunk, *this, 16 - N); } // Perform a lookup assuming no value is larger than 16 - template - simdjson_constexpr simd8 lookup_16(simd8 lookup_table) const { + simdjson_inline simd8 lookup_16(simd8 lookup_table) const { return lookup_table.apply_lookup_16_to(*this); } template - simdjson_constexpr simd8 lookup_16( + simdjson_inline simd8 lookup_16( L replace0, L replace1, L replace2, L replace3, L replace4, L replace5, L replace6, L replace7, L replace8, L replace9, L replace10, L replace11, @@ -498,7 +440,7 @@ simdjson_constexpr int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, } template - simdjson_constexpr simd8 apply_lookup_16_to(const simd8 original) { + simdjson_inline simd8 apply_lookup_16_to(const simd8 original) { return vqtbl1q_s8(*this, simd8(original)); } }; @@ -513,22 +455,24 @@ simdjson_constexpr int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, simd8x64& operator=(const simd8& other) = delete; // no assignment allowed simd8x64() = delete; // no default constructor allowed - simdjson_constexpr simd8x64(const simd8 chunk0, const simd8 chunk1, const simd8 chunk2, const simd8 chunk3) : chunks{chunk0, chunk1, chunk2, chunk3} {} - simdjson_constexpr simd8x64(const T ptr[64]) : chunks{simd8::load(ptr), simd8::load(ptr+16), simd8::load(ptr+32), simd8::load(ptr+48)} {} + simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1, const simd8 chunk2, const simd8 chunk3) : chunks{chunk0, chunk1, chunk2, chunk3} {} + simdjson_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr), simd8::load(ptr+16), simd8::load(ptr+32), simd8::load(ptr+48)} {} + simdjson_inline simd8x64(simd8x64&& o) noexcept = default; + simdjson_inline simd8x64& operator=(simd8x64&& other) noexcept = default; - simdjson_constexpr void store(T ptr[64]) const { + simdjson_inline void store(T ptr[64]) const { this->chunks[0].store(ptr+sizeof(simd8)*0); this->chunks[1].store(ptr+sizeof(simd8)*1); this->chunks[2].store(ptr+sizeof(simd8)*2); this->chunks[3].store(ptr+sizeof(simd8)*3); } - simdjson_constexpr simd8 reduce_or() const { + simdjson_inline simd8 reduce_or() const { return (this->chunks[0] | this->chunks[1]) | (this->chunks[2] | this->chunks[3]); } - simdjson_constexpr uint64_t compress(uint64_t mask, T * output) const { + simdjson_inline uint64_t compress(uint64_t mask, T * output) const { uint64_t popcounts = vget_lane_u64(vreinterpret_u64_u8(vcnt_u8(vcreate_u8(~mask))), 0); // compute the prefix sum of the popcounts of each byte uint64_t offsets = popcounts * 0x0101010101010101; @@ -539,7 +483,7 @@ simdjson_constexpr int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, return offsets >> 56; } - simdjson_constexpr uint64_t to_bitmask() const { + simdjson_inline uint64_t to_bitmask() const { #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO const simd_t bit_mask = make_uint8x16_t( 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80, @@ -559,7 +503,7 @@ simdjson_constexpr int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, return vgetq_lane_u64(vreinterpretq_u64_u8(sum0), 0); } - simdjson_constexpr uint64_t eq(const T m) const { + simdjson_inline uint64_t eq(const T m) const { const simd8 mask = simd8::splat(m); return simd8x64( this->chunks[0] == mask, @@ -569,22 +513,16 @@ simdjson_constexpr int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, ).to_bitmask(); } - simdjson_inline simd8x64 lookup_16(simd8 lookup_table) const { + simdjson_inline simd8x64 lookup_16(const simd8& lookup_table) const { return { this->chunks[0].lookup_16(lookup_table), this->chunks[1].lookup_16(lookup_table), this->chunks[2].lookup_16(lookup_table), - this->chunks[3].lookup_16(lookup_table), + this->chunks[3].lookup_16(lookup_table) }; } - template - simdjson_inline uint64_t eq_any(V ...values) const { - static constexpr const simd8 LOOKUP_TABLE = simd8::create_eq_lookup_16_table({values...}); - return eq(lookup_16(LOOKUP_TABLE)); - } - - simdjson_constexpr uint64_t lteq(const T m) const { + simdjson_inline uint64_t lteq(const T m) const { const simd8 mask = simd8::splat(m); return simd8x64( this->chunks[0] <= mask, @@ -593,10 +531,119 @@ simdjson_constexpr int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, this->chunks[3] <= mask ).to_bitmask(); } + + simdjson_inline simd8x64 operator&(const simd8x64& other) const { + return { + this->chunks[0] & other.chunks[0], + this->chunks[1] & other.chunks[1], + this->chunks[2] & other.chunks[2], + this->chunks[3] & other.chunks[3] + }; + } + + simdjson_inline simd8x64 operator&(const simd8& other) const { + return { + this->chunks[0] & other, + this->chunks[1] & other, + this->chunks[2] & other, + this->chunks[3] & other + }; + } + + simdjson_inline simd8x64 operator|(const simd8x64& other) const { + return { + this->chunks[0] | other.chunks[0], + this->chunks[1] | other.chunks[1], + this->chunks[2] | other.chunks[2], + this->chunks[3] | other.chunks[3] + }; + } + + simdjson_inline simd8x64 operator|(const simd8& other) const { + return { + this->chunks[0] | other, + this->chunks[1] | other, + this->chunks[2] | other, + this->chunks[3] | other + }; + } + + simdjson_inline simd8x64 operator^(const simd8x64& other) const { + return { + this->chunks[0] ^ other.chunks[0], + this->chunks[1] ^ other.chunks[1], + this->chunks[2] ^ other.chunks[2], + this->chunks[3] ^ other.chunks[3] + }; + } + + simdjson_inline simd8x64 operator^(const simd8& other) const { + return { + this->chunks[0] ^ other, + this->chunks[1] ^ other, + this->chunks[2] ^ other, + this->chunks[3] ^ other + }; + } + + simdjson_inline simd8x64 bit_andnot(const simd8x64& other) const { + return { + this->chunks[0].bit_andnot(other.chunks[0]), + this->chunks[1].bit_andnot(other.chunks[1]), + this->chunks[2].bit_andnot(other.chunks[2]), + this->chunks[3].bit_andnot(other.chunks[3]) + }; + } + + simdjson_inline simd8x64 bit_andnot(const simd8& other) const { + return { + this->chunks[0].bit_andnot(other), + this->chunks[1].bit_andnot(other), + this->chunks[2].bit_andnot(other), + this->chunks[3].bit_andnot(other) + }; + } + + template + simdjson_inline simd8x64 shr() const noexcept { + return { + this->chunks[0].template shr(), + this->chunks[1].template shr(), + this->chunks[2].template shr(), + this->chunks[3].template shr() + }; + } + + template + simdjson_inline simd8x64 shl() const noexcept { + return { + this->chunks[0].template shl(), + this->chunks[1].template shl(), + this->chunks[2].template shl(), + this->chunks[3].template shl() + }; + } + + simdjson_inline simd8x64 any_bits_set(const simd8& bits) const { + return { + this->chunks[0].any_bits_set(bits), + this->chunks[1].any_bits_set(bits), + this->chunks[2].any_bits_set(bits), + this->chunks[3].any_bits_set(bits) + }; + } + + simdjson_inline simd8x64 any_bits_set(const simd8x64& bits) const { + return { + this->chunks[0].any_bits_set(bits.chunks[0]), + this->chunks[1].any_bits_set(bits.chunks[1]), + this->chunks[2].any_bits_set(bits.chunks[2]), + this->chunks[3].any_bits_set(bits.chunks[3]) + }; + } }; // struct simd8x64 } // namespace simd -} // unnamed namespace } // namespace arm64 } // namespace simdjson diff --git a/include/simdjson/common_defs.h b/include/simdjson/common_defs.h index 008769da9..94ee60386 100644 --- a/include/simdjson/common_defs.h +++ b/include/simdjson/common_defs.h @@ -168,9 +168,31 @@ double from_chars(const char *first, const char* end) noexcept; #endif #ifndef simdjson_constexpr -// Force inlinine for most simdjson functions. -#define simdjson_constexpr constexpr simdjson_really_inline +#if __cpp_constexpr +#define simdjson_constexpr constexpr simdjson_inline +#else +#define simdjson_constexpr simdjson_inline #endif +#endif +// simdjson_constexpr + +#ifndef simdjson_consteval +#if __cpp_consteval +#define simdjson_consteval consteval simdjson_inline +#else +#define simdjson_consteval simdjson_constexpr +#endif +#endif // simdjson_consteval + +#ifndef simdjson_constinit +#if __cpp_constinit +#define simdjson_constinit constinit +#elif __cpp_consteval +#define simdjson_constinit consteval +#else +#define simdjson_constinit constexpr +#endif +#endif // simdjson_constinit #if SIMDJSON_VISUAL_STUDIO /** diff --git a/include/simdjson/generic/lookup_table.h b/include/simdjson/generic/lookup_table.h new file mode 100644 index 000000000..3bcc67886 --- /dev/null +++ b/include/simdjson/generic/lookup_table.h @@ -0,0 +1,296 @@ +#ifndef SIMDJSON_GENERIC_SIMD_LOOKUP_TABLE_H + +#ifndef SIMDJSON_CONDITIONAL_INCLUDE +#define SIMDJSON_GENERIC_SIMD_LOOKUP_TABLE_H +#include "simdjson/generic/base.h" +#endif // SIMDJSON_CONDITIONAL_INCLUDE + +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { +namespace simd { + +template +struct simd8_buffer { + T buf[simd8::LANES] = {0}; + simdjson_consteval T& operator[](size_t i) noexcept { return buf[i]; } + simdjson_consteval T operator[](size_t i) const noexcept { return buf[i]; } + simdjson_inline operator simd8() const noexcept { return buf; } +}; + + +struct nibble_range; + +struct byte_range { + struct _exclusive{}; + static simdjson_constinit _exclusive exclusive{}; + + simdjson_consteval byte_range(uint8_t start, uint16_t end, const _exclusive&) noexcept : _start{start}, _end{end} { + SIMDJSON_ASSUME(start < end && end <= 256); + } + simdjson_consteval byte_range(uint8_t first, uint8_t last) noexcept : byte_range(first, static_cast(last+1), exclusive) {} + simdjson_consteval byte_range(uint8_t first) noexcept : byte_range(first, first) {} + + simdjson_consteval size_t size() const noexcept { return static_cast(_end - _start); } + simdjson_consteval byte_range operator|(const byte_range& other) const noexcept { + SIMDJSON_ASSUME((_end + 1 >= other._start) || (other._end + 1 >= _start)); + return {std::min(_start, other._start), std::max(_end, other._end), exclusive}; + } + + simdjson_consteval bool includes(uint8_t byte) const noexcept { return _start <= byte && byte < _end; } + + struct nibble_iter { + int nibble; + const int last_nibble; + simdjson_consteval nibble_iter& operator++() noexcept { + if (nibble == last_nibble) { + nibble = -1; + } else { + ++nibble; + nibble %= 16; + } + return *this; + } + simdjson_consteval nibble_iter operator++(int) noexcept { auto copy = *this; ++*this; return copy; } + simdjson_consteval bool operator==(const nibble_iter& other) const noexcept { return nibble == other.nibble; } + simdjson_consteval bool operator!=(const nibble_iter& other) const noexcept { return nibble != other.nibble; } + simdjson_consteval uint8_t operator*() const noexcept { return static_cast(nibble); } + simdjson_consteval nibble_iter begin() const noexcept { return *this; } + simdjson_consteval nibble_iter end() const noexcept { return {-1, last_nibble}; } + }; + + simdjson_consteval nibble_iter nibble(int shift) const noexcept { + SIMDJSON_ASSUME(_start < _end); + auto first_nibble = _start >> shift; + auto last_nibble = (_end-1) >> shift; + if ((last_nibble - first_nibble) >= 16) { return {0x00, 0x0F}; } + return {static_cast(first_nibble & 0x0F), static_cast(last_nibble & 0x0F)}; + } + + struct _iter { + uint16_t value; + simdjson_consteval _iter& operator++() noexcept { ++value; return *this; } + simdjson_consteval _iter operator++(int) noexcept { auto copy = *this; ++*this; return copy; } + simdjson_consteval bool operator==(const _iter& other) const noexcept { return value == other.value; } + simdjson_consteval bool operator!=(const _iter& other) const noexcept { return value != other.value; } + simdjson_consteval uint8_t operator*() const noexcept { return static_cast(value); } + }; + + simdjson_consteval _iter begin() const noexcept { return _iter{_start}; } + simdjson_consteval _iter end() const noexcept { return _iter{_end}; } + + uint8_t _start; + const uint16_t _end; +}; + +namespace { + +struct _lookup_entry_range; + +struct _lookup_entry : byte_range { + const uint8_t value; + + simdjson_consteval _lookup_entry(const byte_range& bytes, uint8_t value) noexcept + : byte_range{bytes}, value{value} {} +}; + +simdjson_consteval simd8_buffer _make_nibble_lookup_table( + std::initializer_list<_lookup_entry> entries, int shift) noexcept { + // Make the buffer + simd8_buffer buf; + for (auto entry : entries) { + for (auto key : entry.nibble(shift)) { + // Repeat the value over and over for longer simd types. + for (uint8_t k = key; k < sizeof(buf); k += 16) { buf[k] |= entry.value; } + } + } + return buf; +} + +} // unnamed namespace + +/** + * Byte lookup table where the key is the high 4 bits of the input, and the value is an + * arbitrary byte. + * + * - Unmatched values yield 0. + * - Multiple keys may yield the same value. + * - Multiple bytes with the same high 4 bits may NOT yield different values. + * + * ``` + * enum ops_t : uint8_t { + * COMMA = 1, + * COLON = 2, + * BRACKET = 3, + * CURLY = 4 + * }; + * static constinit const high_nibble_lookup OPS( + * {',', COMMA}, + * {':', COLON}, + * {'[', BRACKET}, + * {']', BRACKET}, + * {'{', CURLY}, + * {'}', CURLY} + * ); + * simd8 lookup_ops(simd8& operators) { return OPS[operators]; } + * ``` + */ +struct high_nibble_lookup { + const simd8_buffer table; + + /** + * Construct a nibble lookup table from the high bits of the input to the output. + * + * @param entries A list of {key, value} pairs (e.g. {'a', 10}). + * @error asserts if multiple keys have the same high 4 bits but different values. + */ + simdjson_consteval high_nibble_lookup(std::initializer_list<_lookup_entry> entries) noexcept + : table{_make_nibble_lookup_table(entries, 4)} {} + simdjson_consteval high_nibble_lookup(const simd8_buffer& table) noexcept : table(table) {} + + /** Look up the value corresponding the higher 4 bits of each input byte, and return it. */ + simdjson_inline simd8 operator[](const simd8& keys) const noexcept { return lookup(keys); } + /** Look up the value corresponding the higher 4 bits of each input byte, and return it. */ + simdjson_inline simd8 lookup(const simd8& keys) const noexcept { return lookup_low(keys.shr<4>()); } + /** + * Look up the value in the table assuming the high 4 key bits are stored in the lower 4 bits. + * @pre all indexes be less than 16. + */ + simdjson_inline simd8 lookup_low(const simd8& shifted_keys) const noexcept { + return shifted_keys.lookup_16(table); + } + + /** + * Look up the value in the table assuming the high 4 key bits are stored in the lower 4 bits. + * @pre all indexes be less than 16. + */ + simdjson_inline simd8x64 lookup_low(const simd8x64& shifted_keys) const noexcept { + return shifted_keys.lookup_16(table); + } + /** Look up the value corresponding the higher 4 bits of each input byte, and return it. */ + simdjson_inline simd8x64 lookup(const simd8x64& keys) const noexcept { return lookup_low(keys.shr<4>()); } + /** Look up the value corresponding the higher 4 bits of each input byte, and return it. */ + simdjson_inline simd8x64 operator[](const simd8x64& keys) const noexcept { return lookup(keys); } + + simdjson_consteval uint8_t operator[](uint8_t key) const noexcept { return lookup(key); } + simdjson_consteval uint8_t lookup(uint8_t key) const noexcept { return table[key >> 4]; } +}; + +/** + * Byte lookup table where the key is the low 4 bits of the input, and the value is an + * arbitrary byte. + * + * - Unmatched values yield 0. + * - Multiple keys may yield the same value. + * - Multiple bytes with the same low 4 bits may NOT yield different values. + * + * ``` + * enum ops_t : uint8_t { + * COMMA = 1, + * COLON = 2, + * BRACKET = 3, + * CURLY = 4 + * }; + * static constinit const high_nibble_lookup OPS( + * {',', COMMA}, + * {':', COLON}, + * {'[', BRACKET}, + * {']', BRACKET}, + * {'{', CURLY}, + * {'}', CURLY} + * ); + * simd8 lookup_ops(simd8& operators) { return OPS[operators]; } + * ``` + */ +struct low_nibble_lookup { + const simd8_buffer table; + + /** + * Construct a nibble lookup table from the low bits of the input to the output. + * + * @param entries A list of {key, value} pairs (e.g. {'a', 0}). + * @error asserts if multiple keys have the same low 4 bits but different values. + */ + simdjson_consteval low_nibble_lookup(std::initializer_list<_lookup_entry> entries) noexcept + : table{_make_nibble_lookup_table(entries, 0)} {} + simdjson_consteval low_nibble_lookup(const simd8_buffer& table) noexcept : table(table) {} + + /** Look up the value corresponding the lower 4 bits of each input byte, and return it. */ + simdjson_inline simd8 operator[](const simd8& keys) const noexcept { return lookup(keys); } + /** Look up the value corresponding the lower 4 bits of each input byte, and return it. */ + simdjson_inline simd8 lookup(const simd8& keys) const noexcept { return lookup_unsafe(keys & 0x0F); } + /** + * Look up the value in the table. Behavior is system-dependent for indexes greater than 16. + * + * - On some platforms like arm64, indexes greater than 16 will not match anything in the table. + * - On platforms like Intel, index bits 4-6 will be ignored, but if the high bit is set, it + * will not match anything in the table. greater than 16 will be ignored, *except* if the high bit is 1, + */ + simdjson_inline simd8 lookup_unsafe(const simd8& shifted_keys) const noexcept { + return shifted_keys.lookup_16(table); + } + + /** Look up the value corresponding the higher 4 bits of each input byte, and return it. */ + simdjson_inline simd8x64 operator[](const simd8x64& keys) const noexcept { return lookup(keys); } + /** Look up the value corresponding the higher 4 bits of each input byte, and return it. */ + simdjson_inline simd8x64 lookup(const simd8x64& keys) const noexcept { return lookup_unsafe(keys & 0x0F); } + /** + * Look up the value in the table. Behavior is system-dependent for indexes greater than 16. + * + * - On some platforms like arm64, indexes greater than 16 will not match anything in the table. + * - On platforms like Intel, index bits 4-6 will be ignored, but if the high bit is set, it + * will not match anything in the table. greater than 16 will be ignored, *except* if the high bit is 1, + */ + simdjson_inline simd8x64 lookup_unsafe(const simd8x64& low_keys) const noexcept { + return low_keys.lookup_16(table); + } + + simdjson_consteval uint8_t operator[](uint8_t key) const noexcept { return lookup(key); } + simdjson_consteval uint8_t lookup(uint8_t key) const noexcept { return table[key & 0x0F]; } +}; + +/** + * Classifies bytes by looking up their lower 4 bits, then their high 4 bits, and &'ing the + * results together. + * + * Pass the bytes you want to match, and the classifications you want for them. + */ +struct byte_classifier { + const low_nibble_lookup low; + const high_nibble_lookup high; + simdjson_consteval byte_classifier(std::initializer_list<_lookup_entry> entries) + : low{entries}, high{entries} {} + + simdjson_inline simd8 classify(const simd8& bytes) const noexcept { + return low.lookup(bytes) & high.lookup(bytes); + } + simdjson_inline simd8x64 classify(const simd8x64& bytes) const noexcept { + return low.lookup(bytes) & high.lookup(bytes); + } + simdjson_consteval uint8_t classify(uint8_t byte) const noexcept { + return low.lookup(byte) & high.lookup(byte); + } + + simdjson_inline simd8 operator[](const simd8& bytes) const noexcept { return classify(bytes); } + simdjson_inline simd8x64 operator[](const simd8x64& bytes) const noexcept { return classify(bytes); } + simdjson_consteval uint8_t operator[](uint8_t byte) const noexcept { return classify(byte); } + + simdjson_inline bool matches_correctly(std::initializer_list<_lookup_entry> entries) const noexcept { + uint8_t expected_output[256] = {}; + for (auto entry : entries) { + for (uint8_t byte : entry) { + expected_output[byte] |= entry.value; + } + } + for (uint8_t byte = 0; byte <= 0xFF; byte++) { + if (expected_output[byte] != classify(byte)) { return false; } + } + return true; + } +}; + + +} // namespace simd +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_SIMD_LOOKUP_TABLE_H \ No newline at end of file diff --git a/include/simdjson/haswell.h b/include/simdjson/haswell.h index 867b7a449..9339f138f 100644 --- a/include/simdjson/haswell.h +++ b/include/simdjson/haswell.h @@ -3,6 +3,7 @@ #include "simdjson/haswell/begin.h" #include "simdjson/generic/amalgamated.h" +#include "simdjson/generic/lookup_table.h" #include "simdjson/haswell/end.h" #endif // SIMDJSON_HASWELL_H \ No newline at end of file diff --git a/include/simdjson/haswell/base.h b/include/simdjson/haswell/base.h index 275bcec00..e8456d397 100644 --- a/include/simdjson/haswell/base.h +++ b/include/simdjson/haswell/base.h @@ -14,7 +14,6 @@ namespace haswell { class implementation; -namespace { namespace simd { template struct simd8; @@ -23,7 +22,6 @@ template <> struct simd8; template struct simd8x64; } // namespace simd -} // unnamed namespace } // namespace haswell } // namespace simdjson diff --git a/include/simdjson/haswell/simd.h b/include/simdjson/haswell/simd.h index 0784585de..0a3bbacae 100644 --- a/include/simdjson/haswell/simd.h +++ b/include/simdjson/haswell/simd.h @@ -10,7 +10,6 @@ namespace simdjson { namespace haswell { -namespace { namespace simd { // Forward-declared so they can be used by splat and friends. @@ -21,23 +20,23 @@ namespace simd { simd_t value; // Zero constructor - simdjson_constexpr base() : value{simd_t()} {} + simdjson_inline base() : value{simd_t()} {} // Conversion from SIMD register - simdjson_constexpr base(const simd_t _value) : value(_value) {} + simdjson_inline base(const simd_t _value) : value(_value) {} // Conversion to SIMD register - simdjson_constexpr operator const simd_t&() const { return this->value; } - simdjson_constexpr operator simd_t&() { return this->value; } + simdjson_inline operator const simd_t&() const { return this->value; } + simdjson_inline operator simd_t&() { return this->value; } // Bit operations - simdjson_constexpr Child operator|(const Child other) const { return _mm256_or_si256(*this, other); } - simdjson_constexpr Child operator&(const Child other) const { return _mm256_and_si256(*this, other); } - simdjson_constexpr Child operator^(const Child other) const { return _mm256_xor_si256(*this, other); } - simdjson_constexpr Child bit_andnot(const Child other) const { return _mm256_andnot_si256(other, *this); } - simdjson_constexpr Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } - simdjson_constexpr Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } - simdjson_constexpr Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } + simdjson_inline Child operator|(const Child other) const { return _mm256_or_si256(*this, other); } + simdjson_inline Child operator&(const Child other) const { return _mm256_and_si256(*this, other); } + simdjson_inline Child operator^(const Child other) const { return _mm256_xor_si256(*this, other); } + simdjson_inline Child bit_andnot(const Child other) const { return _mm256_andnot_si256(other, *this); } + simdjson_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } + simdjson_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } + simdjson_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } }; // Forward-declared so they can be used by splat and friends. @@ -49,16 +48,16 @@ namespace simd { using typename base>::simd_t; static constexpr const int LANES = sizeof(simd_t); using bitmask_t = uint32_t; - static_assert(sizeof(bitmask_t)*8 == LANES); + static_assert(sizeof(bitmask_t)*8 == LANES, "Bitmask type's bits must equal the simd type's bytes"); - simdjson_constexpr base8() : base>() {} - simdjson_constexpr base8(const simd_t _value) : base>(_value) {} + simdjson_inline base8() : base>() {} + simdjson_inline base8(const simd_t _value) : base>(_value) {} - simdjson_constexpr Mask eq(const simd8 rhs) const { return _mm256_cmpeq_epi8(*this, rhs); } - friend simdjson_constexpr Mask operator==(const simd8 lhs, const simd8 rhs) { return lhs.eq(rhs); } + simdjson_inline Mask eq(const simd8 rhs) const { return _mm256_cmpeq_epi8(*this, rhs); } + friend simdjson_inline Mask operator==(const simd8 lhs, const simd8 rhs) { return lhs.eq(rhs); } template - simdjson_constexpr simd8 prev(const simd8 prev_chunk) const { + simdjson_inline simd8 prev(const simd8 prev_chunk) const { return _mm256_alignr_epi8(*this, _mm256_permute2x128_si256(prev_chunk, *this, 0x21), 16 - N); } }; @@ -66,29 +65,29 @@ namespace simd { // SIMD byte mask type (returned by things like eq and gt) template<> struct simd8: base8 { - static simdjson_constexpr simd8 splat(bool _value) { return _mm256_set1_epi8(uint8_t(-(!!_value))); } + static simdjson_inline simd8 splat(bool _value) { return _mm256_set1_epi8(uint8_t(-(!!_value))); } - simdjson_constexpr simd8() : base8() {} - simdjson_constexpr simd8(const simd_t _value) : base8(_value) {} + simdjson_inline simd8() : base8() {} + simdjson_inline simd8(const simd_t _value) : base8(_value) {} // Splat constructor - simdjson_constexpr simd8(bool _value) : base8(splat(_value)) {} + simdjson_inline simd8(bool _value) : base8(splat(_value)) {} - simdjson_constexpr int to_bitmask() const { return _mm256_movemask_epi8(*this); } - simdjson_constexpr bool any() const { return !_mm256_testz_si256(*this, *this); } - simdjson_constexpr simd8 operator~() const { return *this ^ true; } + simdjson_inline int to_bitmask() const { return _mm256_movemask_epi8(*this); } + simdjson_inline bool any() const { return !_mm256_testz_si256(*this, *this); } + simdjson_inline simd8 operator~() const { return *this ^ true; } }; template struct base8_numeric: base8 { using typename base8::simd_t; using base8::LANES; - static simdjson_constexpr simd8 splat(T _value) { return _mm256_set1_epi8(_value); } - static simdjson_constexpr simd8 zero() { return _mm256_setzero_si256(); } - static simdjson_constexpr simd8 load(const T values[32]) { + static simdjson_inline simd8 splat(T _value) { return _mm256_set1_epi8(_value); } + static simdjson_inline simd8 zero() { return _mm256_setzero_si256(); } + static simdjson_inline simd8 load(const T values[32]) { return _mm256_loadu_si256(reinterpret_cast(values)); } // Repeat 16 values as many times as necessary (usually for lookup tables) - static simdjson_constexpr simd8 repeat_16( + static simdjson_inline simd8 repeat_16( T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 ) { @@ -100,24 +99,23 @@ namespace simd { ); } - simdjson_constexpr base8_numeric() : base8() {} - simdjson_constexpr base8_numeric(const simd_t _value) : base8(_value) {} + simdjson_inline base8_numeric() : base8() {} + simdjson_inline base8_numeric(const simd_t _value) : base8(_value) {} // Store to array - simdjson_constexpr void store(T dst[32]) const { return _mm256_storeu_si256(reinterpret_cast(dst), *this); } + simdjson_inline void store(T dst[32]) const { return _mm256_storeu_si256(reinterpret_cast(dst), *this); } // Addition/subtraction are the same for signed and unsigned - simdjson_constexpr simd8 operator+(const simd8 other) const { return _mm256_add_epi8(*this, other); } - simdjson_constexpr simd8 operator-(const simd8 other) const { return _mm256_sub_epi8(*this, other); } - simdjson_constexpr simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } - simdjson_constexpr simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } + simdjson_inline simd8 operator+(const simd8 other) const { return _mm256_add_epi8(*this, other); } + simdjson_inline simd8 operator-(const simd8 other) const { return _mm256_sub_epi8(*this, other); } + simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } + simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } // Override to distinguish from bool version - simdjson_constexpr simd8 operator~() const { return *this ^ 0xFFu; } + simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) - template - simdjson_constexpr simd8 lookup_16(simd8 lookup_table) const { + simdjson_inline simd8 lookup_16(const simd8& lookup_table) const { return _mm256_shuffle_epi8(lookup_table, *this); } @@ -129,7 +127,7 @@ namespace simd { // signature simd8 compress(uint32_t mask) would be // sensible, but the AVX ISA makes this kind of approach difficult. template - simdjson_constexpr void compress(uint32_t mask, L * output) const { + simdjson_inline void compress(uint32_t mask, L * output) const { using internal::thintable_epi8; using internal::BitsSetTable256mul2; using internal::pshufb_combine_table; @@ -172,79 +170,19 @@ namespace simd { v128 = _mm256_extractf128_si256(almostthere, 1); _mm_storeu_si128( reinterpret_cast<__m128i *>(output + 16 - count_ones(mask & 0xFFFF)), v128); } - - struct lane_with_value { int lane; T value; }; - - /** - * Initialize a simd8 by filling in only specific lanes. - * - * @param entries A set of index/value pairs, like {{1, 'a'}, {2, 'b'}, ...} - * @param default_value The value to use for other lanes. - */ - static simdjson_constexpr simd8 create_sparse( - std::initializer_list entries, - T default_value = {} - ) noexcept { - bool filled[LANES] = {0}; - uint8_t table[LANES] = {default_value}; - for (auto [lane, value] : entries) { - assert(lane < LANES); - assert(!filled[lane]); - filled[lane] = true; - table[lane] = value; - } - return table; - } - - static simdjson_constexpr simd8 create_eq_lookup_16_table(std::initializer_list values) { - bool filled[16] = {0}; - - // Set the defaults to 0, except at 0 itself (which we set to 1 so it won't accidentally match 0). - uint8_t table[LANES] = {0}; - for (int lane = 0; lane < 16; lane += 16) { table[lane] = 1; } - - for (T value : values) { - int lane = value & 0x0F; - assert(!filled[lane]); - filled[lane] = true; - // Repeat the value at the same position in each 16-byte section of lanes. - for (; lane < LANES; lane += 16) { table[lane] = value; } - } - return table; - } - - template - simdjson_inline simd8 eq_any(V ...values) const { - static constexpr const simd8 LOOKUP_TABLE = create_eq_lookup_16_table({values...}); - return eq(lookup_16(LOOKUP_TABLE)); - } - - template - simdjson_constexpr simd8 lookup_16( - L replace0, L replace1, L replace2, L replace3, - L replace4, L replace5, L replace6, L replace7, - L replace8, L replace9, L replace10, L replace11, - L replace12, L replace13, L replace14, L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, - replace4, replace5, replace6, replace7, - replace8, replace9, replace10, replace11, - replace12, replace13, replace14, replace15 - )); - } }; // Signed bytes template<> struct simd8 : base8_numeric { - simdjson_constexpr simd8() : base8_numeric() {} - simdjson_constexpr simd8(const simd_t _value) : base8_numeric(_value) {} + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const simd_t _value) : base8_numeric(_value) {} // Splat constructor - simdjson_constexpr simd8(int8_t _value) : simd8(splat(_value)) {} + simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} // Array constructor - simdjson_constexpr simd8(const int8_t values[32]) : simd8(load(values)) {} + simdjson_inline simd8(const int8_t values[32]) : simd8(load(values)) {} // Member-by-member initialization - simdjson_constexpr simd8( + simdjson_inline simd8( int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15, int8_t v16, int8_t v17, int8_t v18, int8_t v19, int8_t v20, int8_t v21, int8_t v22, int8_t v23, @@ -256,7 +194,7 @@ namespace simd { v24,v25,v26,v27,v28,v29,v30,v31 )) {} // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_constexpr static simd8 repeat_16( + simdjson_inline static simd8 repeat_16( int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 ) { @@ -269,23 +207,23 @@ namespace simd { } // Order-sensitive comparisons - simdjson_constexpr simd8 max_val(const simd8 other) const { return _mm256_max_epi8(*this, other); } - simdjson_constexpr simd8 min_val(const simd8 other) const { return _mm256_min_epi8(*this, other); } - simdjson_constexpr simd8 operator>(const simd8 other) const { return _mm256_cmpgt_epi8(*this, other); } - simdjson_constexpr simd8 operator<(const simd8 other) const { return _mm256_cmpgt_epi8(other, *this); } + simdjson_inline simd8 max_val(const simd8 other) const { return _mm256_max_epi8(*this, other); } + simdjson_inline simd8 min_val(const simd8 other) const { return _mm256_min_epi8(*this, other); } + simdjson_inline simd8 operator>(const simd8 other) const { return _mm256_cmpgt_epi8(*this, other); } + simdjson_inline simd8 operator<(const simd8 other) const { return _mm256_cmpgt_epi8(other, *this); } }; // Unsigned bytes template<> struct simd8: base8_numeric { - simdjson_constexpr simd8() : base8_numeric() {} - simdjson_constexpr simd8(const simd_t _value) : base8_numeric(_value) {} + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const simd_t _value) : base8_numeric(_value) {} // Splat constructor - simdjson_constexpr simd8(uint8_t _value) : simd8(splat(_value)) {} + simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} // Array constructor - simdjson_constexpr simd8(const uint8_t values[32]) : simd8(load(values)) {} + simdjson_inline simd8(const uint8_t values[32]) : simd8(load(values)) {} // Member-by-member initialization - simdjson_constexpr simd8( + simdjson_inline simd8( uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15, uint8_t v16, uint8_t v17, uint8_t v18, uint8_t v19, uint8_t v20, uint8_t v21, uint8_t v22, uint8_t v23, @@ -297,7 +235,7 @@ namespace simd { v24,v25,v26,v27,v28,v29,v30,v31 )) {} // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_constexpr static simd8 repeat_16( + simdjson_inline static simd8 repeat_16( uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 ) { @@ -310,39 +248,39 @@ namespace simd { } // Saturated math - simdjson_constexpr simd8 saturating_add(const simd8 other) const { return _mm256_adds_epu8(*this, other); } - simdjson_constexpr simd8 saturating_sub(const simd8 other) const { return _mm256_subs_epu8(*this, other); } + simdjson_inline simd8 saturating_add(const simd8 other) const { return _mm256_adds_epu8(*this, other); } + simdjson_inline simd8 saturating_sub(const simd8 other) const { return _mm256_subs_epu8(*this, other); } // Order-specific operations - simdjson_constexpr simd8 max_val(const simd8 other) const { return _mm256_max_epu8(*this, other); } - simdjson_constexpr simd8 min_val(const simd8 other) const { return _mm256_min_epu8(other, *this); } + simdjson_inline simd8 max_val(const simd8 other) const { return _mm256_max_epu8(*this, other); } + simdjson_inline simd8 min_val(const simd8 other) const { return _mm256_min_epu8(other, *this); } // Same as >, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_constexpr simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } + simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } // Same as <, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_constexpr simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } - simdjson_constexpr simd8 operator<=(const simd8 other) const { return other.max_val(*this) == other; } - simdjson_constexpr simd8 operator>=(const simd8 other) const { return other.min_val(*this) == other; } - simdjson_constexpr simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } - simdjson_constexpr simd8 operator<(const simd8 other) const { return this->lt_bits(other).any_bits_set(); } + simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } + simdjson_inline simd8 operator<=(const simd8 other) const { return other.max_val(*this) == other; } + simdjson_inline simd8 operator>=(const simd8 other) const { return other.min_val(*this) == other; } + simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } + simdjson_inline simd8 operator<(const simd8 other) const { return this->lt_bits(other).any_bits_set(); } // Bit-specific operations - simdjson_constexpr simd8 bits_not_set() const { return *this == uint8_t(0); } - simdjson_constexpr simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } - simdjson_constexpr simd8 any_bits_set() const { return ~this->bits_not_set(); } - simdjson_constexpr simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } - simdjson_constexpr bool is_ascii() const { return _mm256_movemask_epi8(*this) == 0; } - simdjson_constexpr bool bits_not_set_anywhere() const { return _mm256_testz_si256(*this, *this); } - simdjson_constexpr bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } - simdjson_constexpr bool bits_not_set_anywhere(simd8 bits) const { return _mm256_testz_si256(*this, bits); } - simdjson_constexpr bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } + simdjson_inline simd8 bits_not_set() const { return *this == uint8_t(0); } + simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } + simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } + simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } + simdjson_inline bool is_ascii() const { return _mm256_movemask_epi8(*this) == 0; } + simdjson_inline bool bits_not_set_anywhere() const { return _mm256_testz_si256(*this, *this); } + simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } + simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { return _mm256_testz_si256(*this, bits); } + simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } template - simdjson_constexpr simd8 shr() const { return simd8(_mm256_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); } + simdjson_inline simd8 shr() const { return simd8(_mm256_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); } template - simdjson_constexpr simd8 shl() const { return simd8(_mm256_slli_epi16(*this, N)) & uint8_t(0xFFu << N); } + simdjson_inline simd8 shl() const { return simd8(_mm256_slli_epi16(*this, N)) & uint8_t(0xFFu << N); } // Get one of the bits and make a bitmask out of it. // e.g. value.get_bit<7>() gets the high bit template - simdjson_constexpr int get_bit() const { return _mm256_movemask_epi8(_mm256_slli_epi16(*this, 7-N)); } + simdjson_inline int get_bit() const { return _mm256_movemask_epi8(_mm256_slli_epi16(*this, 7-N)); } }; template @@ -355,10 +293,12 @@ namespace simd { simd8x64& operator=(const simd8& other) = delete; // no assignment allowed simd8x64() = delete; // no default constructor allowed - simdjson_constexpr simd8x64(const simd8 chunk0, const simd8 chunk1) : chunks{chunk0, chunk1} {} - simdjson_constexpr simd8x64(const T ptr[64]) : chunks{simd8::load(ptr), simd8::load(ptr+32)} {} + simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1) : chunks{chunk0, chunk1} {} + simdjson_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr), simd8::load(ptr+32)} {} + simdjson_inline simd8x64(simd8x64&& o) noexcept = default; + simdjson_inline simd8x64& operator=(simd8x64&& other) noexcept = default; - simdjson_constexpr uint64_t compress(uint64_t mask, T * output) const { + simdjson_inline uint64_t compress(uint64_t mask, T * output) const { uint32_t mask1 = uint32_t(mask); uint32_t mask2 = uint32_t(mask >> 32); this->chunks[0].compress(mask1, output); @@ -366,30 +306,22 @@ namespace simd { return 64 - count_ones(mask); } - simdjson_constexpr void store(T ptr[64]) const { + simdjson_inline void store(T ptr[64]) const { this->chunks[0].store(ptr+sizeof(simd8)*0); this->chunks[1].store(ptr+sizeof(simd8)*1); } - simdjson_constexpr uint64_t to_bitmask() const { + simdjson_inline uint64_t to_bitmask() const { uint64_t r_lo = uint32_t(this->chunks[0].to_bitmask()); uint64_t r_hi = this->chunks[1].to_bitmask(); return r_lo | (r_hi << 32); } - simdjson_constexpr simd8 reduce_or() const { + simdjson_inline simd8 reduce_or() const { return this->chunks[0] | this->chunks[1]; } - simdjson_constexpr simd8x64 bit_or(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] | mask, - this->chunks[1] | mask - ); - } - - simdjson_constexpr uint64_t eq(const T m) const { + simdjson_inline uint64_t eq(const T m) const { const simd8 mask = simd8::splat(m); return simd8x64( this->chunks[0] == mask, @@ -397,38 +329,117 @@ namespace simd { ).to_bitmask(); } - simdjson_constexpr uint64_t eq(const simd8x64 &other) const { + simdjson_inline uint64_t eq(const simd8x64 &other) const { return simd8x64( this->chunks[0] == other.chunks[0], this->chunks[1] == other.chunks[1] ).to_bitmask(); } - simdjson_inline simd8x64 lookup_16(simd8 lookup_table) const { + simdjson_inline simd8x64 lookup_16(const simd8& lookup_table) const { return { this->chunks[0].lookup_16(lookup_table), this->chunks[1].lookup_16(lookup_table) }; } - template - simdjson_inline uint64_t eq_any(V ...values) const { - static constexpr const simd8 LOOKUP_TABLE = simd8::create_eq_lookup_16_table({values...}); - return eq(lookup_16(LOOKUP_TABLE)); - } - - simdjson_constexpr uint64_t lteq(const T m) const { + simdjson_inline uint64_t lteq(const T m) const { const simd8 mask = simd8::splat(m); return simd8x64( this->chunks[0] <= mask, this->chunks[1] <= mask ).to_bitmask(); } + + simdjson_inline simd8x64 operator&(const simd8x64& other) const { + return { + this->chunks[0] & other.chunks[0], + this->chunks[1] & other.chunks[1] + }; + } + + simdjson_inline simd8x64 operator&(const simd8& other) const { + return { + this->chunks[0] & other, + this->chunks[1] & other + }; + } + + simdjson_inline simd8x64 operator|(const simd8x64& other) const { + return { + this->chunks[0] | other.chunks[0], + this->chunks[1] | other.chunks[1] + }; + } + + simdjson_inline simd8x64 operator|(const simd8& other) const { + return { + this->chunks[0] | other, + this->chunks[1] | other + }; + } + + simdjson_inline simd8x64 operator^(const simd8x64& other) const { + return { + this->chunks[0] ^ other.chunks[0], + this->chunks[1] ^ other.chunks[1] + }; + } + + simdjson_inline simd8x64 operator^(const simd8& other) const { + return { + this->chunks[0] ^ other, + this->chunks[1] ^ other + }; + } + + simdjson_inline simd8x64 bit_andnot(const simd8x64& other) const { + return { + this->chunks[0].bit_andnot(other.chunks[0]), + this->chunks[1].bit_andnot(other.chunks[1]) + }; + } + + simdjson_inline simd8x64 bit_andnot(const simd8& other) const { + return { + this->chunks[0].bit_andnot(other), + this->chunks[1].bit_andnot(other), + }; + } + + template + simdjson_inline simd8x64 shr() const noexcept { + return { + this->chunks[0].template shr(), + this->chunks[1].template shr() + }; + } + + template + simdjson_inline simd8x64 shl() const noexcept { + return { + this->chunks[0].template shl(), + this->chunks[1].template shl() + }; + } + + simdjson_inline simd8x64 any_bits_set(const simd8& bits) const { + return { + this->chunks[0].any_bits_set(bits), + this->chunks[1].any_bits_set(bits) + }; + } + + simdjson_inline simd8x64 any_bits_set(const simd8x64& bits) const { + return { + this->chunks[0].any_bits_set(bits.chunks[0]), + this->chunks[1].any_bits_set(bits.chunks[1]) + }; + } }; // struct simd8x64 } // namespace simd -} // unnamed namespace } // namespace haswell } // namespace simdjson diff --git a/include/simdjson/icelake.h b/include/simdjson/icelake.h index 964296034..00e3e2583 100644 --- a/include/simdjson/icelake.h +++ b/include/simdjson/icelake.h @@ -3,6 +3,7 @@ #include "simdjson/icelake/begin.h" #include "simdjson/generic/amalgamated.h" +#include "simdjson/generic/lookup_table.h" #include "simdjson/icelake/end.h" #endif // SIMDJSON_ICELAKE_H \ No newline at end of file diff --git a/include/simdjson/icelake/base.h b/include/simdjson/icelake/base.h index bb5b5008c..7068b19b2 100644 --- a/include/simdjson/icelake/base.h +++ b/include/simdjson/icelake/base.h @@ -14,7 +14,6 @@ namespace icelake { class implementation; -namespace { namespace simd { template struct simd8; @@ -22,7 +21,6 @@ template <> struct simd8; template <> struct simd8; template struct simd8x64; -} // namespace simd } // unnamed namespace } // namespace icelake diff --git a/include/simdjson/icelake/simd.h b/include/simdjson/icelake/simd.h index f15f70d25..4a340ba51 100644 --- a/include/simdjson/icelake/simd.h +++ b/include/simdjson/icelake/simd.h @@ -34,7 +34,6 @@ inline simd_t _mm512_set_epi8(uint8_t a0, uint8_t a1, uint8_t a2, uint8_t a3, ui namespace simdjson { namespace icelake { -namespace { namespace simd { // Forward-declared so they can be used by splat and friends. @@ -44,23 +43,23 @@ namespace simd { simd_t value; // Zero constructor - simdjson_constexpr base() : value{simd_t()} {} + simdjson_inline base() : value{simd_t()} {} // Conversion from SIMD register - simdjson_constexpr base(const simd_t _value) : value(_value) {} + simdjson_inline base(const simd_t _value) : value(_value) {} // Conversion to SIMD register - simdjson_constexpr operator const simd_t&() const { return this->value; } - simdjson_constexpr operator simd_t&() { return this->value; } + simdjson_inline operator const simd_t&() const { return this->value; } + simdjson_inline operator simd_t&() { return this->value; } // Bit operations - simdjson_constexpr Child operator|(const Child other) const { return _mm512_or_si512(*this, other); } - simdjson_constexpr Child operator&(const Child other) const { return _mm512_and_si512(*this, other); } - simdjson_constexpr Child operator^(const Child other) const { return _mm512_xor_si512(*this, other); } - simdjson_constexpr Child bit_andnot(const Child other) const { return _mm512_andnot_si512(other, *this); } - simdjson_constexpr Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } - simdjson_constexpr Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } - simdjson_constexpr Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } + simdjson_inline Child operator|(const Child other) const { return _mm512_or_si512(*this, other); } + simdjson_inline Child operator&(const Child other) const { return _mm512_and_si512(*this, other); } + simdjson_inline Child operator^(const Child other) const { return _mm512_xor_si512(*this, other); } + simdjson_inline Child bit_andnot(const Child other) const { return _mm512_andnot_si512(other, *this); } + simdjson_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } + simdjson_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } + simdjson_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } }; // Forward-declared so they can be used by splat and friends. @@ -72,16 +71,16 @@ namespace simd { using typename base>::simd_t; static constexpr const int LANES = sizeof(simd_t); using bitmask_t = uint64_t; - static_assert(sizeof(bitmask_t)*8 == LANES); + static_assert(sizeof(bitmask_t)*8 == LANES, "Bitmask type's bits must equal the simd type's bytes"); - simdjson_constexpr base8() : base>() {} - simdjson_constexpr base8(const simd_t _value) : base>(_value) {} + simdjson_inline base8() : base>() {} + simdjson_inline base8(const simd_t _value) : base>(_value) {} - simdjson_constexpr uint64_t eq(const simd8 rhs) const { return _mm512_cmpeq_epi8_mask(*this, rhs); } - friend simdjson_constexpr uint64_t operator==(const simd8 lhs, const simd8 rhs) { return lhs.eq(rhs); } + simdjson_inline uint64_t eq(const simd8 rhs) const { return _mm512_cmpeq_epi8_mask(*this, rhs); } + friend simdjson_inline uint64_t operator==(const simd8 lhs, const simd8 rhs) { return lhs.eq(rhs); } template - simdjson_constexpr simd8 prev(const simd8 prev_chunk) const { + simdjson_inline simd8 prev(const simd8 prev_chunk) const { // workaround for compilers unable to figure out that 16 - N is a constant (GCC 8) constexpr int shift = 16 - N; return _mm512_alignr_epi8(*this, _mm512_permutex2var_epi64(prev_chunk, _mm512_set_epi64(13, 12, 11, 10, 9, 8, 7, 6), *this), shift); @@ -91,14 +90,14 @@ namespace simd { // SIMD byte mask type (returned by things like eq and gt) template<> struct simd8: base8 { - static simdjson_constexpr simd8 splat(bool _value) { return _mm512_set1_epi8(uint8_t(-(!!_value))); } + static simdjson_inline simd8 splat(bool _value) { return _mm512_set1_epi8(uint8_t(-(!!_value))); } - simdjson_constexpr simd8() : base8() {} - simdjson_constexpr simd8(const simd_t _value) : base8(_value) {} + simdjson_inline simd8() : base8() {} + simdjson_inline simd8(const simd_t _value) : base8(_value) {} // Splat constructor - simdjson_constexpr simd8(bool _value) : base8(splat(_value)) {} - simdjson_constexpr bool any() const { return !!_mm512_test_epi8_mask (*this, *this); } - simdjson_constexpr simd8 operator~() const { return *this ^ true; } + simdjson_inline simd8(bool _value) : base8(splat(_value)) {} + simdjson_inline bool any() const { return !!_mm512_test_epi8_mask (*this, *this); } + simdjson_inline simd8 operator~() const { return *this ^ true; } }; template @@ -106,13 +105,13 @@ namespace simd { using typename base8::simd_t; using base8::LANES; - static simdjson_constexpr simd8 splat(T _value) { return _mm512_set1_epi8(_value); } - static simdjson_constexpr simd8 zero() { return _mm512_setzero_si512(); } - static simdjson_constexpr simd8 load(const T values[64]) { + static simdjson_inline simd8 splat(T _value) { return _mm512_set1_epi8(_value); } + static simdjson_inline simd8 zero() { return _mm512_setzero_si512(); } + static simdjson_inline simd8 load(const T values[64]) { return _mm512_loadu_si512(reinterpret_cast(values)); } // Repeat 16 values as many times as necessary (usually for lookup tables) - static simdjson_constexpr simd8 repeat_16( + static simdjson_inline simd8 repeat_16( T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 ) { @@ -128,24 +127,23 @@ namespace simd { ); } - simdjson_constexpr base8_numeric() : base8() {} - simdjson_constexpr base8_numeric(const simd_t _value) : base8(_value) {} + simdjson_inline base8_numeric() : base8() {} + simdjson_inline base8_numeric(const simd_t _value) : base8(_value) {} // Store to array - simdjson_constexpr void store(T dst[64]) const { return _mm512_storeu_si512(reinterpret_cast(dst), *this); } + simdjson_inline void store(T dst[64]) const { return _mm512_storeu_si512(reinterpret_cast(dst), *this); } // Addition/subtraction are the same for signed and unsigned - simdjson_constexpr simd8 operator+(const simd8 other) const { return _mm512_add_epi8(*this, other); } - simdjson_constexpr simd8 operator-(const simd8 other) const { return _mm512_sub_epi8(*this, other); } - simdjson_constexpr simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } - simdjson_constexpr simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } + simdjson_inline simd8 operator+(const simd8 other) const { return _mm512_add_epi8(*this, other); } + simdjson_inline simd8 operator-(const simd8 other) const { return _mm512_sub_epi8(*this, other); } + simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } + simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } // Override to distinguish from bool version - simdjson_constexpr simd8 operator~() const { return *this ^ 0xFFu; } + simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) - template - simdjson_constexpr simd8 lookup_16(simd8 lookup_table) const { + simdjson_inline simd8 lookup_16(const simd8& lookup_table) const { return _mm512_shuffle_epi8(lookup_table, *this); } @@ -157,82 +155,22 @@ namespace simd { // signature simd8 compress(uint32_t mask) would be // sensible, but the AVX ISA makes this kind of approach difficult. template - simdjson_constexpr void compress(uint64_t mask, L * output) const { + simdjson_inline void compress(uint64_t mask, L * output) const { _mm512_mask_compressstoreu_epi8 (output,~mask,*this); } - - struct lane_with_value { int lane; T value; }; - - /** - * Initialize a simd8 by filling in only specific lanes. - * - * @param entries A set of index/value pairs, like {{1, 'a'}, {2, 'b'}, ...} - * @param default_value The value to use for other lanes. - */ - static simdjson_constexpr simd8 create_sparse( - std::initializer_list entries, - T default_value = {} - ) noexcept { - bool filled[LANES] = {0}; - uint8_t table[LANES] = {default_value}; - for (auto [lane, value] : entries) { - assert(lane < LANES); - assert(!filled[lane]); - filled[lane] = true; - table[lane] = value; - } - return table; - } - - static simdjson_constexpr simd8 create_eq_lookup_16_table(std::initializer_list values) { - bool filled[16] = {0}; - - // Set the defaults to 0, except at 0 itself (which we set to 1 so it won't accidentally match 0). - uint8_t table[LANES] = {0}; - for (int lane = 0; lane < 16; lane += 16) { table[lane] = 1; } - - for (T value : values) { - int lane = value & 0x0F; - assert(!filled[lane]); - filled[lane] = true; - // Repeat the value at the same position in each 16-byte section of lanes. - for (; lane < LANES; lane += 16) { table[lane] = value; } - } - return table; - } - - template - simdjson_inline simd8 eq_any(V ...values) const { - static constexpr const simd8 LOOKUP_TABLE = create_eq_lookup_16_table({values...}); - return eq(lookup_16(LOOKUP_TABLE)); - } - - template - simdjson_constexpr simd8 lookup_16( - L replace0, L replace1, L replace2, L replace3, - L replace4, L replace5, L replace6, L replace7, - L replace8, L replace9, L replace10, L replace11, - L replace12, L replace13, L replace14, L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, - replace4, replace5, replace6, replace7, - replace8, replace9, replace10, replace11, - replace12, replace13, replace14, replace15 - )); - } }; // Signed bytes template<> struct simd8 : base8_numeric { - simdjson_constexpr simd8() : base8_numeric() {} - simdjson_constexpr simd8(const simd_t _value) : base8_numeric(_value) {} + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const simd_t _value) : base8_numeric(_value) {} // Splat constructor - simdjson_constexpr simd8(int8_t _value) : simd8(splat(_value)) {} + simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} // Array constructor - simdjson_constexpr simd8(const int8_t values[64]) : simd8(load(values)) {} + simdjson_inline simd8(const int8_t values[64]) : simd8(load(values)) {} // Member-by-member initialization - simdjson_constexpr simd8( + simdjson_inline simd8( int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15, int8_t v16, int8_t v17, int8_t v18, int8_t v19, int8_t v20, int8_t v21, int8_t v22, int8_t v23, @@ -253,7 +191,7 @@ namespace simd { )) {} // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_constexpr static simd8 repeat_16( + simdjson_inline static simd8 repeat_16( int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 ) { @@ -270,8 +208,8 @@ namespace simd { } // Order-sensitive comparisons - simdjson_constexpr simd8 max_val(const simd8 other) const { return _mm512_max_epi8(*this, other); } - simdjson_constexpr simd8 min_val(const simd8 other) const { return _mm512_min_epi8(*this, other); } + simdjson_inline simd8 max_val(const simd8 other) const { return _mm512_max_epi8(*this, other); } + simdjson_inline simd8 min_val(const simd8 other) const { return _mm512_min_epi8(*this, other); } simdjson_inline simd8 operator>(const simd8 other) const { return _mm512_maskz_abs_epi8(_mm512_cmpgt_epi8_mask(*this, other),_mm512_set1_epi8(uint8_t(0x80))); } simdjson_inline simd8 operator<(const simd8 other) const { return _mm512_maskz_abs_epi8(_mm512_cmpgt_epi8_mask(other, *this),_mm512_set1_epi8(uint8_t(0x80))); } @@ -280,14 +218,14 @@ namespace simd { // Unsigned bytes template<> struct simd8: base8_numeric { - simdjson_constexpr simd8() : base8_numeric() {} - simdjson_constexpr simd8(const simd_t _value) : base8_numeric(_value) {} + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const simd_t _value) : base8_numeric(_value) {} // Splat constructor - simdjson_constexpr simd8(uint8_t _value) : simd8(splat(_value)) {} + simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} // Array constructor - simdjson_constexpr simd8(const uint8_t values[64]) : simd8(load(values)) {} + simdjson_inline simd8(const uint8_t values[64]) : simd8(load(values)) {} // Member-by-member initialization - simdjson_constexpr simd8( + simdjson_inline simd8( uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15, uint8_t v16, uint8_t v17, uint8_t v18, uint8_t v19, uint8_t v20, uint8_t v21, uint8_t v22, uint8_t v23, @@ -308,7 +246,7 @@ namespace simd { )) {} // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_constexpr static simd8 repeat_16( + simdjson_inline static simd8 repeat_16( uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 ) { @@ -325,42 +263,42 @@ namespace simd { } // Saturated math - simdjson_constexpr simd8 saturating_add(const simd8 other) const { return _mm512_adds_epu8(*this, other); } - simdjson_constexpr simd8 saturating_sub(const simd8 other) const { return _mm512_subs_epu8(*this, other); } + simdjson_inline simd8 saturating_add(const simd8 other) const { return _mm512_adds_epu8(*this, other); } + simdjson_inline simd8 saturating_sub(const simd8 other) const { return _mm512_subs_epu8(*this, other); } // Order-specific operations - simdjson_constexpr simd8 max_val(const simd8 other) const { return _mm512_max_epu8(*this, other); } - simdjson_constexpr simd8 min_val(const simd8 other) const { return _mm512_min_epu8(other, *this); } + simdjson_inline simd8 max_val(const simd8 other) const { return _mm512_max_epu8(*this, other); } + simdjson_inline simd8 min_val(const simd8 other) const { return _mm512_min_epu8(other, *this); } // Same as >, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_constexpr simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } + simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } // Same as <, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_constexpr simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } - simdjson_constexpr uint64_t operator<=(const simd8 other) const { return other.max_val(*this) == other; } - simdjson_constexpr uint64_t operator>=(const simd8 other) const { return other.min_val(*this) == other; } - simdjson_constexpr simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } - simdjson_constexpr simd8 operator<(const simd8 other) const { return this->lt_bits(other).any_bits_set(); } + simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } + simdjson_inline uint64_t operator<=(const simd8 other) const { return other.max_val(*this) == other; } + simdjson_inline uint64_t operator>=(const simd8 other) const { return other.min_val(*this) == other; } + simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } + simdjson_inline simd8 operator<(const simd8 other) const { return this->lt_bits(other).any_bits_set(); } // Bit-specific operations simdjson_inline simd8 bits_not_set() const { return _mm512_mask_blend_epi8(*this == uint8_t(0), _mm512_set1_epi8(0), _mm512_set1_epi8(-1)); } - simdjson_constexpr simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } - simdjson_constexpr simd8 any_bits_set() const { return ~this->bits_not_set(); } - simdjson_constexpr simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } + simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } + simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } + simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } - simdjson_constexpr bool is_ascii() const { return _mm512_movepi8_mask(*this) == 0; } - simdjson_constexpr bool bits_not_set_anywhere() const { + simdjson_inline bool is_ascii() const { return _mm512_movepi8_mask(*this) == 0; } + simdjson_inline bool bits_not_set_anywhere() const { return !_mm512_test_epi8_mask(*this, *this); } - simdjson_constexpr bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } - simdjson_constexpr bool bits_not_set_anywhere(simd8 bits) const { return !_mm512_test_epi8_mask(*this, bits); } - simdjson_constexpr bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } + simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } + simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { return !_mm512_test_epi8_mask(*this, bits); } + simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } template - simdjson_constexpr simd8 shr() const { return simd8(_mm512_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); } + simdjson_inline simd8 shr() const { return simd8(_mm512_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); } template - simdjson_constexpr simd8 shl() const { return simd8(_mm512_slli_epi16(*this, N)) & uint8_t(0xFFu << N); } + simdjson_inline simd8 shl() const { return simd8(_mm512_slli_epi16(*this, N)) & uint8_t(0xFFu << N); } // Get one of the bits and make a bitmask out of it. // e.g. value.get_bit<7>() gets the high bit template - simdjson_constexpr uint64_t get_bit() const { return _mm512_movepi8_mask(_mm512_slli_epi16(*this, 7-N)); } + simdjson_inline uint64_t get_bit() const { return _mm512_movepi8_mask(_mm512_slli_epi16(*this, 7-N)); } }; template @@ -373,58 +311,120 @@ namespace simd { simd8x64& operator=(const simd8& other) = delete; // no assignment allowed simd8x64() = delete; // no default constructor allowed - simdjson_constexpr simd8x64(const simd8 chunk0, const simd8 chunk1) : chunks{chunk0, chunk1} {} - simdjson_constexpr simd8x64(const simd8 chunk0) : chunks{chunk0} {} - simdjson_constexpr simd8x64(const T ptr[64]) : chunks{simd8::load(ptr)} {} + simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1) : chunks{chunk0, chunk1} {} + simdjson_inline simd8x64(const simd8 chunk0) : chunks{chunk0} {} + simdjson_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr)} {} + simdjson_inline simd8x64(simd8x64&& o) noexcept = default; + simdjson_inline simd8x64& operator=(simd8x64&& other) noexcept = default; - simdjson_constexpr uint64_t compress(uint64_t mask, T * output) const { + simdjson_inline uint64_t compress(uint64_t mask, T * output) const { this->chunks[0].compress(mask, output); return 64 - count_ones(mask); } - simdjson_constexpr void store(T ptr[64]) const { + simdjson_inline void store(T ptr[64]) const { this->chunks[0].store(ptr+sizeof(simd8)*0); } - simdjson_constexpr simd8 reduce_or() const { + simdjson_inline simd8 reduce_or() const { return this->chunks[0]; } - simdjson_constexpr simd8x64 bit_or(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] | mask - ); - } - - simdjson_constexpr uint64_t eq(const T m) const { + simdjson_inline uint64_t eq(const T m) const { const simd8 mask = simd8::splat(m); return this->chunks[0] == mask; } - simdjson_constexpr uint64_t eq(const simd8x64 &other) const { + simdjson_inline uint64_t eq(const simd8x64 &other) const { return this->chunks[0] == other.chunks[0]; } - simdjson_inline uint64_t lookup_16(simd8 lookup_table) const { + simdjson_inline simd8x64 lookup_16(const simd8& lookup_table) const { return { this->chunks[0].lookup_16(lookup_table) }; } - template - simdjson_inline uint64_t eq_any(V ...values) const { - static constexpr const simd8 LOOKUP_TABLE = create_eq_lookup_16_table({values...}); - return eq(lookup_16(LOOKUP_TABLE)); - } - - simdjson_constexpr uint64_t lteq(const T m) const { + simdjson_inline uint64_t lteq(const T m) const { const simd8 mask = simd8::splat(m); return this->chunks[0] <= mask; } + + simdjson_inline simd8x64 operator&(const simd8x64& other) const { + return { + this->chunks[0] & other.chunks[0] + }; + } + + simdjson_inline simd8x64 operator&(const simd8& other) const { + return { + this->chunks[0] & other + }; + } + + simdjson_inline simd8x64 operator|(const simd8x64& other) const { + return { + this->chunks[0] | other.chunks[0] + }; + } + + simdjson_inline simd8x64 operator|(const simd8& other) const { + return { + this->chunks[0] | other + }; + } + + simdjson_inline simd8x64 operator^(const simd8x64& other) const { + return { + this->chunks[0] ^ other.chunks[0] + }; + } + + simdjson_inline simd8x64 operator^(const simd8& other) const { + return { + this->chunks[0] ^ other + }; + } + + simdjson_inline simd8x64 bit_andnot(const simd8x64& other) const { + return { + this->chunks[0].bit_andnot(other.chunks[0]) + }; + } + + simdjson_inline simd8x64 bit_andnot(const simd8& other) const { + return { + this->chunks[0].bit_andnot(other) + }; + } + + template + simdjson_inline simd8x64 shr() const noexcept { + return { + this->chunks[0].template shr() + }; + } + + template + simdjson_inline simd8x64 shl() const noexcept { + return { + this->chunks[0].template shl() + }; + } + + simdjson_inline simd8x64 any_bits_set(const simd8& bits) const { + return { + this->chunks[0].any_bits_set(bits) + }; + } + + simdjson_inline simd8x64 any_bits_set(const simd8x64& bits) const { + return { + this->chunks[0].any_bits_set(bits.chunks[0]) + }; + } }; // struct simd8x64 } // namespace simd -} // unnamed namespace } // namespace icelake } // namespace simdjson diff --git a/include/simdjson/ppc64.h b/include/simdjson/ppc64.h index 6337118a1..aa68f6b2b 100644 --- a/include/simdjson/ppc64.h +++ b/include/simdjson/ppc64.h @@ -3,6 +3,7 @@ #include "simdjson/ppc64/begin.h" #include "simdjson/generic/amalgamated.h" +#include "simdjson/generic/lookup_table.h" #include "simdjson/ppc64/end.h" #endif // SIMDJSON_PPC64_H \ No newline at end of file diff --git a/include/simdjson/ppc64/base.h b/include/simdjson/ppc64/base.h index 301347065..8e001c5ce 100644 --- a/include/simdjson/ppc64/base.h +++ b/include/simdjson/ppc64/base.h @@ -13,7 +13,6 @@ namespace ppc64 { class implementation; -namespace { namespace simd { template struct simd8; @@ -22,7 +21,6 @@ template <> struct simd8; template struct simd8x64; } // namespace simd -} // unnamed namespace } // namespace ppc64 } // namespace simdjson diff --git a/include/simdjson/ppc64/simd.h b/include/simdjson/ppc64/simd.h index 5e077401c..4a29faa9d 100644 --- a/include/simdjson/ppc64/simd.h +++ b/include/simdjson/ppc64/simd.h @@ -11,51 +11,57 @@ namespace simdjson { namespace ppc64 { -namespace { namespace simd { -using __m128i = __vector unsigned char; +#if !(SIMDJSON_IS_PPC64 && SIMDJSON_IS_PPC64_VMX) && !defined(SIMDJSON_CONDITIONAL_INCLUDE) + // Make errors a bit more manageable when editing on non-ARM + struct __m128u { uint8_t buf[16]; }; + using __m128i = __m128u; +#else +using __m128u = __vector unsigned char; +using __m128i = __vector signed char; +#endif template struct base { - using simd_t = __m128i; + using simd_t = __m128u; simd_t value; // Zero constructor - simdjson_constexpr base() : value{simd_t()} {} + simdjson_inline base() : value{simd_t()} {} // Conversion from SIMD register - simdjson_constexpr base(const simd_t _value) : value(_value) {} + simdjson_inline base(const simd_t _value) : value(_value) {} // Conversion to SIMD register - simdjson_constexpr operator const simd_t &() const { + simdjson_inline operator const simd_t &() const { return this->value; } - simdjson_constexpr operator simd_t &() { return this->value; } + simdjson_inline operator simd_t &() { return this->value; } // Bit operations - simdjson_constexpr Child operator|(const Child other) const { + simdjson_inline Child operator|(const Child other) const { return vec_or(this->value, (simd_t)other); } - simdjson_constexpr Child operator&(const Child other) const { + simdjson_inline Child operator&(const Child other) const { return vec_and(this->value, (simd_t)other); } - simdjson_constexpr Child operator^(const Child other) const { + simdjson_inline Child operator^(const Child other) const { return vec_xor(this->value, (simd_t)other); } - simdjson_constexpr Child bit_andnot(const Child other) const { + simdjson_inline Child bit_andnot(const Child other) const { return vec_andc(this->value, (simd_t)other); } - simdjson_constexpr Child &operator|=(const Child other) { + simdjson_inline Child &operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } - simdjson_constexpr Child &operator&=(const Child other) { + simdjson_inline Child &operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } - simdjson_constexpr Child &operator^=(const Child other) { + simdjson_inline Child &operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; @@ -67,16 +73,16 @@ struct base8 : base> { using typename base>::simd_t; static constexpr const int LANES = sizeof(simd_t); using bitmask_t = uint16_t; - static_assert(sizeof(bitmask_t)*8 == LANES); + static_assert(sizeof(bitmask_t)*8 == LANES, "Bitmask type's bits must equal the simd type's bytes"); - simdjson_constexpr base8() : base>() {} - simdjson_constexpr base8(const simd_t _value) : base>(_value) {} + simdjson_inline base8() : base>() {} + simdjson_inline base8(const simd_t _value) : base>(_value) {} - simdjson_constexpr Mask eq(const simd8 rhs) const { return (simd_t)vec_cmpeq(this->value, (simd_t)rhs); } - friend simdjson_constexpr Mask operator==(const simd8 lhs, const simd8 rhs) { return lhs.eq(rhs); } + simdjson_inline Mask eq(const simd8 rhs) const { return (simd_t)vec_cmpeq(this->value, (simd_t)rhs); } + friend simdjson_inline Mask operator==(const simd8 lhs, const simd8 rhs) { return lhs.eq(rhs); } template - simdjson_constexpr simd8 prev(simd8 prev_chunk) const { + simdjson_inline simd8 prev(const simd8& prev_chunk) const { simd_t chunk = this->value; #ifdef __LITTLE_ENDIAN__ chunk = (simd_t)vec_reve(this->value); @@ -94,18 +100,18 @@ struct base8 : base> { template <> struct simd8 : base8 { using typename base8::simd_t; - static simdjson_constexpr simd8 splat(bool _value) { + static simdjson_inline simd8 splat(bool _value) { return (simd_t)vec_splats((unsigned char)(-(!!_value))); } - simdjson_constexpr simd8() : base8() {} - simdjson_constexpr simd8(const simd_t _value) + simdjson_inline simd8() : base8() {} + simdjson_inline simd8(const simd_t _value) : base8(_value) {} // Splat constructor - simdjson_constexpr simd8(bool _value) + simdjson_inline simd8(bool _value) : base8(splat(_value)) {} - simdjson_constexpr int to_bitmask() const { + simdjson_inline int to_bitmask() const { __vector unsigned long long result; const simd_t perm_mask = {0x78, 0x70, 0x68, 0x60, 0x58, 0x50, 0x48, 0x40, 0x38, 0x30, 0x28, 0x20, 0x18, 0x10, 0x08, 0x00}; @@ -118,10 +124,10 @@ template <> struct simd8 : base8 { return static_cast(result[0]); #endif } - simdjson_constexpr bool any() const { + simdjson_inline bool any() const { return !vec_all_eq(this->value, (simd_t)vec_splats(0)); } - simdjson_constexpr simd8 operator~() const { + simdjson_inline simd8 operator~() const { return this->value ^ (simd_t)splat(true); } }; @@ -130,16 +136,16 @@ template struct base8_numeric : base8 { using typename base8::simd_t; using base8::LANES; - static simdjson_constexpr simd8 splat(T value) { + static simdjson_inline simd8 splat(T value) { (void)value; return (simd_t)vec_splats(value); } - static simdjson_constexpr simd8 zero() { return splat(0); } - static simdjson_constexpr simd8 load(const T values[16]) { + static simdjson_inline simd8 zero() { return splat(0); } + static simdjson_inline simd8 load(const T values[16]) { return (simd_t)(vec_vsx_ld(0, reinterpret_cast(values))); } // Repeat 16 values as many times as necessary (usually for lookup tables) - static simdjson_constexpr simd8 repeat_16(T v0, T v1, T v2, T v3, T v4, + static simdjson_inline simd8 repeat_16(T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15) { @@ -147,84 +153,37 @@ template struct base8_numeric : base8 { v14, v15); } - simdjson_constexpr base8_numeric() : base8() {} - simdjson_constexpr base8_numeric(const simd_t _value) + simdjson_inline base8_numeric() : base8() {} + simdjson_inline base8_numeric(const simd_t _value) : base8(_value) {} // Store to array - simdjson_constexpr void store(T dst[16]) const { + simdjson_inline void store(T dst[16]) const { vec_vsx_st(this->value, 0, reinterpret_cast(dst)); } // Override to distinguish from bool version - simdjson_constexpr simd8 operator~() const { return *this ^ 0xFFu; } + simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } // Addition/subtraction are the same for signed and unsigned - simdjson_constexpr simd8 operator+(const simd8 other) const { + simdjson_inline simd8 operator+(const simd8 other) const { return (simd_t)((simd_t)this->value + (simd_t)other); } - simdjson_constexpr simd8 operator-(const simd8 other) const { + simdjson_inline simd8 operator-(const simd8 other) const { return (simd_t)((simd_t)this->value - (simd_t)other); } - simdjson_constexpr simd8 &operator+=(const simd8 other) { + simdjson_inline simd8 &operator+=(const simd8 other) { *this = *this + other; return *static_cast *>(this); } - simdjson_constexpr simd8 &operator-=(const simd8 other) { + simdjson_inline simd8 &operator-=(const simd8 other) { *this = *this - other; return *static_cast *>(this); } - struct lane_with_value { int lane; T value; }; - - /** - * Initialize a simd8 by filling in only specific lanes. - * - * @param entries A set of index/value pairs, like {{1, 'a'}, {2, 'b'}, ...} - * @param default_value The value to use for other lanes. - */ - static simdjson_constexpr simd8 create_sparse( - std::initializer_list entries, - T default_value = {} - ) noexcept { - bool filled[LANES] = {0}; - uint8_t table[LANES] = {default_value}; - for (auto [lane, value] : entries) { - assert(lane < LANES); - assert(!filled[lane]); - filled[lane] = true; - table[lane] = value; - } - return table; - } - - static simdjson_constexpr simd8 create_eq_lookup_16_table(std::initializer_list values) { - bool filled[16] = {0}; - - // Set the defaults to 0, except at 0 itself (which we set to 1 so it won't accidentally match 0). - uint8_t table[LANES] = {0}; - for (int lane = 0; lane < 16; lane += 16) { table[lane] = 1; } - - for (T value : values) { - int lane = value & 0x0F; - assert(!filled[lane]); - filled[lane] = true; - // Repeat the value at the same position in each 16-byte section of lanes. - for (; lane < LANES; lane += 16) { table[lane] = value; } - } - return table; - } - - template - simdjson_inline simd8 eq_any(V ...values) const { - static constexpr const simd8 LOOKUP_TABLE = create_eq_lookup_16_table({values...}); - return eq(lookup_16(LOOKUP_TABLE)); - } - // Perform a lookup assuming the value is between 0 and 16 (undefined behavior // for out of range values) - template - simdjson_constexpr simd8 lookup_16(simd8 lookup_table) const { + simdjson_inline simd8 lookup_16(const simd8& lookup_table) const { return (simd_t)vec_perm((simd_t)lookup_table, (simd_t)lookup_table, this->value); } @@ -235,7 +194,7 @@ template struct base8_numeric : base8 { // seems like a function with the signature simd8 compress(uint32_t mask) // would be sensible, but the AVX ISA makes this kind of approach difficult. template - simdjson_constexpr void compress(uint16_t mask, L *output) const { + simdjson_inline void compress(uint16_t mask, L *output) const { using internal::BitsSetTable256mul2; using internal::pshufb_combine_table; using internal::thintable_epi8; @@ -272,39 +231,27 @@ template struct base8_numeric : base8 { simd_t answer = vec_perm(pruned, (simd_t)vec_splats(0), compactmask); vec_vsx_st(answer, 0, reinterpret_cast(output)); } - - template - simdjson_constexpr simd8 - lookup_16(L replace0, L replace1, L replace2, L replace3, L replace4, - L replace5, L replace6, L replace7, L replace8, L replace9, - L replace10, L replace11, L replace12, L replace13, L replace14, - L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, replace4, replace5, replace6, - replace7, replace8, replace9, replace10, replace11, replace12, - replace13, replace14, replace15)); - } }; // Signed bytes template <> struct simd8 : base8_numeric { - simdjson_constexpr simd8() : base8_numeric() {} - simdjson_constexpr simd8(const simd_t _value) + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const simd_t _value) : base8_numeric(_value) {} // Splat constructor - simdjson_constexpr simd8(int8_t _value) : simd8(splat(_value)) {} + simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} // Array constructor - simdjson_constexpr simd8(const int8_t *values) : simd8(load(values)) {} + simdjson_inline simd8(const int8_t *values) : simd8(load(values)) {} // Member-by-member initialization - simdjson_constexpr simd8(int8_t v0, int8_t v1, int8_t v2, int8_t v3, + simdjson_inline simd8(int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15) - : simd8((simd_t)(__vector signed char){v0, v1, v2, v3, v4, v5, v6, v7, + : simd8((simd_t)(__m128i){v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15}) {} // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_constexpr static simd8 + simdjson_inline static simd8 repeat_16(int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15) { @@ -313,46 +260,46 @@ template <> struct simd8 : base8_numeric { } // Order-sensitive comparisons - simdjson_constexpr simd8 + simdjson_inline simd8 max_val(const simd8 other) const { - return (simd_t)vec_max((__vector signed char)this->value, - (__vector signed char)(simd_t)other); + return (simd_t)vec_max((__m128i)this->value, + (__m128i)(simd_t)other); } - simdjson_constexpr simd8 + simdjson_inline simd8 min_val(const simd8 other) const { - return (simd_t)vec_min((__vector signed char)this->value, - (__vector signed char)(simd_t)other); + return (simd_t)vec_min((__m128i)this->value, + (__m128i)(simd_t)other); } - simdjson_constexpr simd8 + simdjson_inline simd8 operator>(const simd8 other) const { - return (simd_t)vec_cmpgt((__vector signed char)this->value, - (__vector signed char)(simd_t)other); + return (simd_t)vec_cmpgt((__m128i)this->value, + (__m128i)(simd_t)other); } - simdjson_constexpr simd8 + simdjson_inline simd8 operator<(const simd8 other) const { - return (simd_t)vec_cmplt((__vector signed char)this->value, - (__vector signed char)(simd_t)other); + return (simd_t)vec_cmplt((__m128i)this->value, + (__m128i)(simd_t)other); } }; // Unsigned bytes template <> struct simd8 : base8_numeric { - simdjson_constexpr simd8() : base8_numeric() {} - simdjson_constexpr simd8(const simd_t _value) + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const simd_t _value) : base8_numeric(_value) {} // Splat constructor - simdjson_constexpr simd8(uint8_t _value) : simd8(splat(_value)) {} + simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} // Array constructor - simdjson_constexpr simd8(const uint8_t *values) : simd8(load(values)) {} + simdjson_inline simd8(const uint8_t *values) : simd8(load(values)) {} // Member-by-member initialization - simdjson_constexpr + simdjson_inline simd8(uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15) : simd8((simd_t){v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15}) {} // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_constexpr static simd8 + simdjson_inline static simd8 repeat_16(uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, @@ -362,82 +309,82 @@ template <> struct simd8 : base8_numeric { } // Saturated math - simdjson_constexpr simd8 + simdjson_inline simd8 saturating_add(const simd8 other) const { return (simd_t)vec_adds(this->value, (simd_t)other); } - simdjson_constexpr simd8 + simdjson_inline simd8 saturating_sub(const simd8 other) const { return (simd_t)vec_subs(this->value, (simd_t)other); } // Order-specific operations - simdjson_constexpr simd8 + simdjson_inline simd8 max_val(const simd8 other) const { return (simd_t)vec_max(this->value, (simd_t)other); } - simdjson_constexpr simd8 + simdjson_inline simd8 min_val(const simd8 other) const { return (simd_t)vec_min(this->value, (simd_t)other); } // Same as >, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_constexpr simd8 + simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } // Same as <, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_constexpr simd8 + simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } - simdjson_constexpr simd8 + simdjson_inline simd8 operator<=(const simd8 other) const { return other.max_val(*this) == other; } - simdjson_constexpr simd8 + simdjson_inline simd8 operator>=(const simd8 other) const { return other.min_val(*this) == other; } - simdjson_constexpr simd8 + simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } - simdjson_constexpr simd8 + simdjson_inline simd8 operator<(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } // Bit-specific operations - simdjson_constexpr simd8 bits_not_set() const { + simdjson_inline simd8 bits_not_set() const { return (simd_t)vec_cmpeq(this->value, (simd_t)vec_splats(uint8_t(0))); } - simdjson_constexpr simd8 bits_not_set(simd8 bits) const { + simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } - simdjson_constexpr simd8 any_bits_set() const { + simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } - simdjson_constexpr simd8 any_bits_set(simd8 bits) const { + simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } - simdjson_constexpr bool bits_not_set_anywhere() const { + simdjson_inline bool bits_not_set_anywhere() const { return vec_all_eq(this->value, (simd_t)vec_splats(0)); } - simdjson_constexpr bool any_bits_set_anywhere() const { + simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } - simdjson_constexpr bool bits_not_set_anywhere(simd8 bits) const { + simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { return vec_all_eq(vec_and(this->value, (simd_t)bits), (simd_t)vec_splats(0)); } - simdjson_constexpr bool any_bits_set_anywhere(simd8 bits) const { + simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } - template simdjson_constexpr simd8 shr() const { + template simdjson_inline simd8 shr() const { return simd8( (simd_t)vec_sr(this->value, (simd_t)vec_splat_u8(N))); } - template simdjson_constexpr simd8 shl() const { + template simdjson_inline simd8 shl() const { return simd8( (simd_t)vec_sl(this->value, (simd_t)vec_splat_u8(N))); } @@ -454,26 +401,28 @@ template struct simd8x64 { operator=(const simd8& other) = delete; // no assignment allowed simd8x64() = delete; // no default constructor allowed - simdjson_constexpr simd8x64(const simd8 chunk0, const simd8 chunk1, + simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1, const simd8 chunk2, const simd8 chunk3) : chunks{chunk0, chunk1, chunk2, chunk3} {} - simdjson_constexpr simd8x64(const T ptr[64]) + simdjson_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr), simd8::load(ptr + 16), simd8::load(ptr + 32), simd8::load(ptr + 48)} {} + simdjson_inline simd8x64(simd8x64&& o) noexcept = default; + simdjson_inline simd8x64& operator=(simd8x64&& other) noexcept = default; - simdjson_constexpr void store(T ptr[64]) const { + simdjson_inline void store(T ptr[64]) const { this->chunks[0].store(ptr + sizeof(simd8) * 0); this->chunks[1].store(ptr + sizeof(simd8) * 1); this->chunks[2].store(ptr + sizeof(simd8) * 2); this->chunks[3].store(ptr + sizeof(simd8) * 3); } - simdjson_constexpr simd8 reduce_or() const { + simdjson_inline simd8 reduce_or() const { return (this->chunks[0] | this->chunks[1]) | (this->chunks[2] | this->chunks[3]); } - simdjson_constexpr uint64_t compress(uint64_t mask, T *output) const { + simdjson_inline uint64_t compress(uint64_t mask, T *output) const { this->chunks[0].compress(uint16_t(mask), output); this->chunks[1].compress(uint16_t(mask >> 16), output + 16 - count_ones(mask & 0xFFFF)); @@ -484,7 +433,7 @@ template struct simd8x64 { return 64 - count_ones(mask); } - simdjson_constexpr uint64_t to_bitmask() const { + simdjson_inline uint64_t to_bitmask() const { uint64_t r0 = uint32_t(this->chunks[0].to_bitmask()); uint64_t r1 = this->chunks[1].to_bitmask(); uint64_t r2 = this->chunks[2].to_bitmask(); @@ -492,14 +441,14 @@ template struct simd8x64 { return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48); } - simdjson_constexpr uint64_t eq(const T m) const { + simdjson_inline uint64_t eq(const T m) const { const simd8 mask = simd8::splat(m); return simd8x64(this->chunks[0] == mask, this->chunks[1] == mask, this->chunks[2] == mask, this->chunks[3] == mask) .to_bitmask(); } - simdjson_constexpr uint64_t eq(const simd8x64 &other) const { + simdjson_inline uint64_t eq(const simd8x64 &other) const { return simd8x64(this->chunks[0] == other.chunks[0], this->chunks[1] == other.chunks[1], this->chunks[2] == other.chunks[2], @@ -507,7 +456,7 @@ template struct simd8x64 { .to_bitmask(); } - simdjson_inline simd8x64 lookup_16(simd8 lookup_table) const { + simdjson_inline simd8x64 lookup_16(const simd8& lookup_table) const { return { this->chunks[0].lookup_16(lookup_table), this->chunks[1].lookup_16(lookup_table), @@ -516,32 +465,125 @@ template struct simd8x64 { }; } - template - simdjson_inline uint64_t eq_any(V ...values) const { - static constexpr const simd8 LOOKUP_TABLE = simd8::create_eq_lookup_16_table({values...}); - return eq(lookup_16(LOOKUP_TABLE)); - } - - template - simdjson_inline uint64_t eq_any(V ...values) const { - return simd8x64( - this->chunks[0].eq_any(values...), - this->chunks[1].eq_any(values...), - this->chunks[2].eq_any(values...), - this->chunks[3].eq_any(values...) - ).to_bitmask(); - } - - simdjson_constexpr uint64_t lteq(const T m) const { + simdjson_inline uint64_t lteq(const T m) const { const simd8 mask = simd8::splat(m); return simd8x64(this->chunks[0] <= mask, this->chunks[1] <= mask, this->chunks[2] <= mask, this->chunks[3] <= mask) .to_bitmask(); } + + simdjson_inline simd8x64 operator&(const simd8x64& other) const { + return { + this->chunks[0] & other.chunks[0], + this->chunks[1] & other.chunks[1], + this->chunks[2] & other.chunks[2], + this->chunks[3] & other.chunks[3] + }; + } + + simdjson_inline simd8x64 operator&(const simd8& other) const { + return { + this->chunks[0] & other, + this->chunks[1] & other, + this->chunks[2] & other, + this->chunks[3] & other + }; + } + + simdjson_inline simd8x64 operator|(const simd8x64& other) const { + return { + this->chunks[0] | other.chunks[0], + this->chunks[1] | other.chunks[1], + this->chunks[2] | other.chunks[2], + this->chunks[3] | other.chunks[3] + }; + } + + simdjson_inline simd8x64 operator|(const simd8& other) const { + return { + this->chunks[0] | other, + this->chunks[1] | other, + this->chunks[2] | other, + this->chunks[3] | other + }; + } + + simdjson_inline simd8x64 operator^(const simd8x64& other) const { + return { + this->chunks[0] ^ other.chunks[0], + this->chunks[1] ^ other.chunks[1], + this->chunks[2] ^ other.chunks[2], + this->chunks[3] ^ other.chunks[3] + }; + } + + simdjson_inline simd8x64 operator^(const simd8& other) const { + return { + this->chunks[0] ^ other, + this->chunks[1] ^ other, + this->chunks[2] ^ other, + this->chunks[3] ^ other + }; + } + + simdjson_inline simd8x64 bit_andnot(const simd8x64& other) const { + return { + this->chunks[0].bit_andnot(other.chunks[0]), + this->chunks[1].bit_andnot(other.chunks[1]), + this->chunks[2].bit_andnot(other.chunks[2]), + this->chunks[3].bit_andnot(other.chunks[3]) + }; + } + + simdjson_inline simd8x64 bit_andnot(const simd8& other) const { + return { + this->chunks[0].bit_andnot(other), + this->chunks[1].bit_andnot(other), + this->chunks[2].bit_andnot(other), + this->chunks[3].bit_andnot(other) + }; + } + + template + simdjson_inline simd8x64 shr() const noexcept { + return { + this->chunks[0].template shr(), + this->chunks[1].template shr(), + this->chunks[2].template shr(), + this->chunks[3].template shr() + }; + } + + template + simdjson_inline simd8x64 shl() const noexcept { + return { + this->chunks[0].template shl(), + this->chunks[1].template shl(), + this->chunks[2].template shl(), + this->chunks[3].template shl() + }; + } + + simdjson_inline simd8x64 any_bits_set(const simd8& bits) const { + return { + this->chunks[0].any_bits_set(bits), + this->chunks[1].any_bits_set(bits), + this->chunks[2].any_bits_set(bits), + this->chunks[3].any_bits_set(bits) + }; + } + + simdjson_inline simd8x64 any_bits_set(const simd8x64& bits) const { + return { + this->chunks[0].any_bits_set(bits.chunks[0]), + this->chunks[1].any_bits_set(bits.chunks[1]), + this->chunks[2].any_bits_set(bits.chunks[2]), + this->chunks[3].any_bits_set(bits.chunks[3]) + }; + } }; // struct simd8x64 } // namespace simd -} // unnamed namespace } // namespace ppc64 } // namespace simdjson diff --git a/include/simdjson/westmere.h b/include/simdjson/westmere.h index f05ba1145..e19c7e19a 100644 --- a/include/simdjson/westmere.h +++ b/include/simdjson/westmere.h @@ -3,6 +3,7 @@ #include "simdjson/westmere/begin.h" #include "simdjson/generic/amalgamated.h" +#include "simdjson/generic/lookup_table.h" #include "simdjson/westmere/end.h" #endif // SIMDJSON_WESTMERE_H \ No newline at end of file diff --git a/include/simdjson/westmere/base.h b/include/simdjson/westmere/base.h index a0a36905b..ffad9dfde 100644 --- a/include/simdjson/westmere/base.h +++ b/include/simdjson/westmere/base.h @@ -14,7 +14,6 @@ namespace westmere { class implementation; -namespace { namespace simd { template struct simd8; @@ -23,7 +22,6 @@ template <> struct simd8; template struct simd8x64; } // namespace simd -} // unnamed namespace } // namespace westmere } // namespace simdjson diff --git a/include/simdjson/westmere/simd.h b/include/simdjson/westmere/simd.h index 2f3cd3ac8..641f07bb7 100644 --- a/include/simdjson/westmere/simd.h +++ b/include/simdjson/westmere/simd.h @@ -9,7 +9,6 @@ namespace simdjson { namespace westmere { -namespace { namespace simd { template @@ -18,23 +17,23 @@ namespace simd { simd_t value; // Zero constructor - simdjson_constexpr base() : value{simd_t()} {} + simdjson_inline base() : value{simd_t()} {} // Conversion from SIMD register - simdjson_constexpr base(const simd_t _value) : value(_value) {} + simdjson_inline base(const simd_t _value) : value(_value) {} // Conversion to SIMD register - simdjson_constexpr operator const simd_t&() const { return this->value; } - simdjson_constexpr operator simd_t&() { return this->value; } + simdjson_inline operator const simd_t&() const { return this->value; } + simdjson_inline operator simd_t&() { return this->value; } // Bit operations - simdjson_constexpr Child operator|(const Child other) const { return _mm_or_si128(*this, other); } - simdjson_constexpr Child operator&(const Child other) const { return _mm_and_si128(*this, other); } - simdjson_constexpr Child operator^(const Child other) const { return _mm_xor_si128(*this, other); } - simdjson_constexpr Child bit_andnot(const Child other) const { return _mm_andnot_si128(other, *this); } - simdjson_constexpr Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } - simdjson_constexpr Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } - simdjson_constexpr Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } + simdjson_inline Child operator|(const Child other) const { return _mm_or_si128(*this, other); } + simdjson_inline Child operator&(const Child other) const { return _mm_and_si128(*this, other); } + simdjson_inline Child operator^(const Child other) const { return _mm_xor_si128(*this, other); } + simdjson_inline Child bit_andnot(const Child other) const { return _mm_andnot_si128(other, *this); } + simdjson_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } + simdjson_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } + simdjson_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } }; template> @@ -42,15 +41,15 @@ namespace simd { using typename base>::simd_t; static constexpr const int LANES = sizeof(simd_t); using bitmask_t = uint16_t; - static_assert(sizeof(bitmask_t)*8 == LANES); + static_assert(sizeof(bitmask_t)*8 == LANES, "Bitmask type's bits must equal the simd type's bytes"); - simdjson_constexpr base8() : base>() {} - simdjson_constexpr base8(const simd_t _value) : base>(_value) {} + simdjson_inline base8() : base>() {} + simdjson_inline base8(const simd_t _value) : base>(_value) {} - friend simdjson_constexpr Mask operator==(const simd8 lhs, const simd8 rhs) { return _mm_cmpeq_epi8(lhs, rhs); } + friend simdjson_inline Mask operator==(const simd8 lhs, const simd8 rhs) { return _mm_cmpeq_epi8(lhs, rhs); } template - simdjson_constexpr simd8 prev(const simd8 prev_chunk) const { + simdjson_inline simd8 prev(const simd8 prev_chunk) const { return _mm_alignr_epi8(*this, prev_chunk, 16 - N); } }; @@ -60,16 +59,16 @@ namespace simd { struct simd8: base8 { using typename base8::simd_t; - static simdjson_constexpr simd8 splat(bool _value) { return _mm_set1_epi8(uint8_t(-(!!_value))); } + static simdjson_inline simd8 splat(bool _value) { return _mm_set1_epi8(uint8_t(-(!!_value))); } - simdjson_constexpr simd8() : base8() {} - simdjson_constexpr simd8(const simd_t _value) : base8(_value) {} + simdjson_inline simd8() : base8() {} + simdjson_inline simd8(const simd_t _value) : base8(_value) {} // Splat constructor - simdjson_constexpr simd8(bool _value) : base8(splat(_value)) {} + simdjson_inline simd8(bool _value) : base8(splat(_value)) {} - simdjson_constexpr auto to_bitmask() const { return _mm_movemask_epi8(*this); } - simdjson_constexpr bool any() const { return !_mm_testz_si128(*this, *this); } - simdjson_constexpr simd8 operator~() const { return *this ^ true; } + simdjson_inline auto to_bitmask() const { return _mm_movemask_epi8(*this); } + simdjson_inline bool any() const { return !_mm_testz_si128(*this, *this); } + simdjson_inline simd8 operator~() const { return *this ^ true; } }; template @@ -77,13 +76,13 @@ namespace simd { using typename base8::simd_t; using base8::LANES; - static simdjson_constexpr simd8 splat(T _value) { return _mm_set1_epi8(_value); } - static simdjson_constexpr simd8 zero() { return _mm_setzero_si128(); } - static simdjson_constexpr simd8 load(const T values[16]) { + static simdjson_inline simd8 splat(T _value) { return _mm_set1_epi8(_value); } + static simdjson_inline simd8 zero() { return _mm_setzero_si128(); } + static simdjson_inline simd8 load(const T values[16]) { return _mm_loadu_si128(reinterpret_cast(values)); } // Repeat 16 values as many times as necessary (usually for lookup tables) - static simdjson_constexpr simd8 repeat_16( + static simdjson_inline simd8 repeat_16( T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 ) { @@ -93,70 +92,23 @@ namespace simd { ); } - simdjson_constexpr base8_numeric() : base8() {} - simdjson_constexpr base8_numeric(const simd_t _value) : base8(_value) {} + simdjson_inline base8_numeric() : base8() {} + simdjson_inline base8_numeric(const simd_t _value) : base8(_value) {} // Store to array - simdjson_constexpr void store(T dst[16]) const { return _mm_storeu_si128(reinterpret_cast(dst), *this); } + simdjson_inline void store(T dst[16]) const { return _mm_storeu_si128(reinterpret_cast(dst), *this); } // Override to distinguish from bool version - simdjson_constexpr simd8 operator~() const { return *this ^ 0xFFu; } + simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } // Addition/subtraction are the same for signed and unsigned - simdjson_constexpr simd8 operator+(const simd8 other) const { return _mm_add_epi8(*this, other); } - simdjson_constexpr simd8 operator-(const simd8 other) const { return _mm_sub_epi8(*this, other); } - simdjson_constexpr simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } - simdjson_constexpr simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } - - struct lane_with_value { int lane; T value; }; - - /** - * Initialize a simd8 by filling in only specific lanes. - * - * @param entries A set of index/value pairs, like {{1, 'a'}, {2, 'b'}, ...} - * @param default_value The value to use for other lanes. - */ - static simdjson_constexpr simd8 create_sparse( - std::initializer_list entries, - T default_value = {} - ) noexcept { - bool filled[LANES] = {0}; - uint8_t table[LANES] = {default_value}; - for (auto [lane, value] : entries) { - assert(lane < LANES); - assert(!filled[lane]); - filled[lane] = true; - table[lane] = value; - } - return table; - } - - static simdjson_constexpr simd8 create_eq_lookup_16_table(std::initializer_list values) { - bool filled[16] = {0}; - - // Set the defaults to 0, except at 0 itself (which we set to 1 so it won't accidentally match 0). - uint8_t table[LANES] = {0}; - for (int lane = 0; lane < 16; lane += 16) { table[lane] = 1; } - - for (T value : values) { - int lane = value & 0x0F; - assert(!filled[lane]); - filled[lane] = true; - // Repeat the value at the same position in each 16-byte section of lanes. - for (; lane < LANES; lane += 16) { table[lane] = value; } - } - return table; - } - - template - simdjson_inline simd8 eq_any(V ...values) const { - static constexpr const simd8 LOOKUP_TABLE = create_eq_lookup_16_table({values...}); - return eq(lookup_16(LOOKUP_TABLE)); - } + simdjson_inline simd8 operator+(const simd8 other) const { return _mm_add_epi8(*this, other); } + simdjson_inline simd8 operator-(const simd8 other) const { return _mm_sub_epi8(*this, other); } + simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } + simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) - template - simdjson_constexpr simd8 lookup_16(simd8 lookup_table) const { + simdjson_inline simd8 lookup_16(const simd8& lookup_table) const { return _mm_shuffle_epi8(lookup_table, *this); } @@ -168,7 +120,7 @@ namespace simd { // signature simd8 compress(uint32_t mask) would be // sensible, but the AVX ISA makes this kind of approach difficult. template - simdjson_constexpr void compress(uint16_t mask, L * output) const { + simdjson_inline void compress(uint16_t mask, L * output) const { using internal::thintable_epi8; using internal::BitsSetTable256mul2; using internal::pshufb_combine_table; @@ -197,33 +149,19 @@ namespace simd { simd_t answer = _mm_shuffle_epi8(pruned, compactmask); _mm_storeu_si128(reinterpret_cast(output), answer); } - - template - simdjson_constexpr simd8 lookup_16( - L replace0, L replace1, L replace2, L replace3, - L replace4, L replace5, L replace6, L replace7, - L replace8, L replace9, L replace10, L replace11, - L replace12, L replace13, L replace14, L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, - replace4, replace5, replace6, replace7, - replace8, replace9, replace10, replace11, - replace12, replace13, replace14, replace15 - )); - } }; // Signed bytes template<> struct simd8 : base8_numeric { - simdjson_constexpr simd8() : base8_numeric() {} - simdjson_constexpr simd8(const simd_t _value) : base8_numeric(_value) {} + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const simd_t _value) : base8_numeric(_value) {} // Splat constructor - simdjson_constexpr simd8(int8_t _value) : simd8(splat(_value)) {} + simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} // Array constructor - simdjson_constexpr simd8(const int8_t* values) : simd8(load(values)) {} + simdjson_inline simd8(const int8_t* values) : simd8(load(values)) {} // Member-by-member initialization - simdjson_constexpr simd8( + simdjson_inline simd8( int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 ) : simd8(_mm_setr_epi8( @@ -231,7 +169,7 @@ namespace simd { v8, v9, v10,v11,v12,v13,v14,v15 )) {} // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_constexpr static simd8 repeat_16( + simdjson_inline static simd8 repeat_16( int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 ) { @@ -242,23 +180,23 @@ namespace simd { } // Order-sensitive comparisons - simdjson_constexpr simd8 max_val(const simd8 other) const { return _mm_max_epi8(*this, other); } - simdjson_constexpr simd8 min_val(const simd8 other) const { return _mm_min_epi8(*this, other); } - simdjson_constexpr simd8 operator>(const simd8 other) const { return _mm_cmpgt_epi8(*this, other); } - simdjson_constexpr simd8 operator<(const simd8 other) const { return _mm_cmpgt_epi8(other, *this); } + simdjson_inline simd8 max_val(const simd8 other) const { return _mm_max_epi8(*this, other); } + simdjson_inline simd8 min_val(const simd8 other) const { return _mm_min_epi8(*this, other); } + simdjson_inline simd8 operator>(const simd8 other) const { return _mm_cmpgt_epi8(*this, other); } + simdjson_inline simd8 operator<(const simd8 other) const { return _mm_cmpgt_epi8(other, *this); } }; // Unsigned bytes template<> struct simd8: base8_numeric { - simdjson_constexpr simd8() : base8_numeric() {} - simdjson_constexpr simd8(const simd_t _value) : base8_numeric(_value) {} + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const simd_t _value) : base8_numeric(_value) {} // Splat constructor - simdjson_constexpr simd8(uint8_t _value) : simd8(splat(_value)) {} + simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} // Array constructor - simdjson_constexpr simd8(const uint8_t* values) : simd8(load(values)) {} + simdjson_inline simd8(const uint8_t* values) : simd8(load(values)) {} // Member-by-member initialization - simdjson_constexpr simd8( + simdjson_inline simd8( uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 ) : simd8(_mm_setr_epi8( @@ -266,7 +204,7 @@ namespace simd { v8, v9, v10,v11,v12,v13,v14,v15 )) {} // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_constexpr static simd8 repeat_16( + simdjson_inline static simd8 repeat_16( uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 ) { @@ -277,39 +215,39 @@ namespace simd { } // Saturated math - simdjson_constexpr simd8 saturating_add(const simd8 other) const { return _mm_adds_epu8(*this, other); } - simdjson_constexpr simd8 saturating_sub(const simd8 other) const { return _mm_subs_epu8(*this, other); } + simdjson_inline simd8 saturating_add(const simd8 other) const { return _mm_adds_epu8(*this, other); } + simdjson_inline simd8 saturating_sub(const simd8 other) const { return _mm_subs_epu8(*this, other); } // Order-specific operations - simdjson_constexpr simd8 max_val(const simd8 other) const { return _mm_max_epu8(*this, other); } - simdjson_constexpr simd8 min_val(const simd8 other) const { return _mm_min_epu8(*this, other); } + simdjson_inline simd8 max_val(const simd8 other) const { return _mm_max_epu8(*this, other); } + simdjson_inline simd8 min_val(const simd8 other) const { return _mm_min_epu8(*this, other); } // Same as >, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_constexpr simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } + simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } // Same as <, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_constexpr simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } - simdjson_constexpr simd8 operator<=(const simd8 other) const { return other.max_val(*this) == other; } - simdjson_constexpr simd8 operator>=(const simd8 other) const { return other.min_val(*this) == other; } - simdjson_constexpr simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } - simdjson_constexpr simd8 operator<(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } + simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } + simdjson_inline simd8 operator<=(const simd8 other) const { return other.max_val(*this) == other; } + simdjson_inline simd8 operator>=(const simd8 other) const { return other.min_val(*this) == other; } + simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } + simdjson_inline simd8 operator<(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } // Bit-specific operations - simdjson_constexpr simd8 bits_not_set() const { return *this == uint8_t(0); } - simdjson_constexpr simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } - simdjson_constexpr simd8 any_bits_set() const { return ~this->bits_not_set(); } - simdjson_constexpr simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } - simdjson_constexpr bool is_ascii() const { return _mm_movemask_epi8(*this) == 0; } - simdjson_constexpr bool bits_not_set_anywhere() const { return _mm_testz_si128(*this, *this); } - simdjson_constexpr bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } - simdjson_constexpr bool bits_not_set_anywhere(simd8 bits) const { return _mm_testz_si128(*this, bits); } - simdjson_constexpr bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } + simdjson_inline simd8 bits_not_set() const { return *this == uint8_t(0); } + simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } + simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } + simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } + simdjson_inline bool is_ascii() const { return _mm_movemask_epi8(*this) == 0; } + simdjson_inline bool bits_not_set_anywhere() const { return _mm_testz_si128(*this, *this); } + simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } + simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { return _mm_testz_si128(*this, bits); } + simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } template - simdjson_constexpr simd8 shr() const { return simd8(_mm_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); } + simdjson_inline simd8 shr() const { return simd8(_mm_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); } template - simdjson_constexpr simd8 shl() const { return simd8(_mm_slli_epi16(*this, N)) & uint8_t(0xFFu << N); } + simdjson_inline simd8 shl() const { return simd8(_mm_slli_epi16(*this, N)) & uint8_t(0xFFu << N); } // Get one of the bits and make a bitmask out of it. // e.g. value.get_bit<7>() gets the high bit template - simdjson_constexpr int get_bit() const { return _mm_movemask_epi8(_mm_slli_epi16(*this, 7-N)); } + simdjson_inline int get_bit() const { return _mm_movemask_epi8(_mm_slli_epi16(*this, 7-N)); } }; template @@ -322,21 +260,23 @@ namespace simd { simd8x64& operator=(const simd8& other) = delete; // no assignment allowed simd8x64() = delete; // no default constructor allowed - simdjson_constexpr simd8x64(const simd8 chunk0, const simd8 chunk1, const simd8 chunk2, const simd8 chunk3) : chunks{chunk0, chunk1, chunk2, chunk3} {} - simdjson_constexpr simd8x64(const T ptr[64]) : chunks{simd8::load(ptr), simd8::load(ptr+16), simd8::load(ptr+32), simd8::load(ptr+48)} {} + simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1, const simd8 chunk2, const simd8 chunk3) : chunks{chunk0, chunk1, chunk2, chunk3} {} + simdjson_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr), simd8::load(ptr+16), simd8::load(ptr+32), simd8::load(ptr+48)} {} + simdjson_inline simd8x64(simd8x64&& o) noexcept = default; + simdjson_inline simd8x64& operator=(simd8x64&& other) noexcept = default; - simdjson_constexpr void store(T ptr[64]) const { + simdjson_inline void store(T ptr[64]) const { this->chunks[0].store(ptr+sizeof(simd8)*0); this->chunks[1].store(ptr+sizeof(simd8)*1); this->chunks[2].store(ptr+sizeof(simd8)*2); this->chunks[3].store(ptr+sizeof(simd8)*3); } - simdjson_constexpr simd8 reduce_or() const { + simdjson_inline simd8 reduce_or() const { return (this->chunks[0] | this->chunks[1]) | (this->chunks[2] | this->chunks[3]); } - simdjson_constexpr uint64_t compress(uint64_t mask, T * output) const { + simdjson_inline uint64_t compress(uint64_t mask, T * output) const { this->chunks[0].compress(uint16_t(mask), output); this->chunks[1].compress(uint16_t(mask >> 16), output + 16 - count_ones(mask & 0xFFFF)); this->chunks[2].compress(uint16_t(mask >> 32), output + 32 - count_ones(mask & 0xFFFFFFFF)); @@ -344,7 +284,7 @@ namespace simd { return 64 - count_ones(mask); } - simdjson_constexpr uint64_t to_bitmask() const { + simdjson_inline uint64_t to_bitmask() const { uint64_t r0 = uint32_t(this->chunks[0].to_bitmask() ); uint64_t r1 = this->chunks[1].to_bitmask() ; uint64_t r2 = this->chunks[2].to_bitmask() ; @@ -352,7 +292,7 @@ namespace simd { return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48); } - simdjson_constexpr uint64_t eq(const T m) const { + simdjson_inline uint64_t eq(const T m) const { const simd8 mask = simd8::splat(m); return simd8x64( this->chunks[0] == mask, @@ -362,7 +302,7 @@ namespace simd { ).to_bitmask(); } - simdjson_constexpr uint64_t eq(const simd8x64 &other) const { + simdjson_inline uint64_t eq(const simd8x64 &other) const { return simd8x64( this->chunks[0] == other.chunks[0], this->chunks[1] == other.chunks[1], @@ -371,22 +311,7 @@ namespace simd { ).to_bitmask(); } - simdjson_inline simd8x64 lookup_16(simd8 lookup_table) const { - return { - this->chunks[0].lookup_16(lookup_table), - this->chunks[1].lookup_16(lookup_table), - this->chunks[2].lookup_16(lookup_table), - this->chunks[3].lookup_16(lookup_table), - }; - } - - template - simdjson_inline uint64_t eq_any(V ...values) const { - static constexpr const simd8 LOOKUP_TABLE = simd8::create_eq_lookup_16_table({values...}); - return eq(lookup_16(LOOKUP_TABLE)); - } - - simdjson_constexpr uint64_t lteq(const T m) const { + simdjson_inline uint64_t lteq(const T m) const { const simd8 mask = simd8::splat(m); return simd8x64( this->chunks[0] <= mask, @@ -395,10 +320,128 @@ namespace simd { this->chunks[3] <= mask ).to_bitmask(); } + + simdjson_inline simd8x64 lookup_16(const simd8& lookup_table) const { + return { + this->chunks[0].lookup_16(lookup_table), + this->chunks[1].lookup_16(lookup_table), + this->chunks[2].lookup_16(lookup_table), + this->chunks[3].lookup_16(lookup_table), + }; + } + + simdjson_inline simd8x64 operator&(const simd8x64& other) const { + return { + this->chunks[0] & other.chunks[0], + this->chunks[1] & other.chunks[1], + this->chunks[2] & other.chunks[2], + this->chunks[3] & other.chunks[3] + }; + } + + simdjson_inline simd8x64 operator&(const simd8& other) const { + return { + this->chunks[0] & other, + this->chunks[1] & other, + this->chunks[2] & other, + this->chunks[3] & other + }; + } + + simdjson_inline simd8x64 operator|(const simd8x64& other) const { + return { + this->chunks[0] | other.chunks[0], + this->chunks[1] | other.chunks[1], + this->chunks[2] | other.chunks[2], + this->chunks[3] | other.chunks[3] + }; + } + + simdjson_inline simd8x64 operator|(const simd8& other) const { + return { + this->chunks[0] | other, + this->chunks[1] | other, + this->chunks[2] | other, + this->chunks[3] | other + }; + } + + simdjson_inline simd8x64 operator^(const simd8x64& other) const { + return { + this->chunks[0] ^ other.chunks[0], + this->chunks[1] ^ other.chunks[1], + this->chunks[2] ^ other.chunks[2], + this->chunks[3] ^ other.chunks[3] + }; + } + + simdjson_inline simd8x64 operator^(const simd8& other) const { + return { + this->chunks[0] ^ other, + this->chunks[1] ^ other, + this->chunks[2] ^ other, + this->chunks[3] ^ other + }; + } + + simdjson_inline simd8x64 bit_andnot(const simd8x64& other) const { + return { + this->chunks[0].bit_andnot(other.chunks[0]), + this->chunks[1].bit_andnot(other.chunks[1]), + this->chunks[2].bit_andnot(other.chunks[2]), + this->chunks[3].bit_andnot(other.chunks[3]) + }; + } + + simdjson_inline simd8x64 bit_andnot(const simd8& other) const { + return { + this->chunks[0].bit_andnot(other), + this->chunks[1].bit_andnot(other), + this->chunks[2].bit_andnot(other), + this->chunks[3].bit_andnot(other) + }; + } + + template + simdjson_inline simd8x64 shr() const noexcept { + return { + this->chunks[0].template shr(), + this->chunks[1].template shr(), + this->chunks[2].template shr(), + this->chunks[3].template shr() + }; + } + + template + simdjson_inline simd8x64 shl() const noexcept { + return { + this->chunks[0].template shl(), + this->chunks[1].template shl(), + this->chunks[2].template shl(), + this->chunks[3].template shl() + }; + } + + simdjson_inline simd8x64 any_bits_set(const simd8& bits) const { + return { + this->chunks[0].any_bits_set(bits), + this->chunks[1].any_bits_set(bits), + this->chunks[2].any_bits_set(bits), + this->chunks[3].any_bits_set(bits) + }; + } + + simdjson_inline simd8x64 any_bits_set(const simd8x64& bits) const { + return { + this->chunks[0].any_bits_set(bits.chunks[0]), + this->chunks[1].any_bits_set(bits.chunks[1]), + this->chunks[2].any_bits_set(bits.chunks[2]), + this->chunks[3].any_bits_set(bits.chunks[3]) + }; + } }; // struct simd8x64 } // namespace simd -} // unnamed namespace } // namespace westmere } // namespace simdjson diff --git a/src/arm64.cpp b/src/arm64.cpp index 2ece3784e..4cf531d42 100644 --- a/src/arm64.cpp +++ b/src/arm64.cpp @@ -37,6 +37,27 @@ namespace { using namespace simd; +enum op_whitespace_t : uint8_t { + OPEN_OR_CLOSE = 1u << 0, + COLON = 1u << 1, + COMMA = 1u << 2, + TAB_CR_LF = 1u << 3, + SPACE = 1u << 4, +}; + +simdjson_constinit byte_classifier OP_WHITESPACE_CLASSIFIER({ + _lookup_entry{ ' ', op_whitespace_t::SPACE }, + { '\t', op_whitespace_t::TAB_CR_LF }, + { '\r', op_whitespace_t::TAB_CR_LF }, + { '\n', op_whitespace_t::TAB_CR_LF }, + { ':', op_whitespace_t::COLON }, + { ',', op_whitespace_t::COMMA }, + { '{', op_whitespace_t::OPEN_OR_CLOSE }, + { '[', op_whitespace_t::OPEN_OR_CLOSE }, + { '}', op_whitespace_t::OPEN_OR_CLOSE }, + { ']', op_whitespace_t::OPEN_OR_CLOSE }, +}); + simdjson_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { // Functional programming causes trouble with Visual Studio. // Keeping this version in comments since it is much nicer: @@ -47,16 +68,7 @@ simdjson_inline json_character_block json_character_block::classify(const simd:: // auto shuf_hi = nib_hi.lookup_16(8, 0, 18, 4, 0, 1, 0, 1, 0, 0, 0, 3, 2, 1, 0, 0); // return shuf_lo & shuf_hi; // }); - const simd8 table1(16, 0, 0, 0, 0, 0, 0, 0, 0, 8, 12, 1, 2, 9, 0, 0); - const simd8 table2(8, 0, 18, 4, 0, 1, 0, 1, 0, 0, 0, 3, 2, 1, 0, 0); - - simd8x64 v( - (in.chunks[0] & 0xf).lookup_16(table1) & (in.chunks[0].shr<4>()).lookup_16(table2), - (in.chunks[1] & 0xf).lookup_16(table1) & (in.chunks[1].shr<4>()).lookup_16(table2), - (in.chunks[2] & 0xf).lookup_16(table1) & (in.chunks[2].shr<4>()).lookup_16(table2), - (in.chunks[3] & 0xf).lookup_16(table1) & (in.chunks[3].shr<4>()).lookup_16(table2) - ); - + simd8x64 op_whitespace = OP_WHITESPACE_CLASSIFIER[in]; // We compute whitespace and op separately. If the code later only use one or the // other, given the fact that all functions are aggressively inlined, we can @@ -74,18 +86,12 @@ simdjson_inline json_character_block json_character_block::classify(const simd:: // there is a small untaken optimization opportunity here. We deliberately // do not pick it up. - uint64_t op = simd8x64( - v.chunks[0].any_bits_set(0x7), - v.chunks[1].any_bits_set(0x7), - v.chunks[2].any_bits_set(0x7), - v.chunks[3].any_bits_set(0x7) + uint64_t op = op_whitespace.any_bits_set( + op_whitespace_t::SPACE | op_whitespace_t::TAB_CR_LF ).to_bitmask(); - uint64_t whitespace = simd8x64( - v.chunks[0].any_bits_set(0x18), - v.chunks[1].any_bits_set(0x18), - v.chunks[2].any_bits_set(0x18), - v.chunks[3].any_bits_set(0x18) + uint64_t whitespace = op_whitespace.any_bits_set( + op_whitespace_t::COLON | op_whitespace_t::COMMA | op_whitespace_t::OPEN_OR_CLOSE ).to_bitmask(); return { whitespace, op }; diff --git a/src/generic/stage1/utf8_lookup4_algorithm.h b/src/generic/stage1/utf8_lookup4_algorithm.h index 64578915a..32b3dc22d 100644 --- a/src/generic/stage1/utf8_lookup4_algorithm.h +++ b/src/generic/stage1/utf8_lookup4_algorithm.h @@ -4,6 +4,7 @@ #define SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H #include #include +#include #endif // SIMDJSON_CONDITIONAL_INCLUDE namespace simdjson { @@ -13,95 +14,95 @@ namespace utf8_validation { using namespace simd; - simdjson_inline simd8 check_special_cases(const simd8 input, const simd8 prev1) { // Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII) // Bit 1 = Too Long (ASCII followed by continuation) // Bit 2 = Overlong 3-byte // Bit 4 = Surrogate // Bit 5 = Overlong 2-byte // Bit 7 = Two Continuations - constexpr const uint8_t TOO_SHORT = 1<<0; // 11______ 0_______ - // 11______ 11______ - constexpr const uint8_t TOO_LONG = 1<<1; // 0_______ 10______ - constexpr const uint8_t OVERLONG_3 = 1<<2; // 11100000 100_____ - constexpr const uint8_t SURROGATE = 1<<4; // 11101101 101_____ - constexpr const uint8_t OVERLONG_2 = 1<<5; // 1100000_ 10______ - constexpr const uint8_t TWO_CONTS = 1<<7; // 10______ 10______ - constexpr const uint8_t TOO_LARGE = 1<<3; // 11110100 1001____ - // 11110100 101_____ - // 11110101 1001____ - // 11110101 101_____ - // 1111011_ 1001____ - // 1111011_ 101_____ - // 11111___ 1001____ - // 11111___ 101_____ - constexpr const uint8_t TOO_LARGE_1000 = 1<<6; - // 11110101 1000____ - // 1111011_ 1000____ - // 11111___ 1000____ - constexpr const uint8_t OVERLONG_4 = 1<<6; // 11110000 1000____ +static constexpr const uint8_t TOO_SHORT = 1<<0; // 11______ 0_______ + // 11______ 11______ +static constexpr const uint8_t TOO_LONG = 1<<1; // 0_______ 10______ +static constexpr const uint8_t OVERLONG_3 = 1<<2; // 11100000 100_____ +static constexpr const uint8_t SURROGATE = 1<<4; // 11101101 101_____ +static constexpr const uint8_t OVERLONG_2 = 1<<5; // 1100000_ 10______ +static constexpr const uint8_t TWO_CONTS = 1<<7; // 10______ 10______ +// TOO_LARGE is any 4-byte bigger than 11110100 1001____, and any 5+-byte +// We split it into two parts, recognizing 11110101 1000____ and up in TOO_LARGE_1000 +static constexpr const uint8_t TOO_LARGE = 1<<3; // 11110100 1001____ = 4-byte 100-111 01-11... + // 11110100 101_____ + // 11110101 1001____ + // 11110101 101_____ + // 1111011_ 1001____ + // 1111011_ 101_____ + // 11111___ 1001____ = 5-byte * 01-11... + // 11111___ 101_____ +static constexpr const uint8_t TOO_LARGE_1000 = 1<<6; + // 11110101 1000____ = 4-byte 101-111 00... + // 1111011_ 1000____ + // 11111___ 1000____ = 5-byte * 00 ... +simdjson_constinit uint8_t OVERLONG_4 = 1<<6; // 11110000 1000____ - const simd8 byte_1_high = prev1.shr<4>().lookup_16( - // 0_______ ________ - TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, - TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, - // 10______ ________ - TWO_CONTS, TWO_CONTS, TWO_CONTS, TWO_CONTS, - // 1100____ ________ - TOO_SHORT | OVERLONG_2, - // 1101____ ________ - TOO_SHORT, - // 1110____ ________ - TOO_SHORT | OVERLONG_3 | SURROGATE, - // 1111____ ________ - TOO_SHORT | TOO_LARGE | TOO_LARGE_1000 | OVERLONG_4 - ); - constexpr const uint8_t CARRY = TOO_SHORT | TOO_LONG | TWO_CONTS; // These all have ____ in byte 1 . - const simd8 byte_1_low = (prev1 & 0x0F).lookup_16( - // ____0000 ________ - CARRY | OVERLONG_3 | OVERLONG_2 | OVERLONG_4, - // ____0001 ________ - CARRY | OVERLONG_2, - // ____001_ ________ - CARRY, - CARRY, +simdjson_constinit byte_range ASCII = { 0b00000000, 0b01111111 }; +simdjson_constinit byte_range CONT = { 0b10000000, 0b10111111 }; +simdjson_constinit byte_range LEAD_2 = { 0b11000000, 0b11011111 }; +simdjson_constinit byte_range LEAD_3 = { 0b11100000, 0b11101111 }; +simdjson_constinit byte_range LEAD_4 = { 0b11110000, 0b11110111 }; +simdjson_constinit byte_range LEAD_5_PLUS = { 0b11111000, 0b11111111 }; +simdjson_constinit byte_range LEAD = LEAD_2 | LEAD_3 | LEAD_4 | LEAD_5_PLUS; - // ____0100 ________ - CARRY | TOO_LARGE, - // ____0101 ________ - CARRY | TOO_LARGE | TOO_LARGE_1000, - // ____011_ ________ - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, +simdjson_constinit const byte_classifier BYTE_1{ + { ASCII, TOO_LONG }, // 0_______ 10______ + { CONT, TWO_CONTS }, // 10______ 10______ + { LEAD, TOO_SHORT }, // 11______ 0_______ + // 11______ 11______ + { 0b11101101, SURROGATE }, // 11101101 101_____ - // ____1___ ________ - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - // ____1101 ________ - CARRY | TOO_LARGE | TOO_LARGE_1000 | SURROGATE, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000 - ); - const simd8 byte_2_high = input.shr<4>().lookup_16( - // ________ 0_______ - TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, - TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, + { { 0b11000000, 0b11000001 }, OVERLONG_2 }, // 1100000_ 10______ + { 0b11100000, OVERLONG_3 }, // 11100000 100_____ + { 0b11110000, OVERLONG_4 }, // 11110000 1000____ + { { 0b11110100, 0b11111111 }, TOO_LARGE }, // 11110100-1111____ 1001____-101_____ + { { 0b11110101, 0b11111111 }, TOO_LARGE_1000 } // 11110101-1111____ 1000____ +}; +simdjson_consteval bool bytes_match(byte_range range, uint8_t value) noexcept { + for (uint8_t i : range) { + if (BYTE_1[i] != value) { return false; } + } + return true; +} +// static_assert(bytes_match(ASCII, TOO_LONG)); +// static_assert(bytes_match(CONT, TWO_CONTS)); - // ________ 1000____ - TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE_1000 | OVERLONG_4, - // ________ 1001____ - TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE, - // ________ 101_____ - TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, - TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, +// static_assert(bytes_match({ 0b11000000, 0b11000001 }, OVERLONG_2 | TOO_SHORT)); +// static_assert(bytes_match({ 0b11000010, 0b11011111 }, TOO_SHORT )); - // ________ 11______ - TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT - ); - return (byte_1_high & byte_1_low & byte_2_high); +// static_assert(int(BYTE_1[0b11100000]) == int(OVERLONG_3 | TOO_SHORT)); +// static_assert(bytes_match({ 0b11100001, 0b11101100 }, TOO_SHORT)); +// static_assert(int(BYTE_1[0b11101101]) == int(SURROGATE | TOO_SHORT)); +// static_assert(bytes_match({ 0b11101110, 0b11101111 }, TOO_SHORT)); + +// static_assert(int(BYTE_1[0b11110000]) == int(OVERLONG_4 | TOO_SHORT)); +// static_assert(bytes_match({ 0b11110001, 0b11110011 }, TOO_SHORT)); +// static_assert(int(BYTE_1[0b11110100]) == int(TOO_SHORT | TOO_LARGE)); +// static_assert(bytes_match({ 0b11110101, 0b11111111 }, TOO_SHORT | TOO_LARGE | TOO_LARGE_1000)); +// static_assert(int(BYTE_1[0xf8]) & TOO_LARGE); + +static constexpr const high_nibble_lookup BYTE_2_HIGH{ + { CONT, TOO_LONG }, // 0_______ 10______ + { CONT, TWO_CONTS }, // 10______ 10______ + { ASCII, TOO_SHORT }, // 11______ 0_______ + { LEAD, TOO_SHORT }, // 11______ 11______ + { { 0b10100000, 0b10111111 }, SURROGATE }, // 11101101 101_____ + + { CONT, OVERLONG_2 }, // 1100000_ 10______ + { { 0b10000000, 0b10011111 }, OVERLONG_3 }, // 11100000 100_____ + { { 0b10000000, 0b10001111 }, OVERLONG_4 }, // 11110000 1000____ + { { 0b10010000, 0b10111111 }, TOO_LARGE }, // 11110100-1111____ 1001____-101_____ + { { 0b10000000, 0b10001111 }, TOO_LARGE_1000 }, // 11110101-1111____ 1000____ +}; + + simdjson_inline simd8 check_special_cases(const simd8 input, const simd8 prev1) { + return BYTE_1[prev1] & BYTE_2_HIGH[input]; } simdjson_inline simd8 check_multibyte_lengths(const simd8 input, const simd8 prev_input, const simd8 sc) { diff --git a/src/haswell.cpp b/src/haswell.cpp index 15216a76a..50bdde994 100644 --- a/src/haswell.cpp +++ b/src/haswell.cpp @@ -38,18 +38,27 @@ namespace { using namespace simd; +static simdjson_constinit low_nibble_lookup WS_MATCH{ + {' ', ' '}, + {'\t', '\t'}, + {'\n', '\n'}, + {'\r', '\r'}, +}; +static simdjson_constinit low_nibble_lookup OP_MATCH{ + {':', ':'}, + {',', ','}, + {'{', '{'}, + {'}', '}'}, + {'[', '{'}, + {']', '}'}, +}; + // This identifies structural characters (comma, colon, braces, brackets), // and ASCII white-space ('\r','\n','\t',' '). simdjson_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { - const uint64_t whitespace = in.eq_any(' ', '\t', '\n', '\r'); // Turn [ and ] into { and } - const simd8x64 curlified{ - in.chunks[0] | 0x20, - in.chunks[1] | 0x20 - }; - const uint64_t op = curlified.eq_any(',', ':', '{', '}'); - - return { whitespace, op }; + const simd8x64 curlified = in | 0x20; + return { in.eq(WS_MATCH[in]), curlified.eq(OP_MATCH[in]) }; } simdjson_inline bool is_ascii(const simd8x64& input) { diff --git a/src/icelake.cpp b/src/icelake.cpp index bd394c496..4aef968b9 100644 --- a/src/icelake.cpp +++ b/src/icelake.cpp @@ -43,22 +43,27 @@ namespace { using namespace simd; +static simdjson_constinit low_nibble_lookup WS_MATCH{ + {' ', ' '}, + {'\t', '\t'}, + {'\n', '\n'}, + {'\r', '\r'}, +}; +static simdjson_constinit low_nibble_lookup OP_MATCH{ + {':', ':'}, + {',', ','}, + {'{', '{'}, + {'}', '}'}, + {'[', '{'}, + {']', '}'}, +}; + // This identifies structural characters (comma, colon, braces, brackets), // and ASCII white-space ('\r','\n','\t',' '). simdjson_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { - // We compute whitespace and op separately. If later code only uses one or the - // other, given the fact that all functions are aggressively inlined, we can - // hope that useless computations will be omitted. This is namely case when - // minifying (we only need whitespace). - - const uint64_t whitespace = in.eq_any(' ', '\t', '\n', '\r'); // Turn [ and ] into { and } - const simd8x64 curlified{ - in.chunks[0] | 0x20 - }; - const uint64_t op = curlified.eq_any(',', ':', '{', '}'); - - return { whitespace, op }; + const simd8x64 curlified = in | 0x20; + return { in.eq(WS_MATCH[in]), curlified.eq(OP_MATCH[in]) }; } simdjson_inline bool is_ascii(const simd8x64& input) { diff --git a/src/ppc64.cpp b/src/ppc64.cpp index 769164c17..7f32cb9ea 100644 --- a/src/ppc64.cpp +++ b/src/ppc64.cpp @@ -37,29 +37,61 @@ namespace { using namespace simd; +enum op_whitespace_t : uint8_t { + OPEN_OR_CLOSE = 1u << 0, + COLON = 1u << 1, + COMMA = 1u << 2, + TAB_CR_LF = 1u << 3, + SPACE = 1u << 4, +}; + +simdjson_constinit byte_classifier OP_WHITESPACE_CLASSIFIER({ + _lookup_entry{ ' ', op_whitespace_t::SPACE }, + { '\t', op_whitespace_t::TAB_CR_LF }, + { '\r', op_whitespace_t::TAB_CR_LF }, + { '\n', op_whitespace_t::TAB_CR_LF }, + { ':', op_whitespace_t::COLON }, + { ',', op_whitespace_t::COMMA }, + { '{', op_whitespace_t::OPEN_OR_CLOSE }, + { '[', op_whitespace_t::OPEN_OR_CLOSE }, + { '}', op_whitespace_t::OPEN_OR_CLOSE }, + { ']', op_whitespace_t::OPEN_OR_CLOSE }, +}); + simdjson_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { - const simd8 table1(16, 0, 0, 0, 0, 0, 0, 0, 0, 8, 12, 1, 2, 9, 0, 0); - const simd8 table2(8, 0, 18, 4, 0, 1, 0, 1, 0, 0, 0, 3, 2, 1, 0, 0); + // Functional programming causes trouble with Visual Studio. + // Keeping this version in comments since it is much nicer: + // auto v = in.map([&](simd8 chunk) { + // auto nib_lo = chunk & 0xf; + // auto nib_hi = chunk.shr<4>(); + // auto shuf_lo = nib_lo.lookup_16(16, 0, 0, 0, 0, 0, 0, 0, 0, 8, 12, 1, 2, 9, 0, 0); + // auto shuf_hi = nib_hi.lookup_16(8, 0, 18, 4, 0, 1, 0, 1, 0, 0, 0, 3, 2, 1, 0, 0); + // return shuf_lo & shuf_hi; + // }); + simd8x64 op_whitespace = OP_WHITESPACE_CLASSIFIER[in]; - simd8x64 v( - (in.chunks[0] & 0xf).lookup_16(table1) & (in.chunks[0].shr<4>()).lookup_16(table2), - (in.chunks[1] & 0xf).lookup_16(table1) & (in.chunks[1].shr<4>()).lookup_16(table2), - (in.chunks[2] & 0xf).lookup_16(table1) & (in.chunks[2].shr<4>()).lookup_16(table2), - (in.chunks[3] & 0xf).lookup_16(table1) & (in.chunks[3].shr<4>()).lookup_16(table2) - ); + // We compute whitespace and op separately. If the code later only use one or the + // other, given the fact that all functions are aggressively inlined, we can + // hope that useless computations will be omitted. This is namely case when + // minifying (we only need whitespace). *However* if we only need spaces, + // it is likely that we will still compute 'v' above with two lookup_16: one + // could do it a bit cheaper. This is in contrast with the x64 implementations + // where we can, efficiently, do the white space and structural matching + // separately. One reason for this difference is that on ARM NEON, the table + // lookups either zero or leave unchanged the characters exceeding 0xF whereas + // on x64, the equivalent instruction (pshufb) automatically applies a mask, + // ignoring the 4 most significant bits. Thus the x64 implementation is + // optimized differently. This being said, if you use this code strictly + // just for minification (or just to identify the structural characters), + // there is a small untaken optimization opportunity here. We deliberately + // do not pick it up. - uint64_t op = simd8x64( - v.chunks[0].any_bits_set(0x7), - v.chunks[1].any_bits_set(0x7), - v.chunks[2].any_bits_set(0x7), - v.chunks[3].any_bits_set(0x7) + uint64_t op = op_whitespace.any_bits_set( + op_whitespace_t::SPACE | op_whitespace_t::TAB_CR_LF ).to_bitmask(); - uint64_t whitespace = simd8x64( - v.chunks[0].any_bits_set(0x18), - v.chunks[1].any_bits_set(0x18), - v.chunks[2].any_bits_set(0x18), - v.chunks[3].any_bits_set(0x18) + uint64_t whitespace = op_whitespace.any_bits_set( + op_whitespace_t::COLON | op_whitespace_t::COMMA | op_whitespace_t::OPEN_OR_CLOSE ).to_bitmask(); return { whitespace, op }; diff --git a/src/westmere.cpp b/src/westmere.cpp index 54a2d2b53..c38e029b3 100644 --- a/src/westmere.cpp +++ b/src/westmere.cpp @@ -38,16 +38,27 @@ namespace { using namespace simd; -simdjson_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { - const uint64_t whitespace = in.eq_any(' ', '\t', '\n', '\r'); - // Turn [ and ] into { and } - const simd8x64 curlified{ - in.chunks[0] | 0x20, - in.chunks[1] | 0x20 - }; - const uint64_t op = curlified.eq_any(',', ':', '{', '}'); +static simdjson_constinit low_nibble_lookup WS_MATCH{ + {' ', ' '}, + {'\t', '\t'}, + {'\n', '\n'}, + {'\r', '\r'}, +}; +static simdjson_constinit low_nibble_lookup OP_MATCH{ + {':', ':'}, + {',', ','}, + {'{', '{'}, + {'}', '}'}, + {'[', '{'}, + {']', '}'}, +}; - return { whitespace, op }; +// This identifies structural characters (comma, colon, braces, brackets), +// and ASCII white-space ('\r','\n','\t',' '). +simdjson_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { + // Turn [ and ] into { and } + const simd8x64 curlified = in | 0x20; + return { in.eq(WS_MATCH[in]), curlified.eq(OP_MATCH[in]) }; } simdjson_inline bool is_ascii(const simd8x64& input) { diff --git a/tests/unicode_tests.cpp b/tests/unicode_tests.cpp index c1722f10a..5dec061bc 100644 --- a/tests/unicode_tests.cpp +++ b/tests/unicode_tests.cpp @@ -237,7 +237,11 @@ void test() { for (size_t i = 0; i < sizeof(badsequences)/sizeof(badsequences[0]); i++) { size_t len = std::strlen(badsequences[i]); if (simdjson::validate_utf8(badsequences[i], len)) { - printf("bug lookup2 badsequences[%zu]\n", i); + printf("bug lookup2 badsequences[%zu] ", i); + for (size_t j = 0; j < len; j++) { + printf("%02x ", (unsigned char)badsequences[i][j]); + } + printf("\n"); abort(); } }