From b01222518d40fda8acfa8907244ccdbc82ee196d Mon Sep 17 00:00:00 2001 From: John Keiser Date: Mon, 19 Aug 2019 20:51:52 -0700 Subject: [PATCH] Genericize bitmask building to make algorithms clearer --- src/haswell/simd_input.h | 23 ++++--- src/haswell/stage1_find_marks.h | 113 +++++++++++++------------------ src/westmere/simd_input.h | 33 ++++----- src/westmere/stage1_find_marks.h | 47 +++---------- 4 files changed, 85 insertions(+), 131 deletions(-) diff --git a/src/haswell/simd_input.h b/src/haswell/simd_input.h index 2d7ead556..8175af286 100644 --- a/src/haswell/simd_input.h +++ b/src/haswell/simd_input.h @@ -18,22 +18,25 @@ struct simd_input { this->hi = _mm256_loadu_si256(reinterpret_cast(ptr + 32)); } + template + really_inline uint64_t build_bitmask(F const& chunk_to_mask) { + uint64_t r0 = static_cast(_mm256_movemask_epi8(chunk_to_mask(this->lo))); + uint64_t r1 = _mm256_movemask_epi8(chunk_to_mask(this->hi)); + return r0 | (r1 << 32); + } + really_inline uint64_t eq(uint8_t m) { const __m256i mask = _mm256_set1_epi8(m); - __m256i cmp_res_0 = _mm256_cmpeq_epi8(this->lo, mask); - uint64_t res_0 = static_cast(_mm256_movemask_epi8(cmp_res_0)); - __m256i cmp_res_1 = _mm256_cmpeq_epi8(this->hi, mask); - uint64_t res_1 = _mm256_movemask_epi8(cmp_res_1); - return res_0 | (res_1 << 32); + return this->build_bitmask([&] (auto chunk) { + return _mm256_cmpeq_epi8(chunk, mask); + }); } really_inline uint64_t lteq(uint8_t m) { const __m256i maxval = _mm256_set1_epi8(m); - __m256i cmp_res_0 = _mm256_cmpeq_epi8(_mm256_max_epu8(maxval, this->lo), maxval); - uint64_t res_0 = static_cast(_mm256_movemask_epi8(cmp_res_0)); - __m256i cmp_res_1 = _mm256_cmpeq_epi8(_mm256_max_epu8(maxval, this->hi), maxval); - uint64_t res_1 = _mm256_movemask_epi8(cmp_res_1); - return res_0 | (res_1 << 32); + return this->build_bitmask([&] (auto chunk) { + return _mm256_cmpeq_epi8(_mm256_max_epu8(maxval, chunk), maxval); + }); } }; // struct simd_input diff --git a/src/haswell/stage1_find_marks.h b/src/haswell/stage1_find_marks.h index 612b7e58e..a2c776f36 100644 --- a/src/haswell/stage1_find_marks.h +++ b/src/haswell/stage1_find_marks.h @@ -25,77 +25,60 @@ static really_inline void find_whitespace_and_structurals(simd_input(_mm256_movemask_epi8(struct_lo)); - uint64_t structural_res_1 = _mm256_movemask_epi8(struct_hi); - structurals = (structural_res_0 | (structural_res_1 << 32)); - const __m256i mask_space = _mm256_set1_epi8(0x20); - __m256i space_lo = _mm256_cmpeq_epi8(in.lo, mask_space); - __m256i space_hi = _mm256_cmpeq_epi8(in.hi, mask_space); - const __m256i mask_linefeed = _mm256_set1_epi8(0x0a); - space_lo = _mm256_or_si256(space_lo, _mm256_cmpeq_epi8(in.lo, mask_linefeed)); - space_hi = _mm256_or_si256(space_hi, _mm256_cmpeq_epi8(in.hi, mask_linefeed)); - const __m256i mask_tab = _mm256_set1_epi8(0x09); - space_lo = _mm256_or_si256(space_lo, _mm256_cmpeq_epi8(in.lo, mask_tab)); - space_hi = _mm256_or_si256(space_hi, _mm256_cmpeq_epi8(in.hi, mask_tab)); - const __m256i mask_carriage = _mm256_set1_epi8(0x0d); - space_lo = _mm256_or_si256(space_lo, _mm256_cmpeq_epi8(in.lo, mask_carriage)); - space_hi = _mm256_or_si256(space_hi, _mm256_cmpeq_epi8(in.hi, mask_carriage)); + // You should never need this naive approach, but it can be useful + // for research purposes + const __m256i mask_open_brace = _mm256_set1_epi8(0x7b); + const __m256i mask_close_brace = _mm256_set1_epi8(0x7d); + const __m256i mask_open_bracket = _mm256_set1_epi8(0x5b); + const __m256i mask_close_bracket = _mm256_set1_epi8(0x5d); + const __m256i mask_column = _mm256_set1_epi8(0x3a); + const __m256i mask_comma = _mm256_set1_epi8(0x2c); + structurals = in->build_bitmask([&](auto in) { + __m256i structurals = _mm256_cmpeq_epi8(in, mask_open_brace); + structurals = _mm256_or_si256(structurals, _mm256_cmpeq_epi8(in, mask_close_brace)); + structurals = _mm256_or_si256(structurals, _mm256_cmpeq_epi8(in, mask_open_bracket)); + structurals = _mm256_or_si256(structurals, _mm256_cmpeq_epi8(in, mask_close_bracket)); + structurals = _mm256_or_si256(structurals, _mm256_cmpeq_epi8(in, mask_column)); + structurals = _mm256_or_si256(structurals, _mm256_cmpeq_epi8(in, mask_comma)); + return structurals; + }); - uint64_t ws_res_0 = static_cast(_mm256_movemask_epi8(space_lo)); - uint64_t ws_res_1 = _mm256_movemask_epi8(space_hi); - whitespace = (ws_res_0 | (ws_res_1 << 32)); - // end of naive approach + const __m256i mask_space = _mm256_set1_epi8(0x20); + const __m256i mask_linefeed = _mm256_set1_epi8(0x0a); + const __m256i mask_tab = _mm256_set1_epi8(0x09); + const __m256i mask_carriage = _mm256_set1_epi8(0x0d); + whitespace = in->build_bitmask([&](auto in) { + __m256i space = _mm256_cmpeq_epi8(in, mask_space); + space = _mm256_or_si256(space, _mm256_cmpeq_epi8(in, mask_linefeed)); + space = _mm256_or_si256(space, _mm256_cmpeq_epi8(in, mask_tab)); + space = _mm256_or_si256(space, _mm256_cmpeq_epi8(in, mask_carriage)); + }); + // end of naive approach #else // SIMDJSON_NAIVE_STRUCTURAL - // clang-format off - const __m256i structural_table = - _mm256_setr_epi8(44, 125, 0, 0, 0xc0u, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 123, - 44, 125, 0, 0, 0xc0u, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 123); - const __m256i white_table = _mm256_setr_epi8( - 32, 100, 100, 100, 17, 100, 113, 2, 100, 9, 10, 112, 100, 13, 100, 100, - 32, 100, 100, 100, 17, 100, 113, 2, 100, 9, 10, 112, 100, 13, 100, 100); - // clang-format on - const __m256i struct_offset = _mm256_set1_epi8(0xd4u); - const __m256i struct_mask = _mm256_set1_epi8(32); - __m256i lo_white = _mm256_cmpeq_epi8(in.lo, _mm256_shuffle_epi8(white_table, in.lo)); - __m256i hi_white = _mm256_cmpeq_epi8(in.hi, _mm256_shuffle_epi8(white_table, in.hi)); - uint64_t ws_res_0 = static_cast(_mm256_movemask_epi8(lo_white)); - uint64_t ws_res_1 = _mm256_movemask_epi8(hi_white); - whitespace = (ws_res_0 | (ws_res_1 << 32)); - __m256i lo_struct_r1 = _mm256_add_epi8(struct_offset, in.lo); - __m256i hi_struct_r1 = _mm256_add_epi8(struct_offset, in.hi); - __m256i lo_struct_r2 = _mm256_or_si256(in.lo, struct_mask); - __m256i hi_struct_r2 = _mm256_or_si256(in.hi, struct_mask); - __m256i lo_struct_r3 = _mm256_shuffle_epi8(structural_table, lo_struct_r1); - __m256i hi_struct_r3 = _mm256_shuffle_epi8(structural_table, hi_struct_r1); - __m256i lo_struct = _mm256_cmpeq_epi8(lo_struct_r2, lo_struct_r3); - __m256i hi_struct = _mm256_cmpeq_epi8(hi_struct_r2, hi_struct_r3); + // clang-format off + const __m256i structural_table = + _mm256_setr_epi8(44, 125, 0, 0, 0xc0u, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 123, + 44, 125, 0, 0, 0xc0u, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 123); + const __m256i white_table = _mm256_setr_epi8( + 32, 100, 100, 100, 17, 100, 113, 2, 100, 9, 10, 112, 100, 13, 100, 100, + 32, 100, 100, 100, 17, 100, 113, 2, 100, 9, 10, 112, 100, 13, 100, 100); + // clang-format on + const __m256i struct_offset = _mm256_set1_epi8(0xd4u); + const __m256i struct_mask = _mm256_set1_epi8(32); + + whitespace = in.build_bitmask([&](auto chunk) { + return _mm256_cmpeq_epi8(chunk, _mm256_shuffle_epi8(white_table, chunk)); + }); + structurals = in.build_bitmask([&](auto chunk) { + __m256i struct_r1 = _mm256_add_epi8(struct_offset, chunk); + __m256i struct_r2 = _mm256_or_si256(chunk, struct_mask); + __m256i struct_r3 = _mm256_shuffle_epi8(structural_table, struct_r1); + return _mm256_cmpeq_epi8(struct_r2, struct_r3); + }); - uint64_t structural_res_0 = static_cast(_mm256_movemask_epi8(lo_struct)); - uint64_t structural_res_1 = _mm256_movemask_epi8(hi_struct); - structurals = (structural_res_0 | (structural_res_1 << 32)); #endif // else SIMDJSON_NAIVE_STRUCTURAL } diff --git a/src/westmere/simd_input.h b/src/westmere/simd_input.h index f17ed2bf0..98ca01811 100644 --- a/src/westmere/simd_input.h +++ b/src/westmere/simd_input.h @@ -22,30 +22,27 @@ struct simd_input { this->v3 = _mm_loadu_si128(reinterpret_cast(ptr + 48)); } + template + really_inline uint64_t build_bitmask(F const& chunk_to_mask) { + uint64_t r0 = static_cast(_mm_movemask_epi8(chunk_to_mask(this->v0))); + uint64_t r1 = _mm_movemask_epi8(chunk_to_mask(this->v1)); + uint64_t r2 = _mm_movemask_epi8(chunk_to_mask(this->v2)); + uint64_t r3 = _mm_movemask_epi8(chunk_to_mask(this->v3)); + return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48); + } + really_inline uint64_t eq(uint8_t m) { const __m128i mask = _mm_set1_epi8(m); - __m128i cmp_res_0 = _mm_cmpeq_epi8(this->v0, mask); - uint64_t res_0 = _mm_movemask_epi8(cmp_res_0); - __m128i cmp_res_1 = _mm_cmpeq_epi8(this->v1, mask); - uint64_t res_1 = _mm_movemask_epi8(cmp_res_1); - __m128i cmp_res_2 = _mm_cmpeq_epi8(this->v2, mask); - uint64_t res_2 = _mm_movemask_epi8(cmp_res_2); - __m128i cmp_res_3 = _mm_cmpeq_epi8(this->v3, mask); - uint64_t res_3 = _mm_movemask_epi8(cmp_res_3); - return res_0 | (res_1 << 16) | (res_2 << 32) | (res_3 << 48); + return this->build_bitmask([&](auto chunk) { + return _mm_cmpeq_epi8(chunk, mask); + }); } really_inline uint64_t lteq(uint8_t m) { const __m128i maxval = _mm_set1_epi8(m); - __m128i cmp_res_0 = _mm_cmpeq_epi8(_mm_max_epu8(maxval, this->v0), maxval); - uint64_t res_0 = _mm_movemask_epi8(cmp_res_0); - __m128i cmp_res_1 = _mm_cmpeq_epi8(_mm_max_epu8(maxval, this->v1), maxval); - uint64_t res_1 = _mm_movemask_epi8(cmp_res_1); - __m128i cmp_res_2 = _mm_cmpeq_epi8(_mm_max_epu8(maxval, this->v2), maxval); - uint64_t res_2 = _mm_movemask_epi8(cmp_res_2); - __m128i cmp_res_3 = _mm_cmpeq_epi8(_mm_max_epu8(maxval, this->v3), maxval); - uint64_t res_3 = _mm_movemask_epi8(cmp_res_3); - return res_0 | (res_1 << 16) | (res_2 << 32) | (res_3 << 48); + return this->build_bitmask([&](auto chunk) { + return _mm_cmpeq_epi8(_mm_max_epu8(maxval, chunk), maxval); + }); } }; // struct simd_input diff --git a/src/westmere/stage1_find_marks.h b/src/westmere/stage1_find_marks.h index d2affcda7..10e162a32 100644 --- a/src/westmere/stage1_find_marks.h +++ b/src/westmere/stage1_find_marks.h @@ -28,45 +28,16 @@ static really_inline void find_whitespace_and_structurals(simd_input