From b01222518d40fda8acfa8907244ccdbc82ee196d Mon Sep 17 00:00:00 2001 From: John Keiser Date: Mon, 19 Aug 2019 20:51:52 -0700 Subject: [PATCH 1/9] Genericize bitmask building to make algorithms clearer --- src/haswell/simd_input.h | 23 ++++--- src/haswell/stage1_find_marks.h | 113 +++++++++++++------------------ src/westmere/simd_input.h | 33 ++++----- src/westmere/stage1_find_marks.h | 47 +++---------- 4 files changed, 85 insertions(+), 131 deletions(-) diff --git a/src/haswell/simd_input.h b/src/haswell/simd_input.h index 2d7ead556..8175af286 100644 --- a/src/haswell/simd_input.h +++ b/src/haswell/simd_input.h @@ -18,22 +18,25 @@ struct simd_input { this->hi = _mm256_loadu_si256(reinterpret_cast(ptr + 32)); } + template + really_inline uint64_t build_bitmask(F const& chunk_to_mask) { + uint64_t r0 = static_cast(_mm256_movemask_epi8(chunk_to_mask(this->lo))); + uint64_t r1 = _mm256_movemask_epi8(chunk_to_mask(this->hi)); + return r0 | (r1 << 32); + } + really_inline uint64_t eq(uint8_t m) { const __m256i mask = _mm256_set1_epi8(m); - __m256i cmp_res_0 = _mm256_cmpeq_epi8(this->lo, mask); - uint64_t res_0 = static_cast(_mm256_movemask_epi8(cmp_res_0)); - __m256i cmp_res_1 = _mm256_cmpeq_epi8(this->hi, mask); - uint64_t res_1 = _mm256_movemask_epi8(cmp_res_1); - return res_0 | (res_1 << 32); + return this->build_bitmask([&] (auto chunk) { + return _mm256_cmpeq_epi8(chunk, mask); + }); } really_inline uint64_t lteq(uint8_t m) { const __m256i maxval = _mm256_set1_epi8(m); - __m256i cmp_res_0 = _mm256_cmpeq_epi8(_mm256_max_epu8(maxval, this->lo), maxval); - uint64_t res_0 = static_cast(_mm256_movemask_epi8(cmp_res_0)); - __m256i cmp_res_1 = _mm256_cmpeq_epi8(_mm256_max_epu8(maxval, this->hi), maxval); - uint64_t res_1 = _mm256_movemask_epi8(cmp_res_1); - return res_0 | (res_1 << 32); + return this->build_bitmask([&] (auto chunk) { + return _mm256_cmpeq_epi8(_mm256_max_epu8(maxval, chunk), maxval); + }); } }; // struct simd_input diff --git a/src/haswell/stage1_find_marks.h b/src/haswell/stage1_find_marks.h index 612b7e58e..a2c776f36 100644 --- a/src/haswell/stage1_find_marks.h +++ b/src/haswell/stage1_find_marks.h @@ -25,77 +25,60 @@ static really_inline void find_whitespace_and_structurals(simd_input(_mm256_movemask_epi8(struct_lo)); - uint64_t structural_res_1 = _mm256_movemask_epi8(struct_hi); - structurals = (structural_res_0 | (structural_res_1 << 32)); - const __m256i mask_space = _mm256_set1_epi8(0x20); - __m256i space_lo = _mm256_cmpeq_epi8(in.lo, mask_space); - __m256i space_hi = _mm256_cmpeq_epi8(in.hi, mask_space); - const __m256i mask_linefeed = _mm256_set1_epi8(0x0a); - space_lo = _mm256_or_si256(space_lo, _mm256_cmpeq_epi8(in.lo, mask_linefeed)); - space_hi = _mm256_or_si256(space_hi, _mm256_cmpeq_epi8(in.hi, mask_linefeed)); - const __m256i mask_tab = _mm256_set1_epi8(0x09); - space_lo = _mm256_or_si256(space_lo, _mm256_cmpeq_epi8(in.lo, mask_tab)); - space_hi = _mm256_or_si256(space_hi, _mm256_cmpeq_epi8(in.hi, mask_tab)); - const __m256i mask_carriage = _mm256_set1_epi8(0x0d); - space_lo = _mm256_or_si256(space_lo, _mm256_cmpeq_epi8(in.lo, mask_carriage)); - space_hi = _mm256_or_si256(space_hi, _mm256_cmpeq_epi8(in.hi, mask_carriage)); + // You should never need this naive approach, but it can be useful + // for research purposes + const __m256i mask_open_brace = _mm256_set1_epi8(0x7b); + const __m256i mask_close_brace = _mm256_set1_epi8(0x7d); + const __m256i mask_open_bracket = _mm256_set1_epi8(0x5b); + const __m256i mask_close_bracket = _mm256_set1_epi8(0x5d); + const __m256i mask_column = _mm256_set1_epi8(0x3a); + const __m256i mask_comma = _mm256_set1_epi8(0x2c); + structurals = in->build_bitmask([&](auto in) { + __m256i structurals = _mm256_cmpeq_epi8(in, mask_open_brace); + structurals = _mm256_or_si256(structurals, _mm256_cmpeq_epi8(in, mask_close_brace)); + structurals = _mm256_or_si256(structurals, _mm256_cmpeq_epi8(in, mask_open_bracket)); + structurals = _mm256_or_si256(structurals, _mm256_cmpeq_epi8(in, mask_close_bracket)); + structurals = _mm256_or_si256(structurals, _mm256_cmpeq_epi8(in, mask_column)); + structurals = _mm256_or_si256(structurals, _mm256_cmpeq_epi8(in, mask_comma)); + return structurals; + }); - uint64_t ws_res_0 = static_cast(_mm256_movemask_epi8(space_lo)); - uint64_t ws_res_1 = _mm256_movemask_epi8(space_hi); - whitespace = (ws_res_0 | (ws_res_1 << 32)); - // end of naive approach + const __m256i mask_space = _mm256_set1_epi8(0x20); + const __m256i mask_linefeed = _mm256_set1_epi8(0x0a); + const __m256i mask_tab = _mm256_set1_epi8(0x09); + const __m256i mask_carriage = _mm256_set1_epi8(0x0d); + whitespace = in->build_bitmask([&](auto in) { + __m256i space = _mm256_cmpeq_epi8(in, mask_space); + space = _mm256_or_si256(space, _mm256_cmpeq_epi8(in, mask_linefeed)); + space = _mm256_or_si256(space, _mm256_cmpeq_epi8(in, mask_tab)); + space = _mm256_or_si256(space, _mm256_cmpeq_epi8(in, mask_carriage)); + }); + // end of naive approach #else // SIMDJSON_NAIVE_STRUCTURAL - // clang-format off - const __m256i structural_table = - _mm256_setr_epi8(44, 125, 0, 0, 0xc0u, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 123, - 44, 125, 0, 0, 0xc0u, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 123); - const __m256i white_table = _mm256_setr_epi8( - 32, 100, 100, 100, 17, 100, 113, 2, 100, 9, 10, 112, 100, 13, 100, 100, - 32, 100, 100, 100, 17, 100, 113, 2, 100, 9, 10, 112, 100, 13, 100, 100); - // clang-format on - const __m256i struct_offset = _mm256_set1_epi8(0xd4u); - const __m256i struct_mask = _mm256_set1_epi8(32); - __m256i lo_white = _mm256_cmpeq_epi8(in.lo, _mm256_shuffle_epi8(white_table, in.lo)); - __m256i hi_white = _mm256_cmpeq_epi8(in.hi, _mm256_shuffle_epi8(white_table, in.hi)); - uint64_t ws_res_0 = static_cast(_mm256_movemask_epi8(lo_white)); - uint64_t ws_res_1 = _mm256_movemask_epi8(hi_white); - whitespace = (ws_res_0 | (ws_res_1 << 32)); - __m256i lo_struct_r1 = _mm256_add_epi8(struct_offset, in.lo); - __m256i hi_struct_r1 = _mm256_add_epi8(struct_offset, in.hi); - __m256i lo_struct_r2 = _mm256_or_si256(in.lo, struct_mask); - __m256i hi_struct_r2 = _mm256_or_si256(in.hi, struct_mask); - __m256i lo_struct_r3 = _mm256_shuffle_epi8(structural_table, lo_struct_r1); - __m256i hi_struct_r3 = _mm256_shuffle_epi8(structural_table, hi_struct_r1); - __m256i lo_struct = _mm256_cmpeq_epi8(lo_struct_r2, lo_struct_r3); - __m256i hi_struct = _mm256_cmpeq_epi8(hi_struct_r2, hi_struct_r3); + // clang-format off + const __m256i structural_table = + _mm256_setr_epi8(44, 125, 0, 0, 0xc0u, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 123, + 44, 125, 0, 0, 0xc0u, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 123); + const __m256i white_table = _mm256_setr_epi8( + 32, 100, 100, 100, 17, 100, 113, 2, 100, 9, 10, 112, 100, 13, 100, 100, + 32, 100, 100, 100, 17, 100, 113, 2, 100, 9, 10, 112, 100, 13, 100, 100); + // clang-format on + const __m256i struct_offset = _mm256_set1_epi8(0xd4u); + const __m256i struct_mask = _mm256_set1_epi8(32); + + whitespace = in.build_bitmask([&](auto chunk) { + return _mm256_cmpeq_epi8(chunk, _mm256_shuffle_epi8(white_table, chunk)); + }); + structurals = in.build_bitmask([&](auto chunk) { + __m256i struct_r1 = _mm256_add_epi8(struct_offset, chunk); + __m256i struct_r2 = _mm256_or_si256(chunk, struct_mask); + __m256i struct_r3 = _mm256_shuffle_epi8(structural_table, struct_r1); + return _mm256_cmpeq_epi8(struct_r2, struct_r3); + }); - uint64_t structural_res_0 = static_cast(_mm256_movemask_epi8(lo_struct)); - uint64_t structural_res_1 = _mm256_movemask_epi8(hi_struct); - structurals = (structural_res_0 | (structural_res_1 << 32)); #endif // else SIMDJSON_NAIVE_STRUCTURAL } diff --git a/src/westmere/simd_input.h b/src/westmere/simd_input.h index f17ed2bf0..98ca01811 100644 --- a/src/westmere/simd_input.h +++ b/src/westmere/simd_input.h @@ -22,30 +22,27 @@ struct simd_input { this->v3 = _mm_loadu_si128(reinterpret_cast(ptr + 48)); } + template + really_inline uint64_t build_bitmask(F const& chunk_to_mask) { + uint64_t r0 = static_cast(_mm_movemask_epi8(chunk_to_mask(this->v0))); + uint64_t r1 = _mm_movemask_epi8(chunk_to_mask(this->v1)); + uint64_t r2 = _mm_movemask_epi8(chunk_to_mask(this->v2)); + uint64_t r3 = _mm_movemask_epi8(chunk_to_mask(this->v3)); + return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48); + } + really_inline uint64_t eq(uint8_t m) { const __m128i mask = _mm_set1_epi8(m); - __m128i cmp_res_0 = _mm_cmpeq_epi8(this->v0, mask); - uint64_t res_0 = _mm_movemask_epi8(cmp_res_0); - __m128i cmp_res_1 = _mm_cmpeq_epi8(this->v1, mask); - uint64_t res_1 = _mm_movemask_epi8(cmp_res_1); - __m128i cmp_res_2 = _mm_cmpeq_epi8(this->v2, mask); - uint64_t res_2 = _mm_movemask_epi8(cmp_res_2); - __m128i cmp_res_3 = _mm_cmpeq_epi8(this->v3, mask); - uint64_t res_3 = _mm_movemask_epi8(cmp_res_3); - return res_0 | (res_1 << 16) | (res_2 << 32) | (res_3 << 48); + return this->build_bitmask([&](auto chunk) { + return _mm_cmpeq_epi8(chunk, mask); + }); } really_inline uint64_t lteq(uint8_t m) { const __m128i maxval = _mm_set1_epi8(m); - __m128i cmp_res_0 = _mm_cmpeq_epi8(_mm_max_epu8(maxval, this->v0), maxval); - uint64_t res_0 = _mm_movemask_epi8(cmp_res_0); - __m128i cmp_res_1 = _mm_cmpeq_epi8(_mm_max_epu8(maxval, this->v1), maxval); - uint64_t res_1 = _mm_movemask_epi8(cmp_res_1); - __m128i cmp_res_2 = _mm_cmpeq_epi8(_mm_max_epu8(maxval, this->v2), maxval); - uint64_t res_2 = _mm_movemask_epi8(cmp_res_2); - __m128i cmp_res_3 = _mm_cmpeq_epi8(_mm_max_epu8(maxval, this->v3), maxval); - uint64_t res_3 = _mm_movemask_epi8(cmp_res_3); - return res_0 | (res_1 << 16) | (res_2 << 32) | (res_3 << 48); + return this->build_bitmask([&](auto chunk) { + return _mm_cmpeq_epi8(_mm_max_epu8(maxval, chunk), maxval); + }); } }; // struct simd_input diff --git a/src/westmere/stage1_find_marks.h b/src/westmere/stage1_find_marks.h index d2affcda7..10e162a32 100644 --- a/src/westmere/stage1_find_marks.h +++ b/src/westmere/stage1_find_marks.h @@ -28,45 +28,16 @@ static really_inline void find_whitespace_and_structurals(simd_input Date: Fri, 23 Aug 2019 09:56:56 -0700 Subject: [PATCH 2/9] Update amalgamated cpp --- singleheader/amalgamation_demo.cpp | 2 +- singleheader/simdjson.cpp | 262 ++++++++++++----------------- singleheader/simdjson.h | 8 +- 3 files changed, 114 insertions(+), 158 deletions(-) diff --git a/singleheader/amalgamation_demo.cpp b/singleheader/amalgamation_demo.cpp index 9a314d69c..a72ef9152 100644 --- a/singleheader/amalgamation_demo.cpp +++ b/singleheader/amalgamation_demo.cpp @@ -1,4 +1,4 @@ -/* auto-generated on Sun Aug 18 15:06:50 DST 2019. Do not edit! */ +/* auto-generated on Fri Aug 23 09:54:21 DST 2019. Do not edit! */ #include #include "simdjson.h" diff --git a/singleheader/simdjson.cpp b/singleheader/simdjson.cpp index 1af63c1ad..b6281cf64 100644 --- a/singleheader/simdjson.cpp +++ b/singleheader/simdjson.cpp @@ -1,4 +1,4 @@ -/* auto-generated on Sun Aug 18 15:06:50 DST 2019. Do not edit! */ +/* auto-generated on Fri Aug 23 09:54:21 DST 2019. Do not edit! */ #include "simdjson.h" /* used for http://dmalloc.com/ Dmalloc - Debug Malloc Library */ @@ -495,13 +495,13 @@ static const Architecture ARCHITECTURE = Architecture::ARM64; #ifdef IS_X86_64 -TARGET_HASWELL + namespace simdjson::haswell { static const Architecture ARCHITECTURE = Architecture::HASWELL; } // namespace simdjson::haswell -UNTARGET_REGION + #endif // IS_X86_64 @@ -515,13 +515,12 @@ UNTARGET_REGION #ifdef IS_X86_64 -TARGET_WESTMERE namespace simdjson::westmere { static const Architecture ARCHITECTURE = Architecture::WESTMERE; } // namespace simdjson::westmere -UNTARGET_REGION + #endif // IS_X86_64 @@ -620,22 +619,25 @@ struct simd_input { this->hi = _mm256_loadu_si256(reinterpret_cast(ptr + 32)); } + template + really_inline uint64_t build_bitmask(F const& chunk_to_mask) { + uint64_t r0 = static_cast(_mm256_movemask_epi8(chunk_to_mask(this->lo))); + uint64_t r1 = _mm256_movemask_epi8(chunk_to_mask(this->hi)); + return r0 | (r1 << 32); + } + really_inline uint64_t eq(uint8_t m) { const __m256i mask = _mm256_set1_epi8(m); - __m256i cmp_res_0 = _mm256_cmpeq_epi8(this->lo, mask); - uint64_t res_0 = static_cast(_mm256_movemask_epi8(cmp_res_0)); - __m256i cmp_res_1 = _mm256_cmpeq_epi8(this->hi, mask); - uint64_t res_1 = _mm256_movemask_epi8(cmp_res_1); - return res_0 | (res_1 << 32); + return this->build_bitmask([&] (auto chunk) { + return _mm256_cmpeq_epi8(chunk, mask); + }); } really_inline uint64_t lteq(uint8_t m) { const __m256i maxval = _mm256_set1_epi8(m); - __m256i cmp_res_0 = _mm256_cmpeq_epi8(_mm256_max_epu8(maxval, this->lo), maxval); - uint64_t res_0 = static_cast(_mm256_movemask_epi8(cmp_res_0)); - __m256i cmp_res_1 = _mm256_cmpeq_epi8(_mm256_max_epu8(maxval, this->hi), maxval); - uint64_t res_1 = _mm256_movemask_epi8(cmp_res_1); - return res_0 | (res_1 << 32); + return this->build_bitmask([&] (auto chunk) { + return _mm256_cmpeq_epi8(_mm256_max_epu8(maxval, chunk), maxval); + }); } }; // struct simd_input @@ -670,30 +672,27 @@ struct simd_input { this->v3 = _mm_loadu_si128(reinterpret_cast(ptr + 48)); } + template + really_inline uint64_t build_bitmask(F const& chunk_to_mask) { + uint64_t r0 = static_cast(_mm_movemask_epi8(chunk_to_mask(this->v0))); + uint64_t r1 = _mm_movemask_epi8(chunk_to_mask(this->v1)); + uint64_t r2 = _mm_movemask_epi8(chunk_to_mask(this->v2)); + uint64_t r3 = _mm_movemask_epi8(chunk_to_mask(this->v3)); + return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48); + } + really_inline uint64_t eq(uint8_t m) { const __m128i mask = _mm_set1_epi8(m); - __m128i cmp_res_0 = _mm_cmpeq_epi8(this->v0, mask); - uint64_t res_0 = _mm_movemask_epi8(cmp_res_0); - __m128i cmp_res_1 = _mm_cmpeq_epi8(this->v1, mask); - uint64_t res_1 = _mm_movemask_epi8(cmp_res_1); - __m128i cmp_res_2 = _mm_cmpeq_epi8(this->v2, mask); - uint64_t res_2 = _mm_movemask_epi8(cmp_res_2); - __m128i cmp_res_3 = _mm_cmpeq_epi8(this->v3, mask); - uint64_t res_3 = _mm_movemask_epi8(cmp_res_3); - return res_0 | (res_1 << 16) | (res_2 << 32) | (res_3 << 48); + return this->build_bitmask([&](auto chunk) { + return _mm_cmpeq_epi8(chunk, mask); + }); } really_inline uint64_t lteq(uint8_t m) { const __m128i maxval = _mm_set1_epi8(m); - __m128i cmp_res_0 = _mm_cmpeq_epi8(_mm_max_epu8(maxval, this->v0), maxval); - uint64_t res_0 = _mm_movemask_epi8(cmp_res_0); - __m128i cmp_res_1 = _mm_cmpeq_epi8(_mm_max_epu8(maxval, this->v1), maxval); - uint64_t res_1 = _mm_movemask_epi8(cmp_res_1); - __m128i cmp_res_2 = _mm_cmpeq_epi8(_mm_max_epu8(maxval, this->v2), maxval); - uint64_t res_2 = _mm_movemask_epi8(cmp_res_2); - __m128i cmp_res_3 = _mm_cmpeq_epi8(_mm_max_epu8(maxval, this->v3), maxval); - uint64_t res_3 = _mm_movemask_epi8(cmp_res_3); - return res_0 | (res_1 << 16) | (res_2 << 32) | (res_3 << 48); + return this->build_bitmask([&](auto chunk) { + return _mm_cmpeq_epi8(_mm_max_epu8(maxval, chunk), maxval); + }); } }; // struct simd_input @@ -1600,7 +1599,7 @@ static really_inline void flatten_bits(uint32_t *base_ptr, uint32_t &base, uint3 #endif // SIMDJSON_NAIVE_FLATTEN // This file contains the common code every implementation uses in stage1 // It is intended to be included multiple times and compiled multiple times -// We assume the file in which it is include already includes +// We assume the file in which it is included already includes // "simdjson/stage1_find_marks.h" (this simplifies amalgation) // return a bitvector indicating where we have characters that end an odd-length @@ -1612,7 +1611,7 @@ static really_inline void flatten_bits(uint32_t *base_ptr, uint32_t &base, uint3 // indicate whether we end an iteration on an odd-length sequence of // backslashes, which modifies our subsequent search for odd-length // sequences of backslashes in an obvious way. -static really_inline uint64_t find_odd_backslash_sequences( +really_inline uint64_t find_odd_backslash_sequences( simd_input in, uint64_t &prev_iter_ends_odd_backslash) { const uint64_t even_bits = 0x5555555555555555ULL; @@ -1659,7 +1658,7 @@ static really_inline uint64_t find_odd_backslash_sequences( // Note that we don't do any error checking to see if we have backslash // sequences outside quotes; these // backslash sequences (of any length) will be detected elsewhere. -static really_inline uint64_t find_quote_mask_and_bits( +really_inline uint64_t find_quote_mask_and_bits( simd_input in, uint64_t odd_ends, uint64_t &prev_iter_inside_quote, uint64_t "e_bits, uint64_t &error_mask) { @@ -1682,7 +1681,7 @@ static really_inline uint64_t find_quote_mask_and_bits( return quote_mask; } -static really_inline uint64_t finalize_structurals( +really_inline uint64_t finalize_structurals( uint64_t structurals, uint64_t whitespace, uint64_t quote_mask, uint64_t quote_bits, uint64_t &prev_iter_ends_pseudo_pred) { // mask off anything inside quotes @@ -1716,7 +1715,7 @@ static really_inline uint64_t finalize_structurals( } // Find structural bits in a 64-byte chunk. -static really_inline void find_structural_bits_64( +really_inline void find_structural_bits_64( const uint8_t *buf, size_t idx, uint32_t *base_ptr, uint32_t &base, uint64_t &prev_iter_ends_odd_backslash, uint64_t &prev_iter_inside_quote, uint64_t &prev_iter_ends_pseudo_pred, uint64_t &structurals, @@ -1748,7 +1747,7 @@ static really_inline void find_structural_bits_64( quote_bits, prev_iter_ends_pseudo_pred); } -static int find_structural_bits(const uint8_t *buf, size_t len, simdjson::ParsedJson &pj) { +int find_structural_bits(const uint8_t *buf, size_t len, simdjson::ParsedJson &pj) { if (len > pj.byte_capacity) { std::cerr << "Your ParsedJson object only supports documents up to " << pj.byte_capacity << " bytes but you are trying to process " @@ -1877,77 +1876,60 @@ static really_inline void find_whitespace_and_structurals(simd_input(_mm256_movemask_epi8(struct_lo)); - uint64_t structural_res_1 = _mm256_movemask_epi8(struct_hi); - structurals = (structural_res_0 | (structural_res_1 << 32)); - const __m256i mask_space = _mm256_set1_epi8(0x20); - __m256i space_lo = _mm256_cmpeq_epi8(in.lo, mask_space); - __m256i space_hi = _mm256_cmpeq_epi8(in.hi, mask_space); - const __m256i mask_linefeed = _mm256_set1_epi8(0x0a); - space_lo = _mm256_or_si256(space_lo, _mm256_cmpeq_epi8(in.lo, mask_linefeed)); - space_hi = _mm256_or_si256(space_hi, _mm256_cmpeq_epi8(in.hi, mask_linefeed)); - const __m256i mask_tab = _mm256_set1_epi8(0x09); - space_lo = _mm256_or_si256(space_lo, _mm256_cmpeq_epi8(in.lo, mask_tab)); - space_hi = _mm256_or_si256(space_hi, _mm256_cmpeq_epi8(in.hi, mask_tab)); - const __m256i mask_carriage = _mm256_set1_epi8(0x0d); - space_lo = _mm256_or_si256(space_lo, _mm256_cmpeq_epi8(in.lo, mask_carriage)); - space_hi = _mm256_or_si256(space_hi, _mm256_cmpeq_epi8(in.hi, mask_carriage)); + // You should never need this naive approach, but it can be useful + // for research purposes + const __m256i mask_open_brace = _mm256_set1_epi8(0x7b); + const __m256i mask_close_brace = _mm256_set1_epi8(0x7d); + const __m256i mask_open_bracket = _mm256_set1_epi8(0x5b); + const __m256i mask_close_bracket = _mm256_set1_epi8(0x5d); + const __m256i mask_column = _mm256_set1_epi8(0x3a); + const __m256i mask_comma = _mm256_set1_epi8(0x2c); + structurals = in->build_bitmask([&](auto in) { + __m256i structurals = _mm256_cmpeq_epi8(in, mask_open_brace); + structurals = _mm256_or_si256(structurals, _mm256_cmpeq_epi8(in, mask_close_brace)); + structurals = _mm256_or_si256(structurals, _mm256_cmpeq_epi8(in, mask_open_bracket)); + structurals = _mm256_or_si256(structurals, _mm256_cmpeq_epi8(in, mask_close_bracket)); + structurals = _mm256_or_si256(structurals, _mm256_cmpeq_epi8(in, mask_column)); + structurals = _mm256_or_si256(structurals, _mm256_cmpeq_epi8(in, mask_comma)); + return structurals; + }); - uint64_t ws_res_0 = static_cast(_mm256_movemask_epi8(space_lo)); - uint64_t ws_res_1 = _mm256_movemask_epi8(space_hi); - whitespace = (ws_res_0 | (ws_res_1 << 32)); - // end of naive approach + const __m256i mask_space = _mm256_set1_epi8(0x20); + const __m256i mask_linefeed = _mm256_set1_epi8(0x0a); + const __m256i mask_tab = _mm256_set1_epi8(0x09); + const __m256i mask_carriage = _mm256_set1_epi8(0x0d); + whitespace = in->build_bitmask([&](auto in) { + __m256i space = _mm256_cmpeq_epi8(in, mask_space); + space = _mm256_or_si256(space, _mm256_cmpeq_epi8(in, mask_linefeed)); + space = _mm256_or_si256(space, _mm256_cmpeq_epi8(in, mask_tab)); + space = _mm256_or_si256(space, _mm256_cmpeq_epi8(in, mask_carriage)); + }); + // end of naive approach #else // SIMDJSON_NAIVE_STRUCTURAL - // clang-format off - const __m256i structural_table = - _mm256_setr_epi8(44, 125, 0, 0, 0xc0u, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 123, - 44, 125, 0, 0, 0xc0u, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 123); - const __m256i white_table = _mm256_setr_epi8( - 32, 100, 100, 100, 17, 100, 113, 2, 100, 9, 10, 112, 100, 13, 100, 100, - 32, 100, 100, 100, 17, 100, 113, 2, 100, 9, 10, 112, 100, 13, 100, 100); - // clang-format on - const __m256i struct_offset = _mm256_set1_epi8(0xd4u); - const __m256i struct_mask = _mm256_set1_epi8(32); - __m256i lo_white = _mm256_cmpeq_epi8(in.lo, _mm256_shuffle_epi8(white_table, in.lo)); - __m256i hi_white = _mm256_cmpeq_epi8(in.hi, _mm256_shuffle_epi8(white_table, in.hi)); - uint64_t ws_res_0 = static_cast(_mm256_movemask_epi8(lo_white)); - uint64_t ws_res_1 = _mm256_movemask_epi8(hi_white); - whitespace = (ws_res_0 | (ws_res_1 << 32)); - __m256i lo_struct_r1 = _mm256_add_epi8(struct_offset, in.lo); - __m256i hi_struct_r1 = _mm256_add_epi8(struct_offset, in.hi); - __m256i lo_struct_r2 = _mm256_or_si256(in.lo, struct_mask); - __m256i hi_struct_r2 = _mm256_or_si256(in.hi, struct_mask); - __m256i lo_struct_r3 = _mm256_shuffle_epi8(structural_table, lo_struct_r1); - __m256i hi_struct_r3 = _mm256_shuffle_epi8(structural_table, hi_struct_r1); - __m256i lo_struct = _mm256_cmpeq_epi8(lo_struct_r2, lo_struct_r3); - __m256i hi_struct = _mm256_cmpeq_epi8(hi_struct_r2, hi_struct_r3); + // clang-format off + const __m256i structural_table = + _mm256_setr_epi8(44, 125, 0, 0, 0xc0u, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 123, + 44, 125, 0, 0, 0xc0u, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 123); + const __m256i white_table = _mm256_setr_epi8( + 32, 100, 100, 100, 17, 100, 113, 2, 100, 9, 10, 112, 100, 13, 100, 100, + 32, 100, 100, 100, 17, 100, 113, 2, 100, 9, 10, 112, 100, 13, 100, 100); + // clang-format on + const __m256i struct_offset = _mm256_set1_epi8(0xd4u); + const __m256i struct_mask = _mm256_set1_epi8(32); + + whitespace = in.build_bitmask([&](auto chunk) { + return _mm256_cmpeq_epi8(chunk, _mm256_shuffle_epi8(white_table, chunk)); + }); + structurals = in.build_bitmask([&](auto chunk) { + __m256i struct_r1 = _mm256_add_epi8(struct_offset, chunk); + __m256i struct_r2 = _mm256_or_si256(chunk, struct_mask); + __m256i struct_r3 = _mm256_shuffle_epi8(structural_table, struct_r1); + return _mm256_cmpeq_epi8(struct_r2, struct_r3); + }); - uint64_t structural_res_0 = static_cast(_mm256_movemask_epi8(lo_struct)); - uint64_t structural_res_1 = _mm256_movemask_epi8(hi_struct); - structurals = (structural_res_0 | (structural_res_1 << 32)); #endif // else SIMDJSON_NAIVE_STRUCTURAL } @@ -2019,7 +2001,7 @@ static really_inline void flatten_bits(uint32_t *base_ptr, uint32_t &base, uint3 // This file contains the common code every implementation uses in stage1 // It is intended to be included multiple times and compiled multiple times -// We assume the file in which it is include already includes +// We assume the file in which it is included already includes // "simdjson/stage1_find_marks.h" (this simplifies amalgation) // return a bitvector indicating where we have characters that end an odd-length @@ -2031,7 +2013,7 @@ static really_inline void flatten_bits(uint32_t *base_ptr, uint32_t &base, uint3 // indicate whether we end an iteration on an odd-length sequence of // backslashes, which modifies our subsequent search for odd-length // sequences of backslashes in an obvious way. -static really_inline uint64_t find_odd_backslash_sequences( +really_inline uint64_t find_odd_backslash_sequences( simd_input in, uint64_t &prev_iter_ends_odd_backslash) { const uint64_t even_bits = 0x5555555555555555ULL; @@ -2078,7 +2060,7 @@ static really_inline uint64_t find_odd_backslash_sequences( // Note that we don't do any error checking to see if we have backslash // sequences outside quotes; these // backslash sequences (of any length) will be detected elsewhere. -static really_inline uint64_t find_quote_mask_and_bits( +really_inline uint64_t find_quote_mask_and_bits( simd_input in, uint64_t odd_ends, uint64_t &prev_iter_inside_quote, uint64_t "e_bits, uint64_t &error_mask) { @@ -2101,7 +2083,7 @@ static really_inline uint64_t find_quote_mask_and_bits( return quote_mask; } -static really_inline uint64_t finalize_structurals( +really_inline uint64_t finalize_structurals( uint64_t structurals, uint64_t whitespace, uint64_t quote_mask, uint64_t quote_bits, uint64_t &prev_iter_ends_pseudo_pred) { // mask off anything inside quotes @@ -2135,7 +2117,7 @@ static really_inline uint64_t finalize_structurals( } // Find structural bits in a 64-byte chunk. -static really_inline void find_structural_bits_64( +really_inline void find_structural_bits_64( const uint8_t *buf, size_t idx, uint32_t *base_ptr, uint32_t &base, uint64_t &prev_iter_ends_odd_backslash, uint64_t &prev_iter_inside_quote, uint64_t &prev_iter_ends_pseudo_pred, uint64_t &structurals, @@ -2167,7 +2149,7 @@ static really_inline void find_structural_bits_64( quote_bits, prev_iter_ends_pseudo_pred); } -static int find_structural_bits(const uint8_t *buf, size_t len, simdjson::ParsedJson &pj) { +int find_structural_bits(const uint8_t *buf, size_t len, simdjson::ParsedJson &pj) { if (len > pj.byte_capacity) { std::cerr << "Your ParsedJson object only supports documents up to " << pj.byte_capacity << " bytes but you are trying to process " @@ -2302,45 +2284,16 @@ static really_inline void find_whitespace_and_structurals(simd_input in, uint64_t &prev_iter_ends_odd_backslash) { const uint64_t even_bits = 0x5555555555555555ULL; @@ -2493,7 +2446,7 @@ static really_inline uint64_t find_odd_backslash_sequences( // Note that we don't do any error checking to see if we have backslash // sequences outside quotes; these // backslash sequences (of any length) will be detected elsewhere. -static really_inline uint64_t find_quote_mask_and_bits( +really_inline uint64_t find_quote_mask_and_bits( simd_input in, uint64_t odd_ends, uint64_t &prev_iter_inside_quote, uint64_t "e_bits, uint64_t &error_mask) { @@ -2516,7 +2469,7 @@ static really_inline uint64_t find_quote_mask_and_bits( return quote_mask; } -static really_inline uint64_t finalize_structurals( +really_inline uint64_t finalize_structurals( uint64_t structurals, uint64_t whitespace, uint64_t quote_mask, uint64_t quote_bits, uint64_t &prev_iter_ends_pseudo_pred) { // mask off anything inside quotes @@ -2550,7 +2503,7 @@ static really_inline uint64_t finalize_structurals( } // Find structural bits in a 64-byte chunk. -static really_inline void find_structural_bits_64( +really_inline void find_structural_bits_64( const uint8_t *buf, size_t idx, uint32_t *base_ptr, uint32_t &base, uint64_t &prev_iter_ends_odd_backslash, uint64_t &prev_iter_inside_quote, uint64_t &prev_iter_ends_pseudo_pred, uint64_t &structurals, @@ -2582,7 +2535,7 @@ static really_inline void find_structural_bits_64( quote_bits, prev_iter_ends_pseudo_pred); } -static int find_structural_bits(const uint8_t *buf, size_t len, simdjson::ParsedJson &pj) { +int find_structural_bits(const uint8_t *buf, size_t len, simdjson::ParsedJson &pj) { if (len > pj.byte_capacity) { std::cerr << "Your ParsedJson object only supports documents up to " << pj.byte_capacity << " bytes but you are trying to process " @@ -2803,7 +2756,6 @@ struct parse_string_helper { #ifdef IS_ARM64 -#include "amd64/architecture.h" namespace simdjson::arm64 { diff --git a/singleheader/simdjson.h b/singleheader/simdjson.h index 5254af9a4..f57438c19 100644 --- a/singleheader/simdjson.h +++ b/singleheader/simdjson.h @@ -1,4 +1,4 @@ -/* auto-generated on Sun Aug 18 15:06:50 DST 2019. Do not edit! */ +/* auto-generated on Fri Aug 23 09:54:21 DST 2019. Do not edit! */ /* begin file include/simdjson/simdjson_version.h */ // /include/simdjson/simdjson_version.h automatically generated by release.py, // do not change by hand @@ -36438,13 +36438,17 @@ public: // (in case of repeated keys, this only finds the first one). // We seek the key using C's strcmp so if your JSON strings contain // NULL chars, this would trigger a false positive: if you expect that - // to be the case, take extra precautions. + // to be the case, take extra precautions. + // Furthermore, we do the comparison character-by-character + // without taking into account Unicode equivalence. inline bool move_to_key(const char *key); // when at {, go one level deep, looking for a given key // if successful, we are left pointing at the value, // if not, we are still pointing at the object ({) // (in case of repeated keys, this only finds the first one). // The string we search for can contain NULL values. + // Furthermore, we do the comparison character-by-character + // without taking into account Unicode equivalence. inline bool move_to_key(const char *key, uint32_t length); // when at a key location within an object, this moves to the accompanying From da0f1cacea1b0e7d9126661ab8fdd80aa7220c01 Mon Sep 17 00:00:00 2001 From: John Keiser Date: Fri, 23 Aug 2019 10:23:40 -0700 Subject: [PATCH 3/9] Remove static modifiers --- singleheader/amalgamation_demo.cpp | 2 +- singleheader/simdjson.cpp | 24 ++++++++++++------------ singleheader/simdjson.h | 2 +- src/arm64/stage1_find_marks.h | 4 ++-- src/generic/stage1_find_marks_flatten.h | 4 ++-- src/haswell/stage1_find_marks.h | 6 +++--- src/westmere/stage1_find_marks.h | 4 ++-- 7 files changed, 23 insertions(+), 23 deletions(-) diff --git a/singleheader/amalgamation_demo.cpp b/singleheader/amalgamation_demo.cpp index a72ef9152..4b132c707 100644 --- a/singleheader/amalgamation_demo.cpp +++ b/singleheader/amalgamation_demo.cpp @@ -1,4 +1,4 @@ -/* auto-generated on Fri Aug 23 09:54:21 DST 2019. Do not edit! */ +/* auto-generated on Fri Aug 23 10:23:28 DST 2019. Do not edit! */ #include #include "simdjson.h" diff --git a/singleheader/simdjson.cpp b/singleheader/simdjson.cpp index b6281cf64..43099bd8c 100644 --- a/singleheader/simdjson.cpp +++ b/singleheader/simdjson.cpp @@ -1,4 +1,4 @@ -/* auto-generated on Fri Aug 23 09:54:21 DST 2019. Do not edit! */ +/* auto-generated on Fri Aug 23 10:23:28 DST 2019. Do not edit! */ #include "simdjson.h" /* used for http://dmalloc.com/ Dmalloc - Debug Malloc Library */ @@ -1451,7 +1451,7 @@ UNTARGET_REGION // westmere namespace simdjson::arm64 { -static really_inline uint64_t compute_quote_mask(uint64_t quote_bits) { +really_inline uint64_t compute_quote_mask(uint64_t quote_bits) { #ifdef __ARM_FEATURE_CRYPTO // some ARM processors lack this extension return vmull_p64(-1ULL, quote_bits); @@ -1460,7 +1460,7 @@ static really_inline uint64_t compute_quote_mask(uint64_t quote_bits) { #endif } -static really_inline void find_whitespace_and_structurals( +really_inline void find_whitespace_and_structurals( simd_input in, uint64_t &whitespace, uint64_t &structurals) { const uint8x16_t low_nibble_mask = @@ -1518,7 +1518,7 @@ static really_inline void find_whitespace_and_structurals( // This is just a naive implementation. It should be normally // disable, but can be used for research purposes to compare // again our optimized version. -static really_inline void flatten_bits(uint32_t *base_ptr, uint32_t &base, uint32_t idx, uint64_t bits) { +really_inline void flatten_bits(uint32_t *base_ptr, uint32_t &base, uint32_t idx, uint64_t bits) { uint32_t *out_ptr = base_ptr + base; idx -= 64; while (bits != 0) { @@ -1536,7 +1536,7 @@ static really_inline void flatten_bits(uint32_t *base_ptr, uint32_t &base, uint3 // base_ptr[base] incrementing base as we go // will potentially store extra values beyond end of valid bits, so base_ptr // needs to be large enough to handle this -static really_inline void flatten_bits(uint32_t *base_ptr, uint32_t &base, uint32_t idx, uint64_t bits) { +really_inline void flatten_bits(uint32_t *base_ptr, uint32_t &base, uint32_t idx, uint64_t bits) { // In some instances, the next branch is expensive because it is mispredicted. // Unfortunately, in other cases, // it helps tremendously. @@ -1864,7 +1864,7 @@ int find_structural_bits(const uint8_t *buf, size_t len, si TARGET_HASWELL namespace simdjson::haswell { -static really_inline uint64_t compute_quote_mask(uint64_t quote_bits) { +really_inline uint64_t compute_quote_mask(uint64_t quote_bits) { // There should be no such thing with a processing supporting avx2 // but not clmul. uint64_t quote_mask = _mm_cvtsi128_si64(_mm_clmulepi64_si128( @@ -1872,7 +1872,7 @@ static really_inline uint64_t compute_quote_mask(uint64_t quote_bits) { return quote_mask; } -static really_inline void find_whitespace_and_structurals(simd_input in, +really_inline void find_whitespace_and_structurals(simd_input in, uint64_t &whitespace, uint64_t &structurals) { #ifdef SIMDJSON_NAIVE_STRUCTURAL @@ -1938,7 +1938,7 @@ static really_inline void find_whitespace_and_structurals(simd_input in, +really_inline void find_whitespace_and_structurals(simd_input in, uint64_t &whitespace, uint64_t &structurals) { const __m128i structural_table = @@ -2306,7 +2306,7 @@ static really_inline void find_whitespace_and_structurals(simd_input in, uint64_t &whitespace, uint64_t &structurals) { const uint8x16_t low_nibble_mask = diff --git a/src/generic/stage1_find_marks_flatten.h b/src/generic/stage1_find_marks_flatten.h index 56e084134..9583759fd 100644 --- a/src/generic/stage1_find_marks_flatten.h +++ b/src/generic/stage1_find_marks_flatten.h @@ -8,7 +8,7 @@ // This is just a naive implementation. It should be normally // disable, but can be used for research purposes to compare // again our optimized version. -static really_inline void flatten_bits(uint32_t *base_ptr, uint32_t &base, uint32_t idx, uint64_t bits) { +really_inline void flatten_bits(uint32_t *base_ptr, uint32_t &base, uint32_t idx, uint64_t bits) { uint32_t *out_ptr = base_ptr + base; idx -= 64; while (bits != 0) { @@ -26,7 +26,7 @@ static really_inline void flatten_bits(uint32_t *base_ptr, uint32_t &base, uint3 // base_ptr[base] incrementing base as we go // will potentially store extra values beyond end of valid bits, so base_ptr // needs to be large enough to handle this -static really_inline void flatten_bits(uint32_t *base_ptr, uint32_t &base, uint32_t idx, uint64_t bits) { +really_inline void flatten_bits(uint32_t *base_ptr, uint32_t &base, uint32_t idx, uint64_t bits) { // In some instances, the next branch is expensive because it is mispredicted. // Unfortunately, in other cases, // it helps tremendously. diff --git a/src/haswell/stage1_find_marks.h b/src/haswell/stage1_find_marks.h index a2c776f36..de490a67e 100644 --- a/src/haswell/stage1_find_marks.h +++ b/src/haswell/stage1_find_marks.h @@ -13,7 +13,7 @@ TARGET_HASWELL namespace simdjson::haswell { -static really_inline uint64_t compute_quote_mask(uint64_t quote_bits) { +really_inline uint64_t compute_quote_mask(uint64_t quote_bits) { // There should be no such thing with a processing supporting avx2 // but not clmul. uint64_t quote_mask = _mm_cvtsi128_si64(_mm_clmulepi64_si128( @@ -21,7 +21,7 @@ static really_inline uint64_t compute_quote_mask(uint64_t quote_bits) { return quote_mask; } -static really_inline void find_whitespace_and_structurals(simd_input in, +really_inline void find_whitespace_and_structurals(simd_input in, uint64_t &whitespace, uint64_t &structurals) { #ifdef SIMDJSON_NAIVE_STRUCTURAL @@ -87,7 +87,7 @@ static really_inline void find_whitespace_and_structurals(simd_input in, +really_inline void find_whitespace_and_structurals(simd_input in, uint64_t &whitespace, uint64_t &structurals) { const __m128i structural_table = From cf4ae61ac62019c86a29fd7b172ff7501b04cc2a Mon Sep 17 00:00:00 2001 From: John Keiser Date: Fri, 23 Aug 2019 10:36:10 -0700 Subject: [PATCH 4/9] Modify checkperf to print out perfdiff command to make it easier to run it yourself without having to recompile the world --- scripts/checkperf.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/checkperf.sh b/scripts/checkperf.sh index ade4c781e..3817a9903 100644 --- a/scripts/checkperf.sh +++ b/scripts/checkperf.sh @@ -27,4 +27,5 @@ make parse make perfdiff echo "Running perfdiff:" +echo ./perfdiff \"$current/parse -t $perftests\" \"$reference/parse -t $perftests\" ./perfdiff "$current/parse -t $perftests" "$reference/parse -t $perftests" From 441963c84cedbf6bc43d1cf46fd0727a95a78265 Mon Sep 17 00:00:00 2001 From: John Keiser Date: Fri, 23 Aug 2019 10:49:26 -0700 Subject: [PATCH 5/9] Add AMD64 build_bitmask --- singleheader/amalgamation_demo.cpp | 2 +- singleheader/simdjson.cpp | 88 ++++++++++++++---------------- singleheader/simdjson.h | 2 +- src/arm64/simd_input.h | 43 +++++++++++---- src/arm64/stage1_find_marks.h | 50 +++++------------ 5 files changed, 92 insertions(+), 93 deletions(-) diff --git a/singleheader/amalgamation_demo.cpp b/singleheader/amalgamation_demo.cpp index 4b132c707..049131797 100644 --- a/singleheader/amalgamation_demo.cpp +++ b/singleheader/amalgamation_demo.cpp @@ -1,4 +1,4 @@ -/* auto-generated on Fri Aug 23 10:23:28 DST 2019. Do not edit! */ +/* auto-generated on Fri Aug 23 11:02:39 DST 2019. Do not edit! */ #include #include "simdjson.h" diff --git a/singleheader/simdjson.cpp b/singleheader/simdjson.cpp index 43099bd8c..0fb694fd5 100644 --- a/singleheader/simdjson.cpp +++ b/singleheader/simdjson.cpp @@ -1,4 +1,4 @@ -/* auto-generated on Fri Aug 23 10:23:28 DST 2019. Do not edit! */ +/* auto-generated on Fri Aug 23 11:02:39 DST 2019. Do not edit! */ #include "simdjson.h" /* used for http://dmalloc.com/ Dmalloc - Debug Malloc Library */ @@ -574,22 +574,38 @@ struct simd_input { this->i3 = vld1q_u8(ptr + 48); } + template + really_inline uint64_t build_bitmask(F const& chunk_to_mask) { + uint8x16_t r0 = chunk_to_mask(this->i0); + uint8x16_t r1 = chunk_to_mask(this->i1); + uint8x16_t r2 = chunk_to_mask(this->i2); + uint8x16_t r3 = chunk_to_mask(this->i3); + return neon_movemask_bulk(r0, r1, r2, r3); + } + + template + really_inline simd_input map(F const& map_chunk) { + simd_input result = { + map_chunk(this->i0), + map_chunk(this->i1), + map_chunk(this->i2), + map_chunk(this->i3) + }; + return result; + } + really_inline uint64_t eq(uint8_t m) { const uint8x16_t mask = vmovq_n_u8(m); - uint8x16_t cmp_res_0 = vceqq_u8(this->i0, mask); - uint8x16_t cmp_res_1 = vceqq_u8(this->i1, mask); - uint8x16_t cmp_res_2 = vceqq_u8(this->i2, mask); - uint8x16_t cmp_res_3 = vceqq_u8(this->i3, mask); - return neon_movemask_bulk(cmp_res_0, cmp_res_1, cmp_res_2, cmp_res_3); + return this->build_bitmask([&](uint8x16_t chunk) { + return vceqq_u8(chunk, mask); + }); } really_inline uint64_t lteq(uint8_t m) { const uint8x16_t mask = vmovq_n_u8(m); - uint8x16_t cmp_res_0 = vcleq_u8(this->i0, mask); - uint8x16_t cmp_res_1 = vcleq_u8(this->i1, mask); - uint8x16_t cmp_res_2 = vcleq_u8(this->i2, mask); - uint8x16_t cmp_res_3 = vcleq_u8(this->i3, mask); - return neon_movemask_bulk(cmp_res_0, cmp_res_1, cmp_res_2, cmp_res_3); + return this->build_bitmask([&](uint8x16_t chunk) { + return vcleq_u8(chunk, mask); + }); } }; // struct simd_input @@ -1467,45 +1483,25 @@ really_inline void find_whitespace_and_structurals( (uint8x16_t){16, 0, 0, 0, 0, 0, 0, 0, 0, 8, 12, 1, 2, 9, 0, 0}; const uint8x16_t high_nibble_mask = (uint8x16_t){8, 0, 18, 4, 0, 1, 0, 1, 0, 0, 0, 3, 2, 1, 0, 0}; - const uint8x16_t structural_shufti_mask = vmovq_n_u8(0x7); - const uint8x16_t whitespace_shufti_mask = vmovq_n_u8(0x18); const uint8x16_t low_nib_and_mask = vmovq_n_u8(0xf); - uint8x16_t nib_0_lo = vandq_u8(in.i0, low_nib_and_mask); - uint8x16_t nib_0_hi = vshrq_n_u8(in.i0, 4); - uint8x16_t shuf_0_lo = vqtbl1q_u8(low_nibble_mask, nib_0_lo); - uint8x16_t shuf_0_hi = vqtbl1q_u8(high_nibble_mask, nib_0_hi); - uint8x16_t v_0 = vandq_u8(shuf_0_lo, shuf_0_hi); + simd_input v = in.map([&](auto chunk) { + uint8x16_t nib_lo = vandq_u8(chunk, low_nib_and_mask); + uint8x16_t nib_hi = vshrq_n_u8(chunk, 4); + uint8x16_t shuf_lo = vqtbl1q_u8(low_nibble_mask, nib_lo); + uint8x16_t shuf_hi = vqtbl1q_u8(high_nibble_mask, nib_hi); + return vandq_u8(shuf_lo, shuf_hi); + }); - uint8x16_t nib_1_lo = vandq_u8(in.i1, low_nib_and_mask); - uint8x16_t nib_1_hi = vshrq_n_u8(in.i1, 4); - uint8x16_t shuf_1_lo = vqtbl1q_u8(low_nibble_mask, nib_1_lo); - uint8x16_t shuf_1_hi = vqtbl1q_u8(high_nibble_mask, nib_1_hi); - uint8x16_t v_1 = vandq_u8(shuf_1_lo, shuf_1_hi); + const uint8x16_t structural_shufti_mask = vmovq_n_u8(0x7); + structurals = v.build_bitmask([&](auto chunk) { + return vtstq_u8(chunk, structural_shufti_mask); + }); - uint8x16_t nib_2_lo = vandq_u8(in.i2, low_nib_and_mask); - uint8x16_t nib_2_hi = vshrq_n_u8(in.i2, 4); - uint8x16_t shuf_2_lo = vqtbl1q_u8(low_nibble_mask, nib_2_lo); - uint8x16_t shuf_2_hi = vqtbl1q_u8(high_nibble_mask, nib_2_hi); - uint8x16_t v_2 = vandq_u8(shuf_2_lo, shuf_2_hi); - - uint8x16_t nib_3_lo = vandq_u8(in.i3, low_nib_and_mask); - uint8x16_t nib_3_hi = vshrq_n_u8(in.i3, 4); - uint8x16_t shuf_3_lo = vqtbl1q_u8(low_nibble_mask, nib_3_lo); - uint8x16_t shuf_3_hi = vqtbl1q_u8(high_nibble_mask, nib_3_hi); - uint8x16_t v_3 = vandq_u8(shuf_3_lo, shuf_3_hi); - - uint8x16_t tmp_0 = vtstq_u8(v_0, structural_shufti_mask); - uint8x16_t tmp_1 = vtstq_u8(v_1, structural_shufti_mask); - uint8x16_t tmp_2 = vtstq_u8(v_2, structural_shufti_mask); - uint8x16_t tmp_3 = vtstq_u8(v_3, structural_shufti_mask); - structurals = neon_movemask_bulk(tmp_0, tmp_1, tmp_2, tmp_3); - - uint8x16_t tmp_ws_0 = vtstq_u8(v_0, whitespace_shufti_mask); - uint8x16_t tmp_ws_1 = vtstq_u8(v_1, whitespace_shufti_mask); - uint8x16_t tmp_ws_2 = vtstq_u8(v_2, whitespace_shufti_mask); - uint8x16_t tmp_ws_3 = vtstq_u8(v_3, whitespace_shufti_mask); - whitespace = neon_movemask_bulk(tmp_ws_0, tmp_ws_1, tmp_ws_2, tmp_ws_3); + const uint8x16_t whitespace_shufti_mask = vmovq_n_u8(0x18); + whitespace = v.build_bitmask([&](auto chunk) { + return vtstq_u8(chunk, whitespace_shufti_mask); + }); } // This file contains a non-architecture-specific version of "flatten" used in stage1. diff --git a/singleheader/simdjson.h b/singleheader/simdjson.h index 95ee157d7..2340da6c6 100644 --- a/singleheader/simdjson.h +++ b/singleheader/simdjson.h @@ -1,4 +1,4 @@ -/* auto-generated on Fri Aug 23 10:23:28 DST 2019. Do not edit! */ +/* auto-generated on Fri Aug 23 11:02:39 DST 2019. Do not edit! */ /* begin file include/simdjson/simdjson_version.h */ // /include/simdjson/simdjson_version.h automatically generated by release.py, // do not change by hand diff --git a/src/arm64/simd_input.h b/src/arm64/simd_input.h index de973ad75..59e6d5937 100644 --- a/src/arm64/simd_input.h +++ b/src/arm64/simd_input.h @@ -46,22 +46,45 @@ struct simd_input { this->i3 = vld1q_u8(ptr + 48); } + really_inline simd_input(uint8x16_t i0, uint8x16_t i1, uint8x16_t i2, uint8x16_t i3) { + this->i0 = i0; + this->i1 = i1; + this->i2 = i2; + this->i3 = i3; + } + + template + really_inline uint64_t build_bitmask(F const& chunk_to_mask) { + uint8x16_t r0 = chunk_to_mask(this->i0); + uint8x16_t r1 = chunk_to_mask(this->i1); + uint8x16_t r2 = chunk_to_mask(this->i2); + uint8x16_t r3 = chunk_to_mask(this->i3); + return neon_movemask_bulk(r0, r1, r2, r3); + } + + template + really_inline simd_input map(F const& map_chunk) { + simd_input result = { + map_chunk(this->i0), + map_chunk(this->i1), + map_chunk(this->i2), + map_chunk(this->i3) + }; + return result; + } + really_inline uint64_t eq(uint8_t m) { const uint8x16_t mask = vmovq_n_u8(m); - uint8x16_t cmp_res_0 = vceqq_u8(this->i0, mask); - uint8x16_t cmp_res_1 = vceqq_u8(this->i1, mask); - uint8x16_t cmp_res_2 = vceqq_u8(this->i2, mask); - uint8x16_t cmp_res_3 = vceqq_u8(this->i3, mask); - return neon_movemask_bulk(cmp_res_0, cmp_res_1, cmp_res_2, cmp_res_3); + return this->build_bitmask([&](uint8x16_t chunk) { + return vceqq_u8(chunk, mask); + }); } really_inline uint64_t lteq(uint8_t m) { const uint8x16_t mask = vmovq_n_u8(m); - uint8x16_t cmp_res_0 = vcleq_u8(this->i0, mask); - uint8x16_t cmp_res_1 = vcleq_u8(this->i1, mask); - uint8x16_t cmp_res_2 = vcleq_u8(this->i2, mask); - uint8x16_t cmp_res_3 = vcleq_u8(this->i3, mask); - return neon_movemask_bulk(cmp_res_0, cmp_res_1, cmp_res_2, cmp_res_3); + return this->build_bitmask([&](uint8x16_t chunk) { + return vcleq_u8(chunk, mask); + }); } }; // struct simd_input diff --git a/src/arm64/stage1_find_marks.h b/src/arm64/stage1_find_marks.h index 3fd0d4291..4f93e2d8f 100644 --- a/src/arm64/stage1_find_marks.h +++ b/src/arm64/stage1_find_marks.h @@ -28,45 +28,25 @@ really_inline void find_whitespace_and_structurals( (uint8x16_t){16, 0, 0, 0, 0, 0, 0, 0, 0, 8, 12, 1, 2, 9, 0, 0}; const uint8x16_t high_nibble_mask = (uint8x16_t){8, 0, 18, 4, 0, 1, 0, 1, 0, 0, 0, 3, 2, 1, 0, 0}; - const uint8x16_t structural_shufti_mask = vmovq_n_u8(0x7); - const uint8x16_t whitespace_shufti_mask = vmovq_n_u8(0x18); const uint8x16_t low_nib_and_mask = vmovq_n_u8(0xf); - uint8x16_t nib_0_lo = vandq_u8(in.i0, low_nib_and_mask); - uint8x16_t nib_0_hi = vshrq_n_u8(in.i0, 4); - uint8x16_t shuf_0_lo = vqtbl1q_u8(low_nibble_mask, nib_0_lo); - uint8x16_t shuf_0_hi = vqtbl1q_u8(high_nibble_mask, nib_0_hi); - uint8x16_t v_0 = vandq_u8(shuf_0_lo, shuf_0_hi); + simd_input v = in.map([&](auto chunk) { + uint8x16_t nib_lo = vandq_u8(chunk, low_nib_and_mask); + uint8x16_t nib_hi = vshrq_n_u8(chunk, 4); + uint8x16_t shuf_lo = vqtbl1q_u8(low_nibble_mask, nib_lo); + uint8x16_t shuf_hi = vqtbl1q_u8(high_nibble_mask, nib_hi); + return vandq_u8(shuf_lo, shuf_hi); + }); - uint8x16_t nib_1_lo = vandq_u8(in.i1, low_nib_and_mask); - uint8x16_t nib_1_hi = vshrq_n_u8(in.i1, 4); - uint8x16_t shuf_1_lo = vqtbl1q_u8(low_nibble_mask, nib_1_lo); - uint8x16_t shuf_1_hi = vqtbl1q_u8(high_nibble_mask, nib_1_hi); - uint8x16_t v_1 = vandq_u8(shuf_1_lo, shuf_1_hi); + const uint8x16_t structural_shufti_mask = vmovq_n_u8(0x7); + structurals = v.build_bitmask([&](auto chunk) { + return vtstq_u8(chunk, structural_shufti_mask); + }); - uint8x16_t nib_2_lo = vandq_u8(in.i2, low_nib_and_mask); - uint8x16_t nib_2_hi = vshrq_n_u8(in.i2, 4); - uint8x16_t shuf_2_lo = vqtbl1q_u8(low_nibble_mask, nib_2_lo); - uint8x16_t shuf_2_hi = vqtbl1q_u8(high_nibble_mask, nib_2_hi); - uint8x16_t v_2 = vandq_u8(shuf_2_lo, shuf_2_hi); - - uint8x16_t nib_3_lo = vandq_u8(in.i3, low_nib_and_mask); - uint8x16_t nib_3_hi = vshrq_n_u8(in.i3, 4); - uint8x16_t shuf_3_lo = vqtbl1q_u8(low_nibble_mask, nib_3_lo); - uint8x16_t shuf_3_hi = vqtbl1q_u8(high_nibble_mask, nib_3_hi); - uint8x16_t v_3 = vandq_u8(shuf_3_lo, shuf_3_hi); - - uint8x16_t tmp_0 = vtstq_u8(v_0, structural_shufti_mask); - uint8x16_t tmp_1 = vtstq_u8(v_1, structural_shufti_mask); - uint8x16_t tmp_2 = vtstq_u8(v_2, structural_shufti_mask); - uint8x16_t tmp_3 = vtstq_u8(v_3, structural_shufti_mask); - structurals = neon_movemask_bulk(tmp_0, tmp_1, tmp_2, tmp_3); - - uint8x16_t tmp_ws_0 = vtstq_u8(v_0, whitespace_shufti_mask); - uint8x16_t tmp_ws_1 = vtstq_u8(v_1, whitespace_shufti_mask); - uint8x16_t tmp_ws_2 = vtstq_u8(v_2, whitespace_shufti_mask); - uint8x16_t tmp_ws_3 = vtstq_u8(v_3, whitespace_shufti_mask); - whitespace = neon_movemask_bulk(tmp_ws_0, tmp_ws_1, tmp_ws_2, tmp_ws_3); + const uint8x16_t whitespace_shufti_mask = vmovq_n_u8(0x18); + whitespace = v.build_bitmask([&](auto chunk) { + return vtstq_u8(chunk, whitespace_shufti_mask); + }); } #include "generic/stage1_find_marks_flatten.h" From 9cc4ddfc8844f14ab6e54d41a2b388746f9845d4 Mon Sep 17 00:00:00 2001 From: John Keiser Date: Fri, 23 Aug 2019 11:48:36 -0700 Subject: [PATCH 6/9] Use map().to_bitmask() instead of build_bitmask() --- src/arm64/simd_input.h | 36 +++++++++++++------------------- src/arm64/stage1_find_marks.h | 8 +++---- src/haswell/simd_input.h | 26 ++++++++++++++++------- src/haswell/stage1_find_marks.h | 16 +++++++------- src/westmere/simd_input.h | 35 +++++++++++++++++++++++-------- src/westmere/stage1_find_marks.h | 8 +++---- 6 files changed, 76 insertions(+), 53 deletions(-) diff --git a/src/arm64/simd_input.h b/src/arm64/simd_input.h index 59e6d5937..fa1c8c146 100644 --- a/src/arm64/simd_input.h +++ b/src/arm64/simd_input.h @@ -46,45 +46,39 @@ struct simd_input { this->i3 = vld1q_u8(ptr + 48); } - really_inline simd_input(uint8x16_t i0, uint8x16_t i1, uint8x16_t i2, uint8x16_t i3) { - this->i0 = i0; - this->i1 = i1; - this->i2 = i2; - this->i3 = i3; - } - - template - really_inline uint64_t build_bitmask(F const& chunk_to_mask) { - uint8x16_t r0 = chunk_to_mask(this->i0); - uint8x16_t r1 = chunk_to_mask(this->i1); - uint8x16_t r2 = chunk_to_mask(this->i2); - uint8x16_t r3 = chunk_to_mask(this->i3); - return neon_movemask_bulk(r0, r1, r2, r3); + really_inline simd_input(uint8x16_t a0, uint8x16_t a1, uint8x16_t a2, uint8x16_t a3) { + this->i0 = a0; + this->i1 = a1; + this->i2 = a2; + this->i3 = a3; } template really_inline simd_input map(F const& map_chunk) { - simd_input result = { + return simd_input( map_chunk(this->i0), map_chunk(this->i1), map_chunk(this->i2), map_chunk(this->i3) - }; - return result; + ); + } + + really_inline uint64_t to_bitmask() { + return neon_movemask_bulk(this->i0, this->i1, this->i2, this->i3); } really_inline uint64_t eq(uint8_t m) { const uint8x16_t mask = vmovq_n_u8(m); - return this->build_bitmask([&](uint8x16_t chunk) { + return this->map([&](uint8x16_t chunk) { return vceqq_u8(chunk, mask); - }); + }).to_bitmask(); } really_inline uint64_t lteq(uint8_t m) { const uint8x16_t mask = vmovq_n_u8(m); - return this->build_bitmask([&](uint8x16_t chunk) { + return this->map([&](uint8x16_t chunk) { return vcleq_u8(chunk, mask); - }); + }).to_bitmask(); } }; // struct simd_input diff --git a/src/arm64/stage1_find_marks.h b/src/arm64/stage1_find_marks.h index 4f93e2d8f..c0a17ea98 100644 --- a/src/arm64/stage1_find_marks.h +++ b/src/arm64/stage1_find_marks.h @@ -39,14 +39,14 @@ really_inline void find_whitespace_and_structurals( }); const uint8x16_t structural_shufti_mask = vmovq_n_u8(0x7); - structurals = v.build_bitmask([&](auto chunk) { + structurals = v.map([&](auto chunk) { return vtstq_u8(chunk, structural_shufti_mask); - }); + }).to_bitmask(); const uint8x16_t whitespace_shufti_mask = vmovq_n_u8(0x18); - whitespace = v.build_bitmask([&](auto chunk) { + whitespace = v.map([&](auto chunk) { return vtstq_u8(chunk, whitespace_shufti_mask); - }); + }).to_bitmask(); } #include "generic/stage1_find_marks_flatten.h" diff --git a/src/haswell/simd_input.h b/src/haswell/simd_input.h index 8175af286..313d380ad 100644 --- a/src/haswell/simd_input.h +++ b/src/haswell/simd_input.h @@ -18,25 +18,37 @@ struct simd_input { this->hi = _mm256_loadu_si256(reinterpret_cast(ptr + 32)); } + really_inline simd_input(__m256i i0, __m256i i1) { + this->lo = i0; + this->hi = i1; + } + template - really_inline uint64_t build_bitmask(F const& chunk_to_mask) { - uint64_t r0 = static_cast(_mm256_movemask_epi8(chunk_to_mask(this->lo))); - uint64_t r1 = _mm256_movemask_epi8(chunk_to_mask(this->hi)); + really_inline simd_input map(F const& map_chunk) { + return simd_input( + map_chunk(this->lo), + map_chunk(this->hi) + ); + } + + really_inline uint64_t to_bitmask() { + uint64_t r0 = static_cast(_mm256_movemask_epi8(this->lo)); + uint64_t r1 = _mm256_movemask_epi8(this->hi); return r0 | (r1 << 32); } really_inline uint64_t eq(uint8_t m) { const __m256i mask = _mm256_set1_epi8(m); - return this->build_bitmask([&] (auto chunk) { + return this->map([&] (auto chunk) { return _mm256_cmpeq_epi8(chunk, mask); - }); + }).to_bitmask(); } really_inline uint64_t lteq(uint8_t m) { const __m256i maxval = _mm256_set1_epi8(m); - return this->build_bitmask([&] (auto chunk) { + return this->map([&] (auto chunk) { return _mm256_cmpeq_epi8(_mm256_max_epu8(maxval, chunk), maxval); - }); + }).to_bitmask(); } }; // struct simd_input diff --git a/src/haswell/stage1_find_marks.h b/src/haswell/stage1_find_marks.h index de490a67e..5dffa8ea0 100644 --- a/src/haswell/stage1_find_marks.h +++ b/src/haswell/stage1_find_marks.h @@ -34,7 +34,7 @@ really_inline void find_whitespace_and_structurals(simd_input in, const __m256i mask_close_bracket = _mm256_set1_epi8(0x5d); const __m256i mask_column = _mm256_set1_epi8(0x3a); const __m256i mask_comma = _mm256_set1_epi8(0x2c); - structurals = in->build_bitmask([&](auto in) { + structurals = in.map([&](auto in) { __m256i structurals = _mm256_cmpeq_epi8(in, mask_open_brace); structurals = _mm256_or_si256(structurals, _mm256_cmpeq_epi8(in, mask_close_brace)); structurals = _mm256_or_si256(structurals, _mm256_cmpeq_epi8(in, mask_open_bracket)); @@ -42,18 +42,18 @@ really_inline void find_whitespace_and_structurals(simd_input in, structurals = _mm256_or_si256(structurals, _mm256_cmpeq_epi8(in, mask_column)); structurals = _mm256_or_si256(structurals, _mm256_cmpeq_epi8(in, mask_comma)); return structurals; - }); + }).to_bitmask(); const __m256i mask_space = _mm256_set1_epi8(0x20); const __m256i mask_linefeed = _mm256_set1_epi8(0x0a); const __m256i mask_tab = _mm256_set1_epi8(0x09); const __m256i mask_carriage = _mm256_set1_epi8(0x0d); - whitespace = in->build_bitmask([&](auto in) { + whitespace = in.map([&](auto in) { __m256i space = _mm256_cmpeq_epi8(in, mask_space); space = _mm256_or_si256(space, _mm256_cmpeq_epi8(in, mask_linefeed)); space = _mm256_or_si256(space, _mm256_cmpeq_epi8(in, mask_tab)); space = _mm256_or_si256(space, _mm256_cmpeq_epi8(in, mask_carriage)); - }); + }).to_bitmask(); // end of naive approach #else // SIMDJSON_NAIVE_STRUCTURAL @@ -69,15 +69,15 @@ really_inline void find_whitespace_and_structurals(simd_input in, const __m256i struct_offset = _mm256_set1_epi8(0xd4u); const __m256i struct_mask = _mm256_set1_epi8(32); - whitespace = in.build_bitmask([&](auto chunk) { + whitespace = in.map([&](auto chunk) { return _mm256_cmpeq_epi8(chunk, _mm256_shuffle_epi8(white_table, chunk)); - }); - structurals = in.build_bitmask([&](auto chunk) { + }).to_bitmask(); + structurals = in.map([&](auto chunk) { __m256i struct_r1 = _mm256_add_epi8(struct_offset, chunk); __m256i struct_r2 = _mm256_or_si256(chunk, struct_mask); __m256i struct_r3 = _mm256_shuffle_epi8(structural_table, struct_r1); return _mm256_cmpeq_epi8(struct_r2, struct_r3); - }); + }).to_bitmask(); #endif // else SIMDJSON_NAIVE_STRUCTURAL } diff --git a/src/westmere/simd_input.h b/src/westmere/simd_input.h index 98ca01811..c0d8ab41d 100644 --- a/src/westmere/simd_input.h +++ b/src/westmere/simd_input.h @@ -22,27 +22,44 @@ struct simd_input { this->v3 = _mm_loadu_si128(reinterpret_cast(ptr + 48)); } + really_inline simd_input(__m128i i0, __m128i i1, __m128i i2, __m128i i3) + { + this->v0 = i0; + this->v1 = i1; + this->v2 = i2; + this->v3 = i3; + } + template - really_inline uint64_t build_bitmask(F const& chunk_to_mask) { - uint64_t r0 = static_cast(_mm_movemask_epi8(chunk_to_mask(this->v0))); - uint64_t r1 = _mm_movemask_epi8(chunk_to_mask(this->v1)); - uint64_t r2 = _mm_movemask_epi8(chunk_to_mask(this->v2)); - uint64_t r3 = _mm_movemask_epi8(chunk_to_mask(this->v3)); + really_inline simd_input map(F const& map_chunk) { + return simd_input( + map_chunk(this->v0), + map_chunk(this->v1), + map_chunk(this->v2), + map_chunk(this->v3) + ); + } + + really_inline uint64_t to_bitmask() { + uint64_t r0 = static_cast(_mm_movemask_epi8(this->v0)); + uint64_t r1 = _mm_movemask_epi8(this->v0); + uint64_t r2 = _mm_movemask_epi8(this->v2); + uint64_t r3 = _mm_movemask_epi8(this->v3); return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48); } really_inline uint64_t eq(uint8_t m) { const __m128i mask = _mm_set1_epi8(m); - return this->build_bitmask([&](auto chunk) { + return this->map([&](auto chunk) { return _mm_cmpeq_epi8(chunk, mask); - }); + }).to_bitmask(); } really_inline uint64_t lteq(uint8_t m) { const __m128i maxval = _mm_set1_epi8(m); - return this->build_bitmask([&](auto chunk) { + return this->map([&](auto chunk) { return _mm_cmpeq_epi8(_mm_max_epu8(maxval, chunk), maxval); - }); + }).to_bitmask(); } }; // struct simd_input diff --git a/src/westmere/stage1_find_marks.h b/src/westmere/stage1_find_marks.h index 2023fb8b6..7395be66c 100644 --- a/src/westmere/stage1_find_marks.h +++ b/src/westmere/stage1_find_marks.h @@ -28,16 +28,16 @@ really_inline void find_whitespace_and_structurals(simd_input in, const __m128i struct_offset = _mm_set1_epi8(0xd4u); const __m128i struct_mask = _mm_set1_epi8(32); - whitespace = in.build_bitmask([&](auto chunk) { + whitespace = in.map([&](auto chunk) { return _mm_cmpeq_epi8(chunk, _mm_shuffle_epi8(white_table, chunk)); - }); + }).to_bitmask(); - structurals = in.build_bitmask([&](auto chunk) { + structurals = in.map([&](auto chunk) { __m128i struct_r1 = _mm_add_epi8(struct_offset, chunk); __m128i struct_r2 = _mm_or_si128(chunk, struct_mask); __m128i struct_r3 = _mm_shuffle_epi8(structural_table, struct_r1); return _mm_cmpeq_epi8(struct_r2, struct_r3); - }); + }).to_bitmask(); } #include "generic/stage1_find_marks_flatten.h" From 169568ca47ca530cd6ca62cd2a6721f92b65bc54 Mon Sep 17 00:00:00 2001 From: John Keiser Date: Fri, 23 Aug 2019 12:31:50 -0700 Subject: [PATCH 7/9] Use map() to interleave instructions for parallelism --- src/arm64/simd_input.h | 8 ++------ src/arm64/stage1_find_marks.h | 8 ++------ src/haswell/simd_input.h | 20 ++++++++------------ src/haswell/stage1_find_marks.h | 18 +++++++++--------- src/simd_input.h | 10 +++++++++- src/westmere/simd_input.h | 10 +++------- src/westmere/stage1_find_marks.h | 19 ++++++++++--------- 7 files changed, 43 insertions(+), 50 deletions(-) diff --git a/src/arm64/simd_input.h b/src/arm64/simd_input.h index fa1c8c146..415f2badd 100644 --- a/src/arm64/simd_input.h +++ b/src/arm64/simd_input.h @@ -69,16 +69,12 @@ struct simd_input { really_inline uint64_t eq(uint8_t m) { const uint8x16_t mask = vmovq_n_u8(m); - return this->map([&](uint8x16_t chunk) { - return vceqq_u8(chunk, mask); - }).to_bitmask(); + return this->MAP_BITMASK( vceqq_u8(chunk, mask) ); } really_inline uint64_t lteq(uint8_t m) { const uint8x16_t mask = vmovq_n_u8(m); - return this->map([&](uint8x16_t chunk) { - return vcleq_u8(chunk, mask); - }).to_bitmask(); + return this->MAP_BITMASK( vcleq_u8(chunk, mask) ); } }; // struct simd_input diff --git a/src/arm64/stage1_find_marks.h b/src/arm64/stage1_find_marks.h index c0a17ea98..30fb6d0bb 100644 --- a/src/arm64/stage1_find_marks.h +++ b/src/arm64/stage1_find_marks.h @@ -39,14 +39,10 @@ really_inline void find_whitespace_and_structurals( }); const uint8x16_t structural_shufti_mask = vmovq_n_u8(0x7); - structurals = v.map([&](auto chunk) { - return vtstq_u8(chunk, structural_shufti_mask); - }).to_bitmask(); + structurals = v.MAP_BITMASK( vtstq_u8(chunk, structural_shufti_mask) ); const uint8x16_t whitespace_shufti_mask = vmovq_n_u8(0x18); - whitespace = v.map([&](auto chunk) { - return vtstq_u8(chunk, whitespace_shufti_mask); - }).to_bitmask(); + whitespace = v.MAP_BITMASK( vtstq_u8(chunk, whitespace_shufti_mask) ); } #include "generic/stage1_find_marks_flatten.h" diff --git a/src/haswell/simd_input.h b/src/haswell/simd_input.h index 313d380ad..0a2a511b7 100644 --- a/src/haswell/simd_input.h +++ b/src/haswell/simd_input.h @@ -18,9 +18,9 @@ struct simd_input { this->hi = _mm256_loadu_si256(reinterpret_cast(ptr + 32)); } - really_inline simd_input(__m256i i0, __m256i i1) { - this->lo = i0; - this->hi = i1; + really_inline simd_input(__m256i a_lo, __m256i a_hi) { + this->lo = a_lo; + this->hi = a_hi; } template @@ -32,23 +32,19 @@ struct simd_input { } really_inline uint64_t to_bitmask() { - uint64_t r0 = static_cast(_mm256_movemask_epi8(this->lo)); - uint64_t r1 = _mm256_movemask_epi8(this->hi); - return r0 | (r1 << 32); + uint64_t r_lo = static_cast(_mm256_movemask_epi8(this->lo)); + uint64_t r_hi = _mm256_movemask_epi8(this->hi); + return r_lo | (r_hi << 32); } really_inline uint64_t eq(uint8_t m) { const __m256i mask = _mm256_set1_epi8(m); - return this->map([&] (auto chunk) { - return _mm256_cmpeq_epi8(chunk, mask); - }).to_bitmask(); + return this->MAP_BITMASK( _mm256_cmpeq_epi8(chunk, mask) ); } really_inline uint64_t lteq(uint8_t m) { const __m256i maxval = _mm256_set1_epi8(m); - return this->map([&] (auto chunk) { - return _mm256_cmpeq_epi8(_mm256_max_epu8(maxval, chunk), maxval); - }).to_bitmask(); + return this->MAP_BITMASK( _mm256_cmpeq_epi8(_mm256_max_epu8(maxval, chunk), maxval) ); } }; // struct simd_input diff --git a/src/haswell/stage1_find_marks.h b/src/haswell/stage1_find_marks.h index 5dffa8ea0..3c0bce143 100644 --- a/src/haswell/stage1_find_marks.h +++ b/src/haswell/stage1_find_marks.h @@ -53,6 +53,7 @@ really_inline void find_whitespace_and_structurals(simd_input in, space = _mm256_or_si256(space, _mm256_cmpeq_epi8(in, mask_linefeed)); space = _mm256_or_si256(space, _mm256_cmpeq_epi8(in, mask_tab)); space = _mm256_or_si256(space, _mm256_cmpeq_epi8(in, mask_carriage)); + return space; }).to_bitmask(); // end of naive approach @@ -69,15 +70,14 @@ really_inline void find_whitespace_and_structurals(simd_input in, const __m256i struct_offset = _mm256_set1_epi8(0xd4u); const __m256i struct_mask = _mm256_set1_epi8(32); - whitespace = in.map([&](auto chunk) { - return _mm256_cmpeq_epi8(chunk, _mm256_shuffle_epi8(white_table, chunk)); - }).to_bitmask(); - structurals = in.map([&](auto chunk) { - __m256i struct_r1 = _mm256_add_epi8(struct_offset, chunk); - __m256i struct_r2 = _mm256_or_si256(chunk, struct_mask); - __m256i struct_r3 = _mm256_shuffle_epi8(structural_table, struct_r1); - return _mm256_cmpeq_epi8(struct_r2, struct_r3); - }).to_bitmask(); + whitespace = in.MAP_BITMASK( _mm256_cmpeq_epi8(chunk, _mm256_shuffle_epi8(white_table, chunk)) ); + auto struct_r1 = in.MAP_CHUNKS( _mm256_add_epi8(struct_offset, chunk) ); + auto struct_r2 = in.MAP_CHUNKS( _mm256_or_si256(chunk, struct_mask) ); + auto struct_r3 = struct_r1.MAP_CHUNKS( _mm256_shuffle_epi8(structural_table, chunk) ); + structurals = simd_input( + _mm256_cmpeq_epi8(struct_r2.lo, struct_r3.lo), + _mm256_cmpeq_epi8(struct_r2.hi, struct_r3.hi) + ).to_bitmask(); #endif // else SIMDJSON_NAIVE_STRUCTURAL } diff --git a/src/simd_input.h b/src/simd_input.h index 62948df43..624e5f78a 100644 --- a/src/simd_input.h +++ b/src/simd_input.h @@ -8,15 +8,23 @@ namespace simdjson { -template +template struct simd_input { simd_input(const uint8_t *ptr); + // Map through each simd register in this input, producing another simd_input. + template + really_inline simd_input map(F const& map_chunk); + // turn this bytemask (usually the result of a simd comparison operation) into a bitmask. + uint64_t to_bitmask(); // a straightforward comparison of a mask against input. uint64_t eq(uint8_t m); // find all values less than or equal than the content of maxval (using unsigned arithmetic) uint64_t lteq(uint8_t m); }; // struct simd_input +#define MAP_CHUNKS(EXPR) map([&](auto chunk) { return EXPR; }) +#define MAP_BITMASK(EXPR) map([&](auto chunk) { return EXPR; }).to_bitmask() + } // namespace simdjson #endif diff --git a/src/westmere/simd_input.h b/src/westmere/simd_input.h index c0d8ab41d..0e8211232 100644 --- a/src/westmere/simd_input.h +++ b/src/westmere/simd_input.h @@ -42,7 +42,7 @@ struct simd_input { really_inline uint64_t to_bitmask() { uint64_t r0 = static_cast(_mm_movemask_epi8(this->v0)); - uint64_t r1 = _mm_movemask_epi8(this->v0); + uint64_t r1 = _mm_movemask_epi8(this->v1); uint64_t r2 = _mm_movemask_epi8(this->v2); uint64_t r3 = _mm_movemask_epi8(this->v3); return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48); @@ -50,16 +50,12 @@ struct simd_input { really_inline uint64_t eq(uint8_t m) { const __m128i mask = _mm_set1_epi8(m); - return this->map([&](auto chunk) { - return _mm_cmpeq_epi8(chunk, mask); - }).to_bitmask(); + return this->MAP_BITMASK( _mm_cmpeq_epi8(chunk, mask) ); } really_inline uint64_t lteq(uint8_t m) { const __m128i maxval = _mm_set1_epi8(m); - return this->map([&](auto chunk) { - return _mm_cmpeq_epi8(_mm_max_epu8(maxval, chunk), maxval); - }).to_bitmask(); + return this->MAP_BITMASK( _mm_cmpeq_epi8(_mm_max_epu8(maxval, chunk), maxval) ); } }; // struct simd_input diff --git a/src/westmere/stage1_find_marks.h b/src/westmere/stage1_find_marks.h index 7395be66c..acaae0d77 100644 --- a/src/westmere/stage1_find_marks.h +++ b/src/westmere/stage1_find_marks.h @@ -28,16 +28,17 @@ really_inline void find_whitespace_and_structurals(simd_input in, const __m128i struct_offset = _mm_set1_epi8(0xd4u); const __m128i struct_mask = _mm_set1_epi8(32); - whitespace = in.map([&](auto chunk) { - return _mm_cmpeq_epi8(chunk, _mm_shuffle_epi8(white_table, chunk)); - }).to_bitmask(); + whitespace = in.MAP_BITMASK( _mm_cmpeq_epi8(chunk, _mm_shuffle_epi8(white_table, chunk)) ); - structurals = in.map([&](auto chunk) { - __m128i struct_r1 = _mm_add_epi8(struct_offset, chunk); - __m128i struct_r2 = _mm_or_si128(chunk, struct_mask); - __m128i struct_r3 = _mm_shuffle_epi8(structural_table, struct_r1); - return _mm_cmpeq_epi8(struct_r2, struct_r3); - }).to_bitmask(); + auto r1 = in.MAP_CHUNKS( _mm_add_epi8(struct_offset, chunk) ); + auto r2 = in.MAP_CHUNKS( _mm_or_si128(chunk, struct_mask) ); + auto r3 = r1.MAP_CHUNKS( _mm_shuffle_epi8(structural_table, chunk) ); + structurals = simd_input( + _mm_cmpeq_epi8(r2.v0, r3.v0), + _mm_cmpeq_epi8(r2.v1, r3.v1), + _mm_cmpeq_epi8(r2.v2, r3.v2), + _mm_cmpeq_epi8(r2.v3, r3.v3) + ).to_bitmask(); } #include "generic/stage1_find_marks_flatten.h" From f4fa5b73400251274034c7b3c589bcbf9220281c Mon Sep 17 00:00:00 2001 From: John Keiser Date: Fri, 23 Aug 2019 15:55:54 -0700 Subject: [PATCH 8/9] Add MAP_CHUNKS2, make parameter name related to input --- src/arm64/simd_input.h | 18 ++++++++++++++++-- src/arm64/stage1_find_marks.h | 6 +++--- src/haswell/simd_input.h | 16 ++++++++++++++-- src/haswell/stage1_find_marks.h | 16 +++++++--------- src/simd_input.h | 9 +++++++-- src/westmere/simd_input.h | 18 ++++++++++++++++-- src/westmere/stage1_find_marks.h | 19 +++++++------------ 7 files changed, 70 insertions(+), 32 deletions(-) diff --git a/src/arm64/simd_input.h b/src/arm64/simd_input.h index 415f2badd..86b5793a8 100644 --- a/src/arm64/simd_input.h +++ b/src/arm64/simd_input.h @@ -63,18 +63,32 @@ struct simd_input { ); } + template + really_inline simd_input map(simd_input b, F const& map_chunk) { + return simd_input( + map_chunk(this->i0, b.i0), + map_chunk(this->i1, b.i1), + map_chunk(this->i2, b.i2), + map_chunk(this->i3, b.i3) + ); + } + really_inline uint64_t to_bitmask() { return neon_movemask_bulk(this->i0, this->i1, this->i2, this->i3); } really_inline uint64_t eq(uint8_t m) { const uint8x16_t mask = vmovq_n_u8(m); - return this->MAP_BITMASK( vceqq_u8(chunk, mask) ); + return this->map( [&](auto a) { + return vceqq_u8(a, mask); + }).to_bitmask(); } really_inline uint64_t lteq(uint8_t m) { const uint8x16_t mask = vmovq_n_u8(m); - return this->MAP_BITMASK( vcleq_u8(chunk, mask) ); + return this->map( [&](auto a) { + return vcleq_u8(a, mask); + }).to_bitmask(); } }; // struct simd_input diff --git a/src/arm64/stage1_find_marks.h b/src/arm64/stage1_find_marks.h index 30fb6d0bb..1cfde5fd1 100644 --- a/src/arm64/stage1_find_marks.h +++ b/src/arm64/stage1_find_marks.h @@ -30,7 +30,7 @@ really_inline void find_whitespace_and_structurals( (uint8x16_t){8, 0, 18, 4, 0, 1, 0, 1, 0, 0, 0, 3, 2, 1, 0, 0}; const uint8x16_t low_nib_and_mask = vmovq_n_u8(0xf); - simd_input v = in.map([&](auto chunk) { + auto v = in.map([&](auto chunk) { uint8x16_t nib_lo = vandq_u8(chunk, low_nib_and_mask); uint8x16_t nib_hi = vshrq_n_u8(chunk, 4); uint8x16_t shuf_lo = vqtbl1q_u8(low_nibble_mask, nib_lo); @@ -39,10 +39,10 @@ really_inline void find_whitespace_and_structurals( }); const uint8x16_t structural_shufti_mask = vmovq_n_u8(0x7); - structurals = v.MAP_BITMASK( vtstq_u8(chunk, structural_shufti_mask) ); + structurals = MAP_BITMASK( v, vtstq_u8(_v, structural_shufti_mask) ); const uint8x16_t whitespace_shufti_mask = vmovq_n_u8(0x18); - whitespace = v.MAP_BITMASK( vtstq_u8(chunk, whitespace_shufti_mask) ); + whitespace = MAP_BITMASK( v, vtstq_u8(_v, whitespace_shufti_mask) ); } #include "generic/stage1_find_marks_flatten.h" diff --git a/src/haswell/simd_input.h b/src/haswell/simd_input.h index 0a2a511b7..3a9f40d69 100644 --- a/src/haswell/simd_input.h +++ b/src/haswell/simd_input.h @@ -31,6 +31,14 @@ struct simd_input { ); } + template + really_inline simd_input map(simd_input b, F const& map_chunk) { + return simd_input( + map_chunk(this->lo, b.lo), + map_chunk(this->hi, b.hi) + ); + } + really_inline uint64_t to_bitmask() { uint64_t r_lo = static_cast(_mm256_movemask_epi8(this->lo)); uint64_t r_hi = _mm256_movemask_epi8(this->hi); @@ -39,12 +47,16 @@ struct simd_input { really_inline uint64_t eq(uint8_t m) { const __m256i mask = _mm256_set1_epi8(m); - return this->MAP_BITMASK( _mm256_cmpeq_epi8(chunk, mask) ); + return this->map( [&](auto a) { + return _mm256_cmpeq_epi8(a, mask); + }).to_bitmask(); } really_inline uint64_t lteq(uint8_t m) { const __m256i maxval = _mm256_set1_epi8(m); - return this->MAP_BITMASK( _mm256_cmpeq_epi8(_mm256_max_epu8(maxval, chunk), maxval) ); + return this->map( [&](auto a) { + return _mm256_cmpeq_epi8(_mm256_max_epu8(maxval, a), maxval); + }).to_bitmask(); } }; // struct simd_input diff --git a/src/haswell/stage1_find_marks.h b/src/haswell/stage1_find_marks.h index 3c0bce143..c6e3d3454 100644 --- a/src/haswell/stage1_find_marks.h +++ b/src/haswell/stage1_find_marks.h @@ -62,7 +62,7 @@ really_inline void find_whitespace_and_structurals(simd_input in, // clang-format off const __m256i structural_table = _mm256_setr_epi8(44, 125, 0, 0, 0xc0u, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 123, - 44, 125, 0, 0, 0xc0u, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 123); + 44, 125, 0, 0, 0xc0u, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 123); const __m256i white_table = _mm256_setr_epi8( 32, 100, 100, 100, 17, 100, 113, 2, 100, 9, 10, 112, 100, 13, 100, 100, 32, 100, 100, 100, 17, 100, 113, 2, 100, 9, 10, 112, 100, 13, 100, 100); @@ -70,14 +70,12 @@ really_inline void find_whitespace_and_structurals(simd_input in, const __m256i struct_offset = _mm256_set1_epi8(0xd4u); const __m256i struct_mask = _mm256_set1_epi8(32); - whitespace = in.MAP_BITMASK( _mm256_cmpeq_epi8(chunk, _mm256_shuffle_epi8(white_table, chunk)) ); - auto struct_r1 = in.MAP_CHUNKS( _mm256_add_epi8(struct_offset, chunk) ); - auto struct_r2 = in.MAP_CHUNKS( _mm256_or_si256(chunk, struct_mask) ); - auto struct_r3 = struct_r1.MAP_CHUNKS( _mm256_shuffle_epi8(structural_table, chunk) ); - structurals = simd_input( - _mm256_cmpeq_epi8(struct_r2.lo, struct_r3.lo), - _mm256_cmpeq_epi8(struct_r2.hi, struct_r3.hi) - ).to_bitmask(); + whitespace = MAP_BITMASK( in, _mm256_cmpeq_epi8(_in, _mm256_shuffle_epi8(white_table, _in)) ); + + auto r1 = MAP_CHUNKS( in, _mm256_add_epi8(struct_offset, _in) ); + auto r2 = MAP_CHUNKS( in, _mm256_or_si256(_in, struct_mask) ); + auto r3 = MAP_CHUNKS( r1, _mm256_shuffle_epi8(structural_table, _r1) ); + structurals = MAP_BITMASK2( r2, r3, _mm256_cmpeq_epi8(_r2, _r3) ); #endif // else SIMDJSON_NAIVE_STRUCTURAL } diff --git a/src/simd_input.h b/src/simd_input.h index 624e5f78a..37bd5863d 100644 --- a/src/simd_input.h +++ b/src/simd_input.h @@ -14,6 +14,9 @@ struct simd_input { // Map through each simd register in this input, producing another simd_input. template really_inline simd_input map(F const& map_chunk); + // Map through each simd register across two inputs, producing a single simd_input. + template + really_inline simd_input map(simd_input b, F const& map_chunk); // turn this bytemask (usually the result of a simd comparison operation) into a bitmask. uint64_t to_bitmask(); // a straightforward comparison of a mask against input. @@ -22,8 +25,10 @@ struct simd_input { uint64_t lteq(uint8_t m); }; // struct simd_input -#define MAP_CHUNKS(EXPR) map([&](auto chunk) { return EXPR; }) -#define MAP_BITMASK(EXPR) map([&](auto chunk) { return EXPR; }).to_bitmask() +#define MAP_CHUNKS(A, EXPR) A.map([&](auto _##A) { return (EXPR); }) +#define MAP_BITMASK(A, EXPR) MAP_CHUNKS(A, EXPR).to_bitmask() +#define MAP_CHUNKS2(A, B, EXPR) A.map((B), [&](auto _##A, auto _##B) { return (EXPR); }) +#define MAP_BITMASK2(A, B, EXPR) MAP_CHUNKS2(A, B, EXPR).to_bitmask() } // namespace simdjson diff --git a/src/westmere/simd_input.h b/src/westmere/simd_input.h index 0e8211232..40f356ee3 100644 --- a/src/westmere/simd_input.h +++ b/src/westmere/simd_input.h @@ -40,6 +40,16 @@ struct simd_input { ); } + template + really_inline simd_input map(simd_input b, F const& map_chunk) { + return simd_input( + map_chunk(this->v0, b.v0), + map_chunk(this->v1, b.v1), + map_chunk(this->v2, b.v2), + map_chunk(this->v3, b.v3) + ); + } + really_inline uint64_t to_bitmask() { uint64_t r0 = static_cast(_mm_movemask_epi8(this->v0)); uint64_t r1 = _mm_movemask_epi8(this->v1); @@ -50,12 +60,16 @@ struct simd_input { really_inline uint64_t eq(uint8_t m) { const __m128i mask = _mm_set1_epi8(m); - return this->MAP_BITMASK( _mm_cmpeq_epi8(chunk, mask) ); + return this->map( [&](auto a) { + return _mm_cmpeq_epi8(a, mask); + }).to_bitmask(); } really_inline uint64_t lteq(uint8_t m) { const __m128i maxval = _mm_set1_epi8(m); - return this->MAP_BITMASK( _mm_cmpeq_epi8(_mm_max_epu8(maxval, chunk), maxval) ); + return this->map( [&](auto a) { + return _mm_cmpeq_epi8(_mm_max_epu8(maxval, a), maxval); + }).to_bitmask(); } }; // struct simd_input diff --git a/src/westmere/stage1_find_marks.h b/src/westmere/stage1_find_marks.h index acaae0d77..50c529d87 100644 --- a/src/westmere/stage1_find_marks.h +++ b/src/westmere/stage1_find_marks.h @@ -23,22 +23,17 @@ really_inline void find_whitespace_and_structurals(simd_input in, const __m128i structural_table = _mm_setr_epi8(44, 125, 0, 0, 0xc0u, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 123); - const __m128i white_table = _mm_setr_epi8(32, 100, 100, 100, 17, 100, 113, 2, - 100, 9, 10, 112, 100, 13, 100, 100); + const __m128i white_table = _mm_setr_epi8(32, 100, 100, 100, 17, 100, 113, 2, + 100, 9, 10, 112, 100, 13, 100, 100); const __m128i struct_offset = _mm_set1_epi8(0xd4u); const __m128i struct_mask = _mm_set1_epi8(32); - whitespace = in.MAP_BITMASK( _mm_cmpeq_epi8(chunk, _mm_shuffle_epi8(white_table, chunk)) ); + whitespace = MAP_BITMASK( in, _mm_cmpeq_epi8(_in, _mm_shuffle_epi8(white_table, _in)) ); - auto r1 = in.MAP_CHUNKS( _mm_add_epi8(struct_offset, chunk) ); - auto r2 = in.MAP_CHUNKS( _mm_or_si128(chunk, struct_mask) ); - auto r3 = r1.MAP_CHUNKS( _mm_shuffle_epi8(structural_table, chunk) ); - structurals = simd_input( - _mm_cmpeq_epi8(r2.v0, r3.v0), - _mm_cmpeq_epi8(r2.v1, r3.v1), - _mm_cmpeq_epi8(r2.v2, r3.v2), - _mm_cmpeq_epi8(r2.v3, r3.v3) - ).to_bitmask(); + auto r1 = MAP_CHUNKS( in, _mm_add_epi8(struct_offset, _in) ); + auto r2 = MAP_CHUNKS( in, _mm_or_si128(_in, struct_mask) ); + auto r3 = MAP_CHUNKS( r1, _mm_shuffle_epi8(structural_table, _r1) ); + structurals = MAP_BITMASK2( r2, r3, _mm_cmpeq_epi8(_r2, _r3) ); } #include "generic/stage1_find_marks_flatten.h" From bf8083888df20d190d7da603315243ab8d15b207 Mon Sep 17 00:00:00 2001 From: John Keiser Date: Mon, 26 Aug 2019 13:35:18 -0700 Subject: [PATCH 9/9] Validate perf against master, not v0.2.1 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5933f9b0d..74ba56315 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -REFERENCE_VERSION = v0.2.1 +REFERENCE_VERSION = master .SUFFIXES: #