diff --git a/cmake/simdjson-flags.cmake b/cmake/simdjson-flags.cmake index d9057cbca..f77685e2e 100644 --- a/cmake/simdjson-flags.cmake +++ b/cmake/simdjson-flags.cmake @@ -86,7 +86,7 @@ else() endif() if(SIMDJSON_VISUAL_STUDIO_BUILD_WITH_DEBUG_INFO_FOR_PROFILING) target_link_options(simdjson-flags INTERFACE /DEBUG ) - target_compile_options(simdjson-flags INTERFACE /Zi ) + target_compile_options(simdjson-flags INTERFACE /Zi) endif() else() target_compile_options(simdjson-internal-flags INTERFACE -fPIC) diff --git a/singleheader/amalgamate_demo.cpp b/singleheader/amalgamate_demo.cpp index d2487ee98..ece273f03 100644 --- a/singleheader/amalgamate_demo.cpp +++ b/singleheader/amalgamate_demo.cpp @@ -1,4 +1,4 @@ -/* auto-generated on Tue 21 Jul 2020 16:54:49 EDT. Do not edit! */ +/* auto-generated on Tue 21 Jul 2020 17:54:23 EDT. Do not edit! */ #include #include "simdjson.h" diff --git a/singleheader/simdjson.cpp b/singleheader/simdjson.cpp index 12e5d1d08..f83542f39 100644 --- a/singleheader/simdjson.cpp +++ b/singleheader/simdjson.cpp @@ -1,4 +1,4 @@ -/* auto-generated on Tue 21 Jul 2020 16:54:49 EDT. Do not edit! */ +/* auto-generated on Tue 21 Jul 2020 17:54:23 EDT. Do not edit! */ /* begin file src/simdjson.cpp */ #include "simdjson.h" @@ -2646,10 +2646,14 @@ really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, int8_ template struct simd8x64 { - static const int NUM_CHUNKS = 64 / sizeof(simd8); + static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); + static_assert(NUM_CHUNKS == 4, "ARM kernel should use four registers per 64-byte block."); const simd8 chunks[NUM_CHUNKS]; - really_inline simd8x64() : chunks{simd8(), simd8(), simd8(), simd8()} {} + simd8x64(const simd8x64& o) = delete; // no copy allowed + simd8x64& operator=(const simd8 other) = delete; // no assignment allowed + simd8x64() = delete; // no default constructor allowed + really_inline simd8x64(const simd8 chunk0, const simd8 chunk1, const simd8 chunk2, const simd8 chunk3) : chunks{chunk0, chunk1, chunk2, chunk3} {} really_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr), simd8::load(ptr+16), simd8::load(ptr+32), simd8::load(ptr+48)} {} @@ -2660,6 +2664,11 @@ really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, int8_ this->chunks[3].store(ptr+sizeof(simd8)*3); } + really_inline simd8 reduce_or() const { + return (this->chunks[0] | this->chunks[1]) | (this->chunks[2] | this->chunks[3]); + } + + really_inline void compress(uint64_t mask, T * output) const { this->chunks[0].compress(uint16_t(mask), output); this->chunks[1].compress(uint16_t(mask >> 16), output + 16 - count_ones(mask & 0xFFFF)); @@ -2667,14 +2676,6 @@ really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, int8_ this->chunks[3].compress(uint16_t(mask >> 48), output + 48 - count_ones(mask & 0xFFFFFFFFFFFF)); } - template - static really_inline void each_index(F const& each) { - each(0); - each(1); - each(2); - each(3); - } - really_inline uint64_t to_bitmask() const { #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO const uint8x16_t bit_mask = make_uint8x16_t( @@ -2740,7 +2741,7 @@ namespace arm64 { using namespace simd; struct json_character_block { - static really_inline json_character_block classify(const simd::simd8x64 in); + static really_inline json_character_block classify(const simd::simd8x64& in); really_inline uint64_t whitespace() const { return _whitespace; } really_inline uint64_t op() const { return _op; } @@ -2750,7 +2751,7 @@ struct json_character_block { uint64_t _op; }; -really_inline json_character_block json_character_block::classify(const simd::simd8x64 in) { +really_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { // Functional programming causes trouble with Visual Studio. // Keeping this version in comments since it is much nicer: // auto v = in.map([&](simd8 chunk) { @@ -2763,7 +2764,7 @@ really_inline json_character_block json_character_block::classify(const simd::si const simd8 table1(16, 0, 0, 0, 0, 0, 0, 0, 0, 8, 12, 1, 2, 9, 0, 0); const simd8 table2(8, 0, 18, 4, 0, 1, 0, 1, 0, 0, 0, 3, 2, 1, 0, 0); - auto v = simd8x64( + simd8x64 v( (in.chunks[0] & 0xf).lookup_16(table1) & (in.chunks[0].shr<4>()).lookup_16(table2), (in.chunks[1] & 0xf).lookup_16(table1) & (in.chunks[1].shr<4>()).lookup_16(table2), (in.chunks[2] & 0xf).lookup_16(table1) & (in.chunks[2].shr<4>()).lookup_16(table2), @@ -2804,12 +2805,12 @@ really_inline json_character_block json_character_block::classify(const simd::si return { whitespace, op }; } -really_inline bool is_ascii(simd8x64 input) { - simd8 bits = (input.chunks[0] | input.chunks[1]) | (input.chunks[2] | input.chunks[3]); +really_inline bool is_ascii(const simd8x64& input) { + simd8 bits = input.reduce_or(); return bits.max() < 0b10000000u; } -really_inline simd8 must_be_continuation(simd8 prev1, simd8 prev2, simd8 prev3) { +really_inline simd8 must_be_continuation(const simd8 prev1, const simd8 prev2, const simd8 prev3) { simd8 is_second_byte = prev1 >= uint8_t(0b11000000u); simd8 is_third_byte = prev2 >= uint8_t(0b11100000u); simd8 is_fourth_byte = prev3 >= uint8_t(0b11110000u); @@ -2821,7 +2822,7 @@ really_inline simd8 must_be_continuation(simd8 prev1, simd8 must_be_2_3_continuation(simd8 prev2, simd8 prev3) { +really_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { simd8 is_third_byte = prev2 >= uint8_t(0b11100000u); simd8 is_fourth_byte = prev3 >= uint8_t(0b11110000u); return is_third_byte ^ is_fourth_byte; @@ -2865,7 +2866,7 @@ UNUSED static char * format_input_text_64(const uint8_t *text) { } // Routines to print masks and text for debugging bitmask operations -UNUSED static char * format_input_text(const simd8x64 in) { +UNUSED static char * format_input_text(const simd8x64& in) { static char *buf = (char*)malloc(sizeof(simd8x64) + 1); in.store((uint8_t*)buf); for (size_t i=0; i); i++) { @@ -2948,7 +2949,7 @@ struct json_string_block { // Scans blocks for string characters, storing the state necessary to do so class json_string_scanner { public: - really_inline json_string_block next(const simd::simd8x64 in); + really_inline json_string_block next(const simd::simd8x64& in); really_inline error_code finish(bool streaming); private: @@ -3014,7 +3015,7 @@ really_inline uint64_t json_string_scanner::find_escaped_branchless(uint64_t bac // // Backslash sequences outside of quotes will be detected in stage 2. // -really_inline json_string_block json_string_scanner::next(const simd::simd8x64 in) { +really_inline json_string_block json_string_scanner::next(const simd::simd8x64& in) { const uint64_t backslash = in.eq('\\'); const uint64_t escaped = find_escaped(backslash); const uint64_t quote = in.eq('"') & ~escaped; @@ -3105,7 +3106,7 @@ private: class json_scanner { public: json_scanner() {} - really_inline json_block next(const simd::simd8x64 in); + really_inline json_block next(const simd::simd8x64& in); really_inline error_code finish(bool streaming); private: @@ -3142,7 +3143,7 @@ really_inline uint64_t follows(const uint64_t match, const uint64_t filler, uint return result; } -really_inline json_block json_scanner::next(const simd::simd8x64 in) { +really_inline json_block json_scanner::next(const simd::simd8x64& in) { json_string_block strings = string_scanner.next(in); json_character_block characters = json_character_block::classify(in); uint64_t follows_scalar = follows(characters.scalar(), prev_scalar); @@ -3188,13 +3189,13 @@ private: {} template really_inline void step(const uint8_t *block_buf, buf_block_reader &reader) noexcept; - really_inline void next(simd::simd8x64 in, json_block block); + really_inline void next(const simd::simd8x64& in, json_block block); really_inline error_code finish(uint8_t *dst_start, size_t &dst_len); json_scanner scanner{}; uint8_t *dst; }; -really_inline void json_minifier::next(simd::simd8x64 in, json_block block) { +really_inline void json_minifier::next(const simd::simd8x64& in, json_block block) { uint64_t mask = block.whitespace(); in.compress(mask, dst); dst += 64 - count_ones(mask); @@ -3347,7 +3348,7 @@ namespace utf8_validation { using namespace simd; - really_inline simd8 check_special_cases(const simd8& input, const simd8& prev1) { + really_inline simd8 check_special_cases(const simd8 input, const simd8 prev1) { // Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII) // Bit 1 = Too Long (ASCII followed by continuation) // Bit 2 = Overlong 3-byte @@ -3437,8 +3438,8 @@ using namespace simd; ); return (byte_1_high & byte_1_low & byte_2_high); } - really_inline simd8 check_multibyte_lengths(const simd8& input, - const simd8& prev_input, const simd8& sc) { + really_inline simd8 check_multibyte_lengths(const simd8 input, + const simd8 prev_input, const simd8 sc) { simd8 prev2 = input.prev<2>(prev_input); simd8 prev3 = input.prev<3>(prev_input); simd8 must23 = simd8(must_be_2_3_continuation(prev2, prev3)); @@ -3450,7 +3451,7 @@ using namespace simd; // Return nonzero if there are incomplete multibyte characters at the end of the block: // e.g. if there is a 4-byte character, but it's 3 bytes from the end. // - really_inline simd8 is_incomplete(const simd8& input) { + really_inline simd8 is_incomplete(const simd8 input) { // If the previous input's last 3 bytes match this, they're too short (they ended at EOF): // ... 1111____ 111_____ 11______ static const uint8_t max_array[32] = { @@ -3474,7 +3475,7 @@ using namespace simd; // // Check whether the current bytes are valid UTF-8. // - really_inline void check_utf8_bytes(const simd8& input, const simd8& prev_input) { + really_inline void check_utf8_bytes(const simd8 input, const simd8 prev_input) { // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ lead bytes // (2, 3, 4-byte leads become large positive numbers instead of small negative numbers) simd8 prev1 = input.prev<1>(prev_input); @@ -3490,7 +3491,9 @@ using namespace simd; } really_inline void check_next_input(const simd8x64& input) { - if (unlikely(!is_ascii(input))) { + if(likely(is_ascii(input))) { + this->error |= this->prev_incomplete; + } else { // you might think that a for-loop would work, but under Visual Studio, it is not good enough. static_assert((simd8x64::NUM_CHUNKS == 2) || (simd8x64::NUM_CHUNKS == 4), "We support either two or four chunks per 64-byte block."); @@ -3505,10 +3508,7 @@ using namespace simd; } this->prev_incomplete = is_incomplete(input.chunks[simd8x64::NUM_CHUNKS-1]); this->prev_input_block = input.chunks[simd8x64::NUM_CHUNKS-1]; - } else { - // If the previous block had incomplete UTF-8 characters at the end, an ASCII block can't - // possibly finish them. - this->error |= this->prev_incomplete; + } } @@ -3595,7 +3595,7 @@ private: really_inline json_structural_indexer(uint32_t *structural_indexes); template really_inline void step(const uint8_t *block, buf_block_reader &reader) noexcept; - really_inline void next(simd::simd8x64 in, json_block block, size_t idx); + really_inline void next(const simd::simd8x64& in, json_block block, size_t idx); really_inline error_code finish(dom_parser_implementation &parser, size_t idx, size_t len, bool partial); json_scanner scanner{}; @@ -3664,7 +3664,7 @@ really_inline void json_structural_indexer::step<64>(const uint8_t *block, buf_b reader.advance(); } -really_inline void json_structural_indexer::next(simd::simd8x64 in, json_block block, size_t idx) { +really_inline void json_structural_indexer::next(const simd::simd8x64& in, json_block block, size_t idx) { uint64_t unescaped = in.lteq(0x1F); checker.check_next_input(in); indexer.write(uint32_t(idx-64), prev_structurals); // Output *last* iteration's structurals to the parser @@ -7902,6 +7902,7 @@ namespace simd { really_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } really_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } really_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } + really_inline bool is_ascii() const { return _mm256_movemask_epi8(*this) == 0; } really_inline bool bits_not_set_anywhere() const { return _mm256_testz_si256(*this, *this); } really_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } really_inline bool bits_not_set_anywhere(simd8 bits) const { return _mm256_testz_si256(*this, bits); } @@ -7918,19 +7919,17 @@ namespace simd { template struct simd8x64 { - static const int NUM_CHUNKS = 64 / sizeof(simd8); + static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); + static_assert(NUM_CHUNKS == 2, "Haswell kernel should use two registers per 64-byte block."); const simd8 chunks[NUM_CHUNKS]; - really_inline simd8x64() : chunks{simd8(), simd8()} {} + simd8x64(const simd8x64& o) = delete; // no copy allowed + simd8x64& operator=(const simd8 other) = delete; // no assignment allowed + simd8x64() = delete; // no default constructor allowed + really_inline simd8x64(const simd8 chunk0, const simd8 chunk1) : chunks{chunk0, chunk1} {} really_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr), simd8::load(ptr+32)} {} - template - static really_inline void each_index(F const& each) { - each(0); - each(1); - } - really_inline void compress(uint64_t mask, T * output) const { uint32_t mask1 = uint32_t(mask); uint32_t mask2 = uint32_t(mask >> 32); @@ -7949,6 +7948,10 @@ namespace simd { return r_lo | (r_hi << 32); } + really_inline simd8 reduce_or() const { + return this->chunks[0] | this->chunks[1]; + } + really_inline simd8x64 bit_or(const T m) const { const simd8 mask = simd8::splat(m); return simd8x64( @@ -7991,7 +7994,7 @@ namespace haswell { using namespace simd; struct json_character_block { - static really_inline json_character_block classify(const simd::simd8x64 in); + static really_inline json_character_block classify(const simd::simd8x64& in); really_inline uint64_t whitespace() const { return _whitespace; } really_inline uint64_t op() const { return _op; } @@ -8001,7 +8004,7 @@ struct json_character_block { uint64_t _op; }; -really_inline json_character_block json_character_block::classify(const simd::simd8x64 in) { +really_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { // These lookups rely on the fact that anything < 127 will match the lower 4 bits, which is why // we can't use the generic lookup_16. auto whitespace_table = simd8::repeat_16(' ', 100, 100, 100, 17, 100, 113, 2, 100, '\t', '\n', 112, 100, '\r', 100, 100); @@ -8024,12 +8027,11 @@ really_inline json_character_block json_character_block::classify(const simd::si return { whitespace, op }; } -really_inline bool is_ascii(simd8x64 input) { - simd8 bits = (input.chunks[0] | input.chunks[1]); - return !bits.any_bits_set_anywhere(0b10000000u); +really_inline bool is_ascii(const simd8x64& input) { + return input.reduce_or().is_ascii(); } -really_inline simd8 must_be_continuation(simd8 prev1, simd8 prev2, simd8 prev3) { +really_inline simd8 must_be_continuation(const simd8 prev1, const simd8 prev2, const simd8 prev3) { simd8 is_second_byte = prev1.saturating_sub(0b11000000u-1); // Only 11______ will be > 0 simd8 is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0 simd8 is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0 @@ -8037,7 +8039,7 @@ really_inline simd8 must_be_continuation(simd8 prev1, simd8(is_second_byte | is_third_byte | is_fourth_byte) > int8_t(0); } -really_inline simd8 must_be_2_3_continuation(simd8 prev2, simd8 prev3) { +really_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { simd8 is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0 simd8 is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0 // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine. @@ -8083,7 +8085,7 @@ UNUSED static char * format_input_text_64(const uint8_t *text) { } // Routines to print masks and text for debugging bitmask operations -UNUSED static char * format_input_text(const simd8x64 in) { +UNUSED static char * format_input_text(const simd8x64& in) { static char *buf = (char*)malloc(sizeof(simd8x64) + 1); in.store((uint8_t*)buf); for (size_t i=0; i); i++) { @@ -8166,7 +8168,7 @@ struct json_string_block { // Scans blocks for string characters, storing the state necessary to do so class json_string_scanner { public: - really_inline json_string_block next(const simd::simd8x64 in); + really_inline json_string_block next(const simd::simd8x64& in); really_inline error_code finish(bool streaming); private: @@ -8232,7 +8234,7 @@ really_inline uint64_t json_string_scanner::find_escaped_branchless(uint64_t bac // // Backslash sequences outside of quotes will be detected in stage 2. // -really_inline json_string_block json_string_scanner::next(const simd::simd8x64 in) { +really_inline json_string_block json_string_scanner::next(const simd::simd8x64& in) { const uint64_t backslash = in.eq('\\'); const uint64_t escaped = find_escaped(backslash); const uint64_t quote = in.eq('"') & ~escaped; @@ -8323,7 +8325,7 @@ private: class json_scanner { public: json_scanner() {} - really_inline json_block next(const simd::simd8x64 in); + really_inline json_block next(const simd::simd8x64& in); really_inline error_code finish(bool streaming); private: @@ -8360,7 +8362,7 @@ really_inline uint64_t follows(const uint64_t match, const uint64_t filler, uint return result; } -really_inline json_block json_scanner::next(const simd::simd8x64 in) { +really_inline json_block json_scanner::next(const simd::simd8x64& in) { json_string_block strings = string_scanner.next(in); json_character_block characters = json_character_block::classify(in); uint64_t follows_scalar = follows(characters.scalar(), prev_scalar); @@ -8404,13 +8406,13 @@ private: {} template really_inline void step(const uint8_t *block_buf, buf_block_reader &reader) noexcept; - really_inline void next(simd::simd8x64 in, json_block block); + really_inline void next(const simd::simd8x64& in, json_block block); really_inline error_code finish(uint8_t *dst_start, size_t &dst_len); json_scanner scanner{}; uint8_t *dst; }; -really_inline void json_minifier::next(simd::simd8x64 in, json_block block) { +really_inline void json_minifier::next(const simd::simd8x64& in, json_block block) { uint64_t mask = block.whitespace(); in.compress(mask, dst); dst += 64 - count_ones(mask); @@ -8563,7 +8565,7 @@ namespace utf8_validation { using namespace simd; - really_inline simd8 check_special_cases(const simd8& input, const simd8& prev1) { + really_inline simd8 check_special_cases(const simd8 input, const simd8 prev1) { // Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII) // Bit 1 = Too Long (ASCII followed by continuation) // Bit 2 = Overlong 3-byte @@ -8653,8 +8655,8 @@ using namespace simd; ); return (byte_1_high & byte_1_low & byte_2_high); } - really_inline simd8 check_multibyte_lengths(const simd8& input, - const simd8& prev_input, const simd8& sc) { + really_inline simd8 check_multibyte_lengths(const simd8 input, + const simd8 prev_input, const simd8 sc) { simd8 prev2 = input.prev<2>(prev_input); simd8 prev3 = input.prev<3>(prev_input); simd8 must23 = simd8(must_be_2_3_continuation(prev2, prev3)); @@ -8666,7 +8668,7 @@ using namespace simd; // Return nonzero if there are incomplete multibyte characters at the end of the block: // e.g. if there is a 4-byte character, but it's 3 bytes from the end. // - really_inline simd8 is_incomplete(const simd8& input) { + really_inline simd8 is_incomplete(const simd8 input) { // If the previous input's last 3 bytes match this, they're too short (they ended at EOF): // ... 1111____ 111_____ 11______ static const uint8_t max_array[32] = { @@ -8690,7 +8692,7 @@ using namespace simd; // // Check whether the current bytes are valid UTF-8. // - really_inline void check_utf8_bytes(const simd8& input, const simd8& prev_input) { + really_inline void check_utf8_bytes(const simd8 input, const simd8 prev_input) { // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ lead bytes // (2, 3, 4-byte leads become large positive numbers instead of small negative numbers) simd8 prev1 = input.prev<1>(prev_input); @@ -8706,7 +8708,9 @@ using namespace simd; } really_inline void check_next_input(const simd8x64& input) { - if (unlikely(!is_ascii(input))) { + if(likely(is_ascii(input))) { + this->error |= this->prev_incomplete; + } else { // you might think that a for-loop would work, but under Visual Studio, it is not good enough. static_assert((simd8x64::NUM_CHUNKS == 2) || (simd8x64::NUM_CHUNKS == 4), "We support either two or four chunks per 64-byte block."); @@ -8721,10 +8725,7 @@ using namespace simd; } this->prev_incomplete = is_incomplete(input.chunks[simd8x64::NUM_CHUNKS-1]); this->prev_input_block = input.chunks[simd8x64::NUM_CHUNKS-1]; - } else { - // If the previous block had incomplete UTF-8 characters at the end, an ASCII block can't - // possibly finish them. - this->error |= this->prev_incomplete; + } } @@ -8811,7 +8812,7 @@ private: really_inline json_structural_indexer(uint32_t *structural_indexes); template really_inline void step(const uint8_t *block, buf_block_reader &reader) noexcept; - really_inline void next(simd::simd8x64 in, json_block block, size_t idx); + really_inline void next(const simd::simd8x64& in, json_block block, size_t idx); really_inline error_code finish(dom_parser_implementation &parser, size_t idx, size_t len, bool partial); json_scanner scanner{}; @@ -8880,7 +8881,7 @@ really_inline void json_structural_indexer::step<64>(const uint8_t *block, buf_b reader.advance(); } -really_inline void json_structural_indexer::next(simd::simd8x64 in, json_block block, size_t idx) { +really_inline void json_structural_indexer::next(const simd::simd8x64& in, json_block block, size_t idx) { uint64_t unescaped = in.lteq(0x1F); checker.check_next_input(in); indexer.write(uint32_t(idx-64), prev_structurals); // Output *last* iteration's structurals to the parser @@ -11006,6 +11007,7 @@ namespace simd { really_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } really_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } really_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } + really_inline bool is_ascii() const { return _mm_movemask_epi8(*this) == 0; } really_inline bool bits_not_set_anywhere() const { return _mm_testz_si128(*this, *this); } really_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } really_inline bool bits_not_set_anywhere(simd8 bits) const { return _mm_testz_si128(*this, bits); } @@ -11022,10 +11024,14 @@ namespace simd { template struct simd8x64 { - static const int NUM_CHUNKS = 64 / sizeof(simd8); + static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); + static_assert(NUM_CHUNKS == 4, "Westmere kernel should use four registers per 64-byte block."); const simd8 chunks[NUM_CHUNKS]; - really_inline simd8x64() : chunks{simd8(), simd8(), simd8(), simd8()} {} + simd8x64(const simd8x64& o) = delete; // no copy allowed + simd8x64& operator=(const simd8 other) = delete; // no assignment allowed + simd8x64() = delete; // no default constructor allowed + really_inline simd8x64(const simd8 chunk0, const simd8 chunk1, const simd8 chunk2, const simd8 chunk3) : chunks{chunk0, chunk1, chunk2, chunk3} {} really_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr), simd8::load(ptr+16), simd8::load(ptr+32), simd8::load(ptr+48)} {} @@ -11036,6 +11042,10 @@ namespace simd { this->chunks[3].store(ptr+sizeof(simd8)*3); } + really_inline simd8 reduce_or() const { + return (this->chunks[0] | this->chunks[1]) | (this->chunks[2] | this->chunks[3]); + } + really_inline void compress(uint64_t mask, T * output) const { this->chunks[0].compress(uint16_t(mask), output); this->chunks[1].compress(uint16_t(mask >> 16), output + 16 - count_ones(mask & 0xFFFF)); @@ -11043,14 +11053,6 @@ namespace simd { this->chunks[3].compress(uint16_t(mask >> 48), output + 48 - count_ones(mask & 0xFFFFFFFFFFFF)); } - template - static really_inline void each_index(F const& each) { - each(0); - each(1); - each(2); - each(3); - } - really_inline uint64_t to_bitmask() const { uint64_t r0 = uint32_t(this->chunks[0].to_bitmask()); uint64_t r1 = this->chunks[1].to_bitmask(); @@ -11108,7 +11110,7 @@ namespace westmere { using namespace simd; struct json_character_block { - static really_inline json_character_block classify(const simd::simd8x64 in); + static really_inline json_character_block classify(const simd::simd8x64& in); really_inline uint64_t whitespace() const { return _whitespace; } really_inline uint64_t op() const { return _op; } @@ -11118,7 +11120,7 @@ struct json_character_block { uint64_t _op; }; -really_inline json_character_block json_character_block::classify(const simd::simd8x64 in) { +really_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { // These lookups rely on the fact that anything < 127 will match the lower 4 bits, which is why // we can't use the generic lookup_16. auto whitespace_table = simd8::repeat_16(' ', 100, 100, 100, 17, 100, 113, 2, 100, '\t', '\n', 112, 100, '\r', 100, 100); @@ -11146,12 +11148,11 @@ really_inline json_character_block json_character_block::classify(const simd::si return { whitespace, op }; } -really_inline bool is_ascii(simd8x64 input) { - simd8 bits = (input.chunks[0] | input.chunks[1]) | (input.chunks[2] | input.chunks[3]); - return !bits.any_bits_set_anywhere(0b10000000u); +really_inline bool is_ascii(const simd8x64& input) { + return input.reduce_or().is_ascii(); } -really_inline simd8 must_be_continuation(simd8 prev1, simd8 prev2, simd8 prev3) { +really_inline simd8 must_be_continuation(const simd8 prev1, const simd8 prev2, const simd8 prev3) { simd8 is_second_byte = prev1.saturating_sub(0b11000000u-1); // Only 11______ will be > 0 simd8 is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0 simd8 is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0 @@ -11159,7 +11160,7 @@ really_inline simd8 must_be_continuation(simd8 prev1, simd8(is_second_byte | is_third_byte | is_fourth_byte) > int8_t(0); } -really_inline simd8 must_be_2_3_continuation(simd8 prev2, simd8 prev3) { +really_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { simd8 is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0 simd8 is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0 // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine. @@ -11205,7 +11206,7 @@ UNUSED static char * format_input_text_64(const uint8_t *text) { } // Routines to print masks and text for debugging bitmask operations -UNUSED static char * format_input_text(const simd8x64 in) { +UNUSED static char * format_input_text(const simd8x64& in) { static char *buf = (char*)malloc(sizeof(simd8x64) + 1); in.store((uint8_t*)buf); for (size_t i=0; i); i++) { @@ -11288,7 +11289,7 @@ struct json_string_block { // Scans blocks for string characters, storing the state necessary to do so class json_string_scanner { public: - really_inline json_string_block next(const simd::simd8x64 in); + really_inline json_string_block next(const simd::simd8x64& in); really_inline error_code finish(bool streaming); private: @@ -11354,7 +11355,7 @@ really_inline uint64_t json_string_scanner::find_escaped_branchless(uint64_t bac // // Backslash sequences outside of quotes will be detected in stage 2. // -really_inline json_string_block json_string_scanner::next(const simd::simd8x64 in) { +really_inline json_string_block json_string_scanner::next(const simd::simd8x64& in) { const uint64_t backslash = in.eq('\\'); const uint64_t escaped = find_escaped(backslash); const uint64_t quote = in.eq('"') & ~escaped; @@ -11445,7 +11446,7 @@ private: class json_scanner { public: json_scanner() {} - really_inline json_block next(const simd::simd8x64 in); + really_inline json_block next(const simd::simd8x64& in); really_inline error_code finish(bool streaming); private: @@ -11482,7 +11483,7 @@ really_inline uint64_t follows(const uint64_t match, const uint64_t filler, uint return result; } -really_inline json_block json_scanner::next(const simd::simd8x64 in) { +really_inline json_block json_scanner::next(const simd::simd8x64& in) { json_string_block strings = string_scanner.next(in); json_character_block characters = json_character_block::classify(in); uint64_t follows_scalar = follows(characters.scalar(), prev_scalar); @@ -11526,13 +11527,13 @@ private: {} template really_inline void step(const uint8_t *block_buf, buf_block_reader &reader) noexcept; - really_inline void next(simd::simd8x64 in, json_block block); + really_inline void next(const simd::simd8x64& in, json_block block); really_inline error_code finish(uint8_t *dst_start, size_t &dst_len); json_scanner scanner{}; uint8_t *dst; }; -really_inline void json_minifier::next(simd::simd8x64 in, json_block block) { +really_inline void json_minifier::next(const simd::simd8x64& in, json_block block) { uint64_t mask = block.whitespace(); in.compress(mask, dst); dst += 64 - count_ones(mask); @@ -11685,7 +11686,7 @@ namespace utf8_validation { using namespace simd; - really_inline simd8 check_special_cases(const simd8& input, const simd8& prev1) { + really_inline simd8 check_special_cases(const simd8 input, const simd8 prev1) { // Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII) // Bit 1 = Too Long (ASCII followed by continuation) // Bit 2 = Overlong 3-byte @@ -11775,8 +11776,8 @@ using namespace simd; ); return (byte_1_high & byte_1_low & byte_2_high); } - really_inline simd8 check_multibyte_lengths(const simd8& input, - const simd8& prev_input, const simd8& sc) { + really_inline simd8 check_multibyte_lengths(const simd8 input, + const simd8 prev_input, const simd8 sc) { simd8 prev2 = input.prev<2>(prev_input); simd8 prev3 = input.prev<3>(prev_input); simd8 must23 = simd8(must_be_2_3_continuation(prev2, prev3)); @@ -11788,7 +11789,7 @@ using namespace simd; // Return nonzero if there are incomplete multibyte characters at the end of the block: // e.g. if there is a 4-byte character, but it's 3 bytes from the end. // - really_inline simd8 is_incomplete(const simd8& input) { + really_inline simd8 is_incomplete(const simd8 input) { // If the previous input's last 3 bytes match this, they're too short (they ended at EOF): // ... 1111____ 111_____ 11______ static const uint8_t max_array[32] = { @@ -11812,7 +11813,7 @@ using namespace simd; // // Check whether the current bytes are valid UTF-8. // - really_inline void check_utf8_bytes(const simd8& input, const simd8& prev_input) { + really_inline void check_utf8_bytes(const simd8 input, const simd8 prev_input) { // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ lead bytes // (2, 3, 4-byte leads become large positive numbers instead of small negative numbers) simd8 prev1 = input.prev<1>(prev_input); @@ -11828,7 +11829,9 @@ using namespace simd; } really_inline void check_next_input(const simd8x64& input) { - if (unlikely(!is_ascii(input))) { + if(likely(is_ascii(input))) { + this->error |= this->prev_incomplete; + } else { // you might think that a for-loop would work, but under Visual Studio, it is not good enough. static_assert((simd8x64::NUM_CHUNKS == 2) || (simd8x64::NUM_CHUNKS == 4), "We support either two or four chunks per 64-byte block."); @@ -11843,10 +11846,7 @@ using namespace simd; } this->prev_incomplete = is_incomplete(input.chunks[simd8x64::NUM_CHUNKS-1]); this->prev_input_block = input.chunks[simd8x64::NUM_CHUNKS-1]; - } else { - // If the previous block had incomplete UTF-8 characters at the end, an ASCII block can't - // possibly finish them. - this->error |= this->prev_incomplete; + } } @@ -11933,7 +11933,7 @@ private: really_inline json_structural_indexer(uint32_t *structural_indexes); template really_inline void step(const uint8_t *block, buf_block_reader &reader) noexcept; - really_inline void next(simd::simd8x64 in, json_block block, size_t idx); + really_inline void next(const simd::simd8x64& in, json_block block, size_t idx); really_inline error_code finish(dom_parser_implementation &parser, size_t idx, size_t len, bool partial); json_scanner scanner{}; @@ -12002,7 +12002,7 @@ really_inline void json_structural_indexer::step<64>(const uint8_t *block, buf_b reader.advance(); } -really_inline void json_structural_indexer::next(simd::simd8x64 in, json_block block, size_t idx) { +really_inline void json_structural_indexer::next(const simd::simd8x64& in, json_block block, size_t idx) { uint64_t unescaped = in.lteq(0x1F); checker.check_next_input(in); indexer.write(uint32_t(idx-64), prev_structurals); // Output *last* iteration's structurals to the parser diff --git a/singleheader/simdjson.h b/singleheader/simdjson.h index f66d2bd14..4a577b32f 100644 --- a/singleheader/simdjson.h +++ b/singleheader/simdjson.h @@ -1,4 +1,4 @@ -/* auto-generated on Tue 21 Jul 2020 16:54:49 EDT. Do not edit! */ +/* auto-generated on Tue 21 Jul 2020 17:54:23 EDT. Do not edit! */ /* begin file include/simdjson.h */ #ifndef SIMDJSON_H #define SIMDJSON_H diff --git a/src/arm64/dom_parser_implementation.cpp b/src/arm64/dom_parser_implementation.cpp index 5ad427838..5ac421d4e 100644 --- a/src/arm64/dom_parser_implementation.cpp +++ b/src/arm64/dom_parser_implementation.cpp @@ -15,7 +15,7 @@ namespace arm64 { using namespace simd; struct json_character_block { - static really_inline json_character_block classify(const simd::simd8x64 in); + static really_inline json_character_block classify(const simd::simd8x64& in); really_inline uint64_t whitespace() const { return _whitespace; } really_inline uint64_t op() const { return _op; } @@ -25,7 +25,7 @@ struct json_character_block { uint64_t _op; }; -really_inline json_character_block json_character_block::classify(const simd::simd8x64 in) { +really_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { // Functional programming causes trouble with Visual Studio. // Keeping this version in comments since it is much nicer: // auto v = in.map([&](simd8 chunk) { @@ -38,7 +38,7 @@ really_inline json_character_block json_character_block::classify(const simd::si const simd8 table1(16, 0, 0, 0, 0, 0, 0, 0, 0, 8, 12, 1, 2, 9, 0, 0); const simd8 table2(8, 0, 18, 4, 0, 1, 0, 1, 0, 0, 0, 3, 2, 1, 0, 0); - auto v = simd8x64( + simd8x64 v( (in.chunks[0] & 0xf).lookup_16(table1) & (in.chunks[0].shr<4>()).lookup_16(table2), (in.chunks[1] & 0xf).lookup_16(table1) & (in.chunks[1].shr<4>()).lookup_16(table2), (in.chunks[2] & 0xf).lookup_16(table1) & (in.chunks[2].shr<4>()).lookup_16(table2), @@ -79,12 +79,12 @@ really_inline json_character_block json_character_block::classify(const simd::si return { whitespace, op }; } -really_inline bool is_ascii(simd8x64 input) { - simd8 bits = (input.chunks[0] | input.chunks[1]) | (input.chunks[2] | input.chunks[3]); +really_inline bool is_ascii(const simd8x64& input) { + simd8 bits = input.reduce_or(); return bits.max() < 0b10000000u; } -really_inline simd8 must_be_continuation(simd8 prev1, simd8 prev2, simd8 prev3) { +really_inline simd8 must_be_continuation(const simd8 prev1, const simd8 prev2, const simd8 prev3) { simd8 is_second_byte = prev1 >= uint8_t(0b11000000u); simd8 is_third_byte = prev2 >= uint8_t(0b11100000u); simd8 is_fourth_byte = prev3 >= uint8_t(0b11110000u); @@ -96,7 +96,7 @@ really_inline simd8 must_be_continuation(simd8 prev1, simd8 must_be_2_3_continuation(simd8 prev2, simd8 prev3) { +really_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { simd8 is_third_byte = prev2 >= uint8_t(0b11100000u); simd8 is_fourth_byte = prev3 >= uint8_t(0b11110000u); return is_third_byte ^ is_fourth_byte; diff --git a/src/arm64/simd.h b/src/arm64/simd.h index a89098cdb..c00ae99e2 100644 --- a/src/arm64/simd.h +++ b/src/arm64/simd.h @@ -413,10 +413,14 @@ really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, int8_ template struct simd8x64 { - static const int NUM_CHUNKS = 64 / sizeof(simd8); + static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); + static_assert(NUM_CHUNKS == 4, "ARM kernel should use four registers per 64-byte block."); const simd8 chunks[NUM_CHUNKS]; - really_inline simd8x64() : chunks{simd8(), simd8(), simd8(), simd8()} {} + simd8x64(const simd8x64& o) = delete; // no copy allowed + simd8x64& operator=(const simd8 other) = delete; // no assignment allowed + simd8x64() = delete; // no default constructor allowed + really_inline simd8x64(const simd8 chunk0, const simd8 chunk1, const simd8 chunk2, const simd8 chunk3) : chunks{chunk0, chunk1, chunk2, chunk3} {} really_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr), simd8::load(ptr+16), simd8::load(ptr+32), simd8::load(ptr+48)} {} @@ -427,6 +431,11 @@ really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, int8_ this->chunks[3].store(ptr+sizeof(simd8)*3); } + really_inline simd8 reduce_or() const { + return (this->chunks[0] | this->chunks[1]) | (this->chunks[2] | this->chunks[3]); + } + + really_inline void compress(uint64_t mask, T * output) const { this->chunks[0].compress(uint16_t(mask), output); this->chunks[1].compress(uint16_t(mask >> 16), output + 16 - count_ones(mask & 0xFFFF)); @@ -434,14 +443,6 @@ really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, int8_ this->chunks[3].compress(uint16_t(mask >> 48), output + 48 - count_ones(mask & 0xFFFFFFFFFFFF)); } - template - static really_inline void each_index(F const& each) { - each(0); - each(1); - each(2); - each(3); - } - really_inline uint64_t to_bitmask() const { #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO const uint8x16_t bit_mask = make_uint8x16_t( diff --git a/src/generic/stage1/buf_block_reader.h b/src/generic/stage1/buf_block_reader.h index 7a82c5a72..fb5c07e7f 100644 --- a/src/generic/stage1/buf_block_reader.h +++ b/src/generic/stage1/buf_block_reader.h @@ -35,7 +35,7 @@ UNUSED static char * format_input_text_64(const uint8_t *text) { } // Routines to print masks and text for debugging bitmask operations -UNUSED static char * format_input_text(const simd8x64 in) { +UNUSED static char * format_input_text(const simd8x64& in) { static char *buf = (char*)malloc(sizeof(simd8x64) + 1); in.store((uint8_t*)buf); for (size_t i=0; i); i++) { diff --git a/src/generic/stage1/json_minifier.h b/src/generic/stage1/json_minifier.h index b51d9f3ad..4055d90ae 100644 --- a/src/generic/stage1/json_minifier.h +++ b/src/generic/stage1/json_minifier.h @@ -16,13 +16,13 @@ private: {} template really_inline void step(const uint8_t *block_buf, buf_block_reader &reader) noexcept; - really_inline void next(simd::simd8x64 in, json_block block); + really_inline void next(const simd::simd8x64& in, json_block block); really_inline error_code finish(uint8_t *dst_start, size_t &dst_len); json_scanner scanner{}; uint8_t *dst; }; -really_inline void json_minifier::next(simd::simd8x64 in, json_block block) { +really_inline void json_minifier::next(const simd::simd8x64& in, json_block block) { uint64_t mask = block.whitespace(); in.compress(mask, dst); dst += 64 - count_ones(mask); diff --git a/src/generic/stage1/json_scanner.h b/src/generic/stage1/json_scanner.h index 8f181ae92..0834c4bd7 100644 --- a/src/generic/stage1/json_scanner.h +++ b/src/generic/stage1/json_scanner.h @@ -49,7 +49,7 @@ private: class json_scanner { public: json_scanner() {} - really_inline json_block next(const simd::simd8x64 in); + really_inline json_block next(const simd::simd8x64& in); really_inline error_code finish(bool streaming); private: @@ -86,7 +86,7 @@ really_inline uint64_t follows(const uint64_t match, const uint64_t filler, uint return result; } -really_inline json_block json_scanner::next(const simd::simd8x64 in) { +really_inline json_block json_scanner::next(const simd::simd8x64& in) { json_string_block strings = string_scanner.next(in); json_character_block characters = json_character_block::classify(in); uint64_t follows_scalar = follows(characters.scalar(), prev_scalar); diff --git a/src/generic/stage1/json_string_scanner.h b/src/generic/stage1/json_string_scanner.h index a9f90aebd..f37855c11 100644 --- a/src/generic/stage1/json_string_scanner.h +++ b/src/generic/stage1/json_string_scanner.h @@ -33,7 +33,7 @@ struct json_string_block { // Scans blocks for string characters, storing the state necessary to do so class json_string_scanner { public: - really_inline json_string_block next(const simd::simd8x64 in); + really_inline json_string_block next(const simd::simd8x64& in); really_inline error_code finish(bool streaming); private: @@ -99,7 +99,7 @@ really_inline uint64_t json_string_scanner::find_escaped_branchless(uint64_t bac // // Backslash sequences outside of quotes will be detected in stage 2. // -really_inline json_string_block json_string_scanner::next(const simd::simd8x64 in) { +really_inline json_string_block json_string_scanner::next(const simd::simd8x64& in) { const uint64_t backslash = in.eq('\\'); const uint64_t escaped = find_escaped(backslash); const uint64_t quote = in.eq('"') & ~escaped; diff --git a/src/generic/stage1/json_structural_indexer.h b/src/generic/stage1/json_structural_indexer.h index 6f80123e7..cf6ec2617 100644 --- a/src/generic/stage1/json_structural_indexer.h +++ b/src/generic/stage1/json_structural_indexer.h @@ -71,7 +71,7 @@ private: really_inline json_structural_indexer(uint32_t *structural_indexes); template really_inline void step(const uint8_t *block, buf_block_reader &reader) noexcept; - really_inline void next(simd::simd8x64 in, json_block block, size_t idx); + really_inline void next(const simd::simd8x64& in, json_block block, size_t idx); really_inline error_code finish(dom_parser_implementation &parser, size_t idx, size_t len, bool partial); json_scanner scanner{}; @@ -140,7 +140,7 @@ really_inline void json_structural_indexer::step<64>(const uint8_t *block, buf_b reader.advance(); } -really_inline void json_structural_indexer::next(simd::simd8x64 in, json_block block, size_t idx) { +really_inline void json_structural_indexer::next(const simd::simd8x64& in, json_block block, size_t idx) { uint64_t unescaped = in.lteq(0x1F); checker.check_next_input(in); indexer.write(uint32_t(idx-64), prev_structurals); // Output *last* iteration's structurals to the parser diff --git a/src/generic/stage1/utf8_fastvalidate_algorithm.h b/src/generic/stage1/utf8_fastvalidate_algorithm.h index 33e969733..ea6a88ce4 100644 --- a/src/generic/stage1/utf8_fastvalidate_algorithm.h +++ b/src/generic/stage1/utf8_fastvalidate_algorithm.h @@ -30,12 +30,12 @@ struct utf8_checker { processed_utf_bytes previous; // all byte values must be no larger than 0xF4 - really_inline void check_smaller_than_0xF4(simd8 current_bytes) { + really_inline void check_smaller_than_0xF4(const simd8 current_bytes) { // unsigned, saturates to 0 below max this->has_error |= current_bytes.saturating_sub(0xF4u); } - really_inline simd8 continuation_lengths(simd8 high_nibbles) { + really_inline simd8 continuation_lengths(const simd8 high_nibbles) { return high_nibbles.lookup_16( 1, 1, 1, 1, 1, 1, 1, 1, // 0xxx (ASCII) 0, 0, 0, 0, // 10xx (continuation) @@ -44,7 +44,7 @@ struct utf8_checker { 4); // 1111, next should be 0 (not checked here) } - really_inline simd8 carry_continuations(simd8 initial_lengths) { + really_inline simd8 carry_continuations(const simd8& initial_lengths) { simd8 prev_carried_continuations = initial_lengths.prev(this->previous.carried_continuations); simd8 right1 = simd8(simd8(prev_carried_continuations).saturating_sub(1)); simd8 sum = initial_lengths + right1; @@ -54,7 +54,7 @@ struct utf8_checker { return sum + right2; } - really_inline void check_continuations(simd8 initial_lengths, simd8 carries) { + really_inline void check_continuations(const simd8& initial_lengths, const simd8& carries) { // overlap || underlap // carry > length && length > 0 || !(carry > length) && !(length > 0) // (carries > length) == (lengths > 0) @@ -76,8 +76,8 @@ struct utf8_checker { // when 0xED is found, next byte must be no larger than 0x9F // when 0xF4 is found, next byte must be no larger than 0x8F // next byte must be continuation, ie sign bit is set, so signed < is ok - really_inline void check_first_continuation_max(simd8 current_bytes, - simd8 off1_current_bytes) { + really_inline void check_first_continuation_max(const simd8 current_bytes, + const simd8 off1_current_bytes) { simd8 prev_ED = off1_current_bytes == 0xEDu; simd8 prev_F4 = off1_current_bytes == 0xF4u; // Check if ED is followed by A0 or greater @@ -94,9 +94,9 @@ struct utf8_checker { // E => < E1 && < A0 // F => < F1 && < 90 // else false && false - really_inline void check_overlong(simd8 current_bytes, - simd8 off1_current_bytes, - simd8 high_nibbles) { + really_inline void check_overlong(const simd8 current_bytes, + const simd8 off1_current_bytes, + const simd8& high_nibbles) { simd8 off1_high_nibbles = high_nibbles.prev(this->previous.high_nibbles); // Two-byte characters must start with at least C2 @@ -132,7 +132,7 @@ struct utf8_checker { // check whether the current bytes are valid UTF-8 // at the end of the function, previous gets updated - really_inline void check_utf8_bytes(simd8 current_bytes) { + really_inline void check_utf8_bytes(const simd8 current_bytes) { struct processed_utf_bytes pb {}; this->count_nibbles(current_bytes, &pb); @@ -151,7 +151,7 @@ struct utf8_checker { this->previous = pb; } - really_inline void check_next_input(simd8 in) { + really_inline void check_next_input(Dconst simd8 in) { if (likely(!in.any_bits_set_anywhere(0x80u))) { this->check_carried_continuations(); } else { @@ -159,8 +159,8 @@ struct utf8_checker { } } - really_inline void check_next_input(simd8x64 in) { - simd8 bits = in.reduce([&](auto a, auto b) { return a | b; }); + really_inline void check_next_input(const simd8x64& in) { + simd8 bits = in.reduce_or(); if (likely(!bits.any_bits_set_anywhere(0x80u))) { // it is ascii, we just check carried continuations. this->check_carried_continuations(); diff --git a/src/generic/stage1/utf8_lookup2_algorithm.h b/src/generic/stage1/utf8_lookup2_algorithm.h index 2acb20669..a5d7ca9a0 100644 --- a/src/generic/stage1/utf8_lookup2_algorithm.h +++ b/src/generic/stage1/utf8_lookup2_algorithm.h @@ -140,7 +140,7 @@ namespace utf8_validation { return byte_1_high & byte_1_low & byte_2_high; } - really_inline simd8 check_multibyte_lengths(simd8 input, simd8 prev_input, simd8 prev1) { + really_inline simd8 check_multibyte_lengths(const simd8 input, const simd8 prev_input, const simd8 prev1) { simd8 prev2 = input.prev<2>(prev_input); simd8 prev3 = input.prev<3>(prev_input); @@ -154,7 +154,7 @@ namespace utf8_validation { // Return nonzero if there are incomplete multibyte characters at the end of the block: // e.g. if there is a 4-byte character, but it's 3 bytes from the end. // - really_inline simd8 is_incomplete(simd8 input) { + really_inline simd8 is_incomplete(const simd8 input) { // If the previous input's last 3 bytes match this, they're too short (they ended at EOF): // ... 1111____ 111_____ 11______ static const uint8_t max_array[32] = { @@ -193,7 +193,7 @@ namespace utf8_validation { this->error |= this->prev_incomplete; } - really_inline void check_next_input(simd8x64 input) { + really_inline void check_next_input(const simd8x64& input) { if (likely(is_ascii(input))) { // If the previous block had incomplete UTF-8 characters at the end, an ASCII block can't // possibly finish them. diff --git a/src/generic/stage1/utf8_lookup3_algorithm.h b/src/generic/stage1/utf8_lookup3_algorithm.h index eaa6c64cb..e49cffccf 100644 --- a/src/generic/stage1/utf8_lookup3_algorithm.h +++ b/src/generic/stage1/utf8_lookup3_algorithm.h @@ -151,7 +151,7 @@ namespace utf8_validation { return byte_1_high & byte_1_low & byte_2_high; } - really_inline simd8 check_multibyte_lengths(simd8 input, simd8 prev_input, + really_inline simd8 check_multibyte_lengths(const simd8 input, const simd8 prev_input, simd8 prev1) { simd8 prev2 = input.prev<2>(prev_input); simd8 prev3 = input.prev<3>(prev_input); @@ -166,7 +166,7 @@ namespace utf8_validation { // Return nonzero if there are incomplete multibyte characters at the end of the block: // e.g. if there is a 4-byte character, but it's 3 bytes from the end. // - really_inline simd8 is_incomplete(simd8 input) { + really_inline simd8 is_incomplete(const simd8 input) { // If the previous input's last 3 bytes match this, they're too short (they ended at EOF): // ... 1111____ 111_____ 11______ static const uint8_t max_array[32] = { @@ -205,16 +205,24 @@ namespace utf8_validation { this->error |= this->prev_incomplete; } - really_inline void check_next_input(simd8x64 input) { - if (likely(is_ascii(input))) { + really_inline void check_next_input(const simd8x64& input) { + if(likely(is_ascii(input))) { // If the previous block had incomplete UTF-8 characters at the end, an ASCII block can't // possibly finish them. this->error |= this->prev_incomplete; } else { - this->check_utf8_bytes(input.chunks[0], this->prev_input_block); - for (int i=1; i::NUM_CHUNKS; i++) { - this->check_utf8_bytes(input.chunks[i], input.chunks[i-1]); - } + // you might think that a for-loop would work, but under Visual Studio, it is not good enough. + static_assert((simd8x64::NUM_CHUNKS == 2) || (simd8x64::NUM_CHUNKS == 4), + "We support either two or four chunks per 64-byte block."); + if(simd8x64::NUM_CHUNKS == 2) { + this->check_utf8_bytes(input.chunks[0], this->prev_input_block); + this->check_utf8_bytes(input.chunks[1], input.chunks[0]); + } else if(simd8x64::NUM_CHUNKS == 4) { + this->check_utf8_bytes(input.chunks[0], this->prev_input_block); + this->check_utf8_bytes(input.chunks[1], input.chunks[0]); + this->check_utf8_bytes(input.chunks[2], input.chunks[1]); + this->check_utf8_bytes(input.chunks[3], input.chunks[2]); + } this->prev_incomplete = is_incomplete(input.chunks[simd8x64::NUM_CHUNKS-1]); this->prev_input_block = input.chunks[simd8x64::NUM_CHUNKS-1]; } diff --git a/src/generic/stage1/utf8_lookup4_algorithm.h b/src/generic/stage1/utf8_lookup4_algorithm.h index f353a3822..745735100 100644 --- a/src/generic/stage1/utf8_lookup4_algorithm.h +++ b/src/generic/stage1/utf8_lookup4_algorithm.h @@ -4,7 +4,7 @@ namespace utf8_validation { using namespace simd; - really_inline simd8 check_special_cases(const simd8& input, const simd8& prev1) { + really_inline simd8 check_special_cases(const simd8 input, const simd8 prev1) { // Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII) // Bit 1 = Too Long (ASCII followed by continuation) // Bit 2 = Overlong 3-byte @@ -94,8 +94,8 @@ using namespace simd; ); return (byte_1_high & byte_1_low & byte_2_high); } - really_inline simd8 check_multibyte_lengths(const simd8& input, - const simd8& prev_input, const simd8& sc) { + really_inline simd8 check_multibyte_lengths(const simd8 input, + const simd8 prev_input, const simd8 sc) { simd8 prev2 = input.prev<2>(prev_input); simd8 prev3 = input.prev<3>(prev_input); simd8 must23 = simd8(must_be_2_3_continuation(prev2, prev3)); @@ -107,7 +107,7 @@ using namespace simd; // Return nonzero if there are incomplete multibyte characters at the end of the block: // e.g. if there is a 4-byte character, but it's 3 bytes from the end. // - really_inline simd8 is_incomplete(const simd8& input) { + really_inline simd8 is_incomplete(const simd8 input) { // If the previous input's last 3 bytes match this, they're too short (they ended at EOF): // ... 1111____ 111_____ 11______ static const uint8_t max_array[32] = { @@ -131,7 +131,7 @@ using namespace simd; // // Check whether the current bytes are valid UTF-8. // - really_inline void check_utf8_bytes(const simd8& input, const simd8& prev_input) { + really_inline void check_utf8_bytes(const simd8 input, const simd8 prev_input) { // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ lead bytes // (2, 3, 4-byte leads become large positive numbers instead of small negative numbers) simd8 prev1 = input.prev<1>(prev_input); @@ -147,7 +147,9 @@ using namespace simd; } really_inline void check_next_input(const simd8x64& input) { - if (unlikely(!is_ascii(input))) { + if(likely(is_ascii(input))) { + this->error |= this->prev_incomplete; + } else { // you might think that a for-loop would work, but under Visual Studio, it is not good enough. static_assert((simd8x64::NUM_CHUNKS == 2) || (simd8x64::NUM_CHUNKS == 4), "We support either two or four chunks per 64-byte block."); @@ -162,10 +164,7 @@ using namespace simd; } this->prev_incomplete = is_incomplete(input.chunks[simd8x64::NUM_CHUNKS-1]); this->prev_input_block = input.chunks[simd8x64::NUM_CHUNKS-1]; - } else { - // If the previous block had incomplete UTF-8 characters at the end, an ASCII block can't - // possibly finish them. - this->error |= this->prev_incomplete; + } } diff --git a/src/generic/stage1/utf8_lookup_algorithm.h b/src/generic/stage1/utf8_lookup_algorithm.h index c72e31026..ad2b97a56 100644 --- a/src/generic/stage1/utf8_lookup_algorithm.h +++ b/src/generic/stage1/utf8_lookup_algorithm.h @@ -276,8 +276,8 @@ struct utf8_checker { this->error |= this->prev_incomplete; } - really_inline void check_next_input(simd8x64 input) { - simd8 bits = input.reduce([&](auto a,auto b) { return a|b; }); + really_inline void check_next_input(const simd8x64& input) { + simd8 bits = input.reduce_or(); if (likely(!bits.any_bits_set_anywhere(0b10000000u))) { // If the previous block had incomplete UTF-8 characters at the end, an ASCII block can't // possibly finish them. diff --git a/src/generic/stage1/utf8_range_algorithm.h b/src/generic/stage1/utf8_range_algorithm.h index 4bf3bf26f..4caba6dc2 100644 --- a/src/generic/stage1/utf8_range_algorithm.h +++ b/src/generic/stage1/utf8_range_algorithm.h @@ -40,7 +40,7 @@ struct utf8_checker { // check whether the current bytes are valid UTF-8 // at the end of the function, previous gets updated - really_inline void check_utf8_bytes(simd8 current_bytes) { + really_inline void check_utf8_bytes(const simd8 current_bytes) { /* high_nibbles = input >> 4 */ const simd8 high_nibbles = current_bytes.shr<4>(); @@ -153,7 +153,7 @@ struct utf8_checker { this->previous.first_len = first_len; } - really_inline void check_next_input(simd8 in) { + really_inline void check_next_input(const simd8 in) { if (likely(!in.any_bits_set_anywhere(0x80u))) { this->check_carried_continuations(); } else { @@ -161,8 +161,8 @@ struct utf8_checker { } } - really_inline void check_next_input(simd8x64 in) { - simd8 bits = in.reduce([&](auto a, auto b) { return a | b; }); + really_inline void check_next_input(const simd8x64& in) { + simd8 bits = in.reduce_or(); if (likely(!bits.any_bits_set_anywhere(0x80u))) { // it is ascii, we just check carried continuations. this->check_carried_continuations(); diff --git a/src/generic/stage1/utf8_zwegner_algorithm.h b/src/generic/stage1/utf8_zwegner_algorithm.h index e59595d61..eb9bd2980 100644 --- a/src/generic/stage1/utf8_zwegner_algorithm.h +++ b/src/generic/stage1/utf8_zwegner_algorithm.h @@ -335,7 +335,7 @@ struct utf8_checker { this->check_special_cases(bytes); } - really_inline void check_next_input(simd8 bytes) { + really_inline void check_next_input(const simd8 bytes) { vmask_t bit_7 = bytes.get_bit<7>(); if (unlikely(bit_7)) { // TODO (@jkeiser): To work with simdjson's caller model, I moved the calculation of @@ -348,7 +348,7 @@ struct utf8_checker { } } - really_inline void check_next_input(simd8x64 in) { + really_inline void check_next_input(const simd8x64& in) { for (int i=0; i::NUM_CHUNKS; i++) { this->check_next_input(in.chunks[i]); } diff --git a/src/haswell/dom_parser_implementation.cpp b/src/haswell/dom_parser_implementation.cpp index 9d3ca8b6e..2a1f18916 100644 --- a/src/haswell/dom_parser_implementation.cpp +++ b/src/haswell/dom_parser_implementation.cpp @@ -16,7 +16,7 @@ namespace haswell { using namespace simd; struct json_character_block { - static really_inline json_character_block classify(const simd::simd8x64 in); + static really_inline json_character_block classify(const simd::simd8x64& in); really_inline uint64_t whitespace() const { return _whitespace; } really_inline uint64_t op() const { return _op; } @@ -26,7 +26,7 @@ struct json_character_block { uint64_t _op; }; -really_inline json_character_block json_character_block::classify(const simd::simd8x64 in) { +really_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { // These lookups rely on the fact that anything < 127 will match the lower 4 bits, which is why // we can't use the generic lookup_16. auto whitespace_table = simd8::repeat_16(' ', 100, 100, 100, 17, 100, 113, 2, 100, '\t', '\n', 112, 100, '\r', 100, 100); @@ -49,12 +49,11 @@ really_inline json_character_block json_character_block::classify(const simd::si return { whitespace, op }; } -really_inline bool is_ascii(simd8x64 input) { - simd8 bits = (input.chunks[0] | input.chunks[1]); - return !bits.any_bits_set_anywhere(0b10000000u); +really_inline bool is_ascii(const simd8x64& input) { + return input.reduce_or().is_ascii(); } -really_inline simd8 must_be_continuation(simd8 prev1, simd8 prev2, simd8 prev3) { +really_inline simd8 must_be_continuation(const simd8 prev1, const simd8 prev2, const simd8 prev3) { simd8 is_second_byte = prev1.saturating_sub(0b11000000u-1); // Only 11______ will be > 0 simd8 is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0 simd8 is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0 @@ -62,7 +61,7 @@ really_inline simd8 must_be_continuation(simd8 prev1, simd8(is_second_byte | is_third_byte | is_fourth_byte) > int8_t(0); } -really_inline simd8 must_be_2_3_continuation(simd8 prev2, simd8 prev3) { +really_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { simd8 is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0 simd8 is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0 // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine. diff --git a/src/haswell/simd.h b/src/haswell/simd.h index 140e01d16..c95f01dfe 100644 --- a/src/haswell/simd.h +++ b/src/haswell/simd.h @@ -275,6 +275,7 @@ namespace simd { really_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } really_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } really_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } + really_inline bool is_ascii() const { return _mm256_movemask_epi8(*this) == 0; } really_inline bool bits_not_set_anywhere() const { return _mm256_testz_si256(*this, *this); } really_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } really_inline bool bits_not_set_anywhere(simd8 bits) const { return _mm256_testz_si256(*this, bits); } @@ -291,19 +292,17 @@ namespace simd { template struct simd8x64 { - static const int NUM_CHUNKS = 64 / sizeof(simd8); + static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); + static_assert(NUM_CHUNKS == 2, "Haswell kernel should use two registers per 64-byte block."); const simd8 chunks[NUM_CHUNKS]; - really_inline simd8x64() : chunks{simd8(), simd8()} {} + simd8x64(const simd8x64& o) = delete; // no copy allowed + simd8x64& operator=(const simd8 other) = delete; // no assignment allowed + simd8x64() = delete; // no default constructor allowed + really_inline simd8x64(const simd8 chunk0, const simd8 chunk1) : chunks{chunk0, chunk1} {} really_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr), simd8::load(ptr+32)} {} - template - static really_inline void each_index(F const& each) { - each(0); - each(1); - } - really_inline void compress(uint64_t mask, T * output) const { uint32_t mask1 = uint32_t(mask); uint32_t mask2 = uint32_t(mask >> 32); @@ -322,6 +321,10 @@ namespace simd { return r_lo | (r_hi << 32); } + really_inline simd8 reduce_or() const { + return this->chunks[0] | this->chunks[1]; + } + really_inline simd8x64 bit_or(const T m) const { const simd8 mask = simd8::splat(m); return simd8x64( diff --git a/src/westmere/dom_parser_implementation.cpp b/src/westmere/dom_parser_implementation.cpp index 59d486b5f..c665e76c0 100644 --- a/src/westmere/dom_parser_implementation.cpp +++ b/src/westmere/dom_parser_implementation.cpp @@ -17,7 +17,7 @@ namespace westmere { using namespace simd; struct json_character_block { - static really_inline json_character_block classify(const simd::simd8x64 in); + static really_inline json_character_block classify(const simd::simd8x64& in); really_inline uint64_t whitespace() const { return _whitespace; } really_inline uint64_t op() const { return _op; } @@ -27,7 +27,7 @@ struct json_character_block { uint64_t _op; }; -really_inline json_character_block json_character_block::classify(const simd::simd8x64 in) { +really_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { // These lookups rely on the fact that anything < 127 will match the lower 4 bits, which is why // we can't use the generic lookup_16. auto whitespace_table = simd8::repeat_16(' ', 100, 100, 100, 17, 100, 113, 2, 100, '\t', '\n', 112, 100, '\r', 100, 100); @@ -55,12 +55,11 @@ really_inline json_character_block json_character_block::classify(const simd::si return { whitespace, op }; } -really_inline bool is_ascii(simd8x64 input) { - simd8 bits = (input.chunks[0] | input.chunks[1]) | (input.chunks[2] | input.chunks[3]); - return !bits.any_bits_set_anywhere(0b10000000u); +really_inline bool is_ascii(const simd8x64& input) { + return input.reduce_or().is_ascii(); } -really_inline simd8 must_be_continuation(simd8 prev1, simd8 prev2, simd8 prev3) { +really_inline simd8 must_be_continuation(const simd8 prev1, const simd8 prev2, const simd8 prev3) { simd8 is_second_byte = prev1.saturating_sub(0b11000000u-1); // Only 11______ will be > 0 simd8 is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0 simd8 is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0 @@ -68,7 +67,7 @@ really_inline simd8 must_be_continuation(simd8 prev1, simd8(is_second_byte | is_third_byte | is_fourth_byte) > int8_t(0); } -really_inline simd8 must_be_2_3_continuation(simd8 prev2, simd8 prev3) { +really_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { simd8 is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0 simd8 is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0 // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine. diff --git a/src/westmere/simd.h b/src/westmere/simd.h index 705d6b2c5..efbe508b0 100644 --- a/src/westmere/simd.h +++ b/src/westmere/simd.h @@ -247,6 +247,7 @@ namespace simd { really_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } really_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } really_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } + really_inline bool is_ascii() const { return _mm_movemask_epi8(*this) == 0; } really_inline bool bits_not_set_anywhere() const { return _mm_testz_si128(*this, *this); } really_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } really_inline bool bits_not_set_anywhere(simd8 bits) const { return _mm_testz_si128(*this, bits); } @@ -263,10 +264,14 @@ namespace simd { template struct simd8x64 { - static const int NUM_CHUNKS = 64 / sizeof(simd8); + static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); + static_assert(NUM_CHUNKS == 4, "Westmere kernel should use four registers per 64-byte block."); const simd8 chunks[NUM_CHUNKS]; - really_inline simd8x64() : chunks{simd8(), simd8(), simd8(), simd8()} {} + simd8x64(const simd8x64& o) = delete; // no copy allowed + simd8x64& operator=(const simd8 other) = delete; // no assignment allowed + simd8x64() = delete; // no default constructor allowed + really_inline simd8x64(const simd8 chunk0, const simd8 chunk1, const simd8 chunk2, const simd8 chunk3) : chunks{chunk0, chunk1, chunk2, chunk3} {} really_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr), simd8::load(ptr+16), simd8::load(ptr+32), simd8::load(ptr+48)} {} @@ -277,6 +282,10 @@ namespace simd { this->chunks[3].store(ptr+sizeof(simd8)*3); } + really_inline simd8 reduce_or() const { + return (this->chunks[0] | this->chunks[1]) | (this->chunks[2] | this->chunks[3]); + } + really_inline void compress(uint64_t mask, T * output) const { this->chunks[0].compress(uint16_t(mask), output); this->chunks[1].compress(uint16_t(mask >> 16), output + 16 - count_ones(mask & 0xFFFF)); @@ -284,14 +293,6 @@ namespace simd { this->chunks[3].compress(uint16_t(mask >> 48), output + 48 - count_ones(mask & 0xFFFFFFFFFFFF)); } - template - static really_inline void each_index(F const& each) { - each(0); - each(1); - each(2); - each(3); - } - really_inline uint64_t to_bitmask() const { uint64_t r0 = uint32_t(this->chunks[0].to_bitmask()); uint64_t r1 = this->chunks[1].to_bitmask();