From b015f8959bfcacfefb4681e4f862d9c7b4e54553 Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Tue, 10 May 2022 11:40:00 -0400 Subject: [PATCH] Adding some optimizations for AVX-512 and making it optional (this could be reversed). --- cmake/developer-options.cmake | 12 +++++ doc/performance.md | 3 +- include/simdjson/implementations.h | 2 +- src/cascadelake/dom_parser_implementation.cpp | 44 +++++++++++++++++++ src/generic/stage1/json_structural_indexer.h | 8 ++++ 5 files changed, 67 insertions(+), 2 deletions(-) diff --git a/cmake/developer-options.cmake b/cmake/developer-options.cmake index c79aae3b3..9b9e99dc6 100644 --- a/cmake/developer-options.cmake +++ b/cmake/developer-options.cmake @@ -175,6 +175,18 @@ if(CMAKE_C_COMPILER_ID MATCHES "Intel") set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -static-intel") endif() + + +option( + SIMDJSON_AVX512_ALLOWED + "Enable AVX-512 instructions (only affects processors with AVX-512 support)." + OFF +) +if(SIMDJSON_AVX512_ALLOWED) + add_compile_definitions(SIMDJSON_AVX512_ALLOWED=1) + message(STATUS "AVX-512 instructions allowed if the CPU supports it.") +endif() + include(CheckSymbolExists) check_symbol_exists(fork unistd.h HAVE_POSIX_FORK) check_symbol_exists(wait sys/wait.h HAVE_POSIX_WAIT) diff --git a/doc/performance.md b/doc/performance.md index d3eaa9db1..81e46ad45 100644 --- a/doc/performance.md +++ b/doc/performance.md @@ -160,6 +160,7 @@ You should not expect the simdjson library to cause *downclocking* of your recen - [Whenever 512-bit AVX-512 instructions are used](https://lemire.me/blog/2018/09/07/avx-512-when-and-how-to-use-these-new-instructions/). - Whenever heavy 256-bit or wider instructions are used. Heavy instructions are those involving floating point operations or integer multiplications (since these execute on the floating point unit). -The simdjson library does not make use of heavy 256-bit instructions. We do use vectorized multiplications, but only using 128-bit registers. Thus there should be no downclocking due to simdjson on recent processors, except when AVX-512 is detected. +The simdjson library does not make use of heavy 256-bit instructions. We do use vectorized multiplications, but only using 128-bit registers. Thus there should be no downclocking due to simdjson on recent processors, except when AVX-512 is allowed and +detected. By default, AVX-512 is disabled: you need to build simdjson with the CMake option `SIMDJSON_AVX512_ALLOWED` set to `ON` (e.g., `cmake -D SIMDJSON_AVX512_ALLOWED=ON -B build && cmake --build build`). You may still be worried about which SIMD instruction set is used by simdjson. Thankfully, [you can always determine and change which architecture-specific implementation is used](implementation-selection.md) by simdjson. Thus even if your CPU supports AVX2, you do not need to use AVX2. You are in control. diff --git a/include/simdjson/implementations.h b/include/simdjson/implementations.h index 4c303d1e6..0ea952734 100644 --- a/include/simdjson/implementations.h +++ b/include/simdjson/implementations.h @@ -17,7 +17,7 @@ // Default Cascadelake to on if this is x86-64. Even if we're not compiled for it, it could be selected // at runtime. #ifndef SIMDJSON_IMPLEMENTATION_CASCADELAKE -#define SIMDJSON_IMPLEMENTATION_CASCADELAKE SIMDJSON_IS_X86_64 +#define SIMDJSON_IMPLEMENTATION_CASCADELAKE ((SIMDJSON_IS_X86_64) && (SIMDJSON_AVX512_ALLOWED)) #endif diff --git a/src/cascadelake/dom_parser_implementation.cpp b/src/cascadelake/dom_parser_implementation.cpp index 8b83311a2..bc4f1e55c 100644 --- a/src/cascadelake/dom_parser_implementation.cpp +++ b/src/cascadelake/dom_parser_implementation.cpp @@ -103,7 +103,51 @@ simdjson_really_inline simd8 must_be_2_3_continuation(const simd8 } // namespace simdjson #include "generic/stage1/utf8_lookup4_algorithm.h" +// defining SIMDJSON_CUSTOM_BIT_INDEXER allows us to provide our own bit_indexer::write +#define SIMDJSON_CUSTOM_BIT_INDEXER #include "generic/stage1/json_structural_indexer.h" +// We must not forget to undefine it now: +#undef SIMDJSON_CUSTOM_BIT_INDEXER + +/** + * We provide a custom version of bit_indexer::write using + * naked intrinsics. + * TODO: make this code more elegant. + */ +namespace simdjson { namespace SIMDJSON_IMPLEMENTATION { namespace { namespace stage1 { +simdjson_really_inline void bit_indexer::write(uint32_t idx, uint64_t bits) { + // In some instances, the next branch is expensive because it is mispredicted. + // Unfortunately, in other cases, + // it helps tremendously. + if (bits == 0) { return; } + __m512i start_index = _mm512_set1_epi32(idx); + __m512i base_index = _mm512_setr_epi32(0,1,2,3,4,5, + 6,7,8,9,10,11,12,13,14,15); + base_index = _mm512_add_epi32(base_index, start_index); + uint16_t mask; + mask = bits & 0xFFFF; + _mm512_mask_compressstoreu_epi32(this->tail, + mask, base_index); + this->tail += count_ones(mask); + const __m512i constant16 = _mm512_set1_epi32(16); + base_index = _mm512_add_epi32(base_index, constant16); + mask = (bits>>16) & 0xFFFF; + _mm512_mask_compressstoreu_epi32(this->tail, + mask, base_index); + this->tail += count_ones(mask); + base_index = _mm512_add_epi32(base_index, constant16); + mask = (bits>>32) & 0xFFFF; + _mm512_mask_compressstoreu_epi32(this->tail, + mask, base_index); + this->tail += count_ones(mask); + base_index = _mm512_add_epi32(base_index, constant16); + mask = bits>>48; + _mm512_mask_compressstoreu_epi32(this->tail, + mask, base_index); + this->tail += count_ones(mask); +} +}}}} + #include "generic/stage1/utf8_validator.h" // diff --git a/src/generic/stage1/json_structural_indexer.h b/src/generic/stage1/json_structural_indexer.h index b0c34cb1d..f3db24b97 100644 --- a/src/generic/stage1/json_structural_indexer.h +++ b/src/generic/stage1/json_structural_indexer.h @@ -25,6 +25,12 @@ public: // base_ptr[base] incrementing base as we go // will potentially store extra values beyond end of valid bits, so base_ptr // needs to be large enough to handle this + // + // If the kernel sets SIMDJSON_CUSTOM_BIT_INDEXER, then it will provide its own + // version of the code. +#ifdef SIMDJSON_CUSTOM_BIT_INDEXER + simdjson_really_inline void write(uint32_t idx, uint64_t bits); +#else simdjson_really_inline void write(uint32_t idx, uint64_t bits) { // In some instances, the next branch is expensive because it is mispredicted. // Unfortunately, in other cases, @@ -117,6 +123,8 @@ public: this->tail += cnt; #endif } +#endif // SIMDJSON_CUSTOM_BIT_INDEXER + }; class json_structural_indexer {