Adding some optimizations for AVX-512 and making it optional (this could be reversed).

cmake/developer-options.cmake

@@ -175,6 +175,18 @@ if(CMAKE_C_COMPILER_ID MATCHES "Intel")
   set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -static-intel")
 endif()
 
+
+
+option(
+    SIMDJSON_AVX512_ALLOWED
+    "Enable AVX-512 instructions (only affects processors with AVX-512 support)."
+    OFF
+)
+if(SIMDJSON_AVX512_ALLOWED)
+  add_compile_definitions(SIMDJSON_AVX512_ALLOWED=1)
+  message(STATUS "AVX-512 instructions allowed if the CPU supports it.")
+endif()
+
 include(CheckSymbolExists)
 check_symbol_exists(fork unistd.h HAVE_POSIX_FORK)
 check_symbol_exists(wait sys/wait.h HAVE_POSIX_WAIT)

doc/performance.md

@@ -160,6 +160,7 @@ You should not expect the simdjson library to cause *downclocking* of your recen
 - [Whenever 512-bit AVX-512 instructions are used](https://lemire.me/blog/2018/09/07/avx-512-when-and-how-to-use-these-new-instructions/).
 - Whenever heavy 256-bit or wider instructions are used. Heavy instructions are those involving floating point operations or integer multiplications (since these execute on the floating point unit).
 
-The simdjson library does not make use of heavy 256-bit instructions. We do use vectorized multiplications, but only using 128-bit registers. Thus there should be no downclocking due to simdjson on recent processors, except when AVX-512 is detected.
+The simdjson library does not make use of heavy 256-bit instructions. We do use vectorized multiplications, but only using 128-bit registers. Thus there should be no downclocking due to simdjson on recent processors, except when AVX-512 is allowed and
+detected. By default, AVX-512 is disabled: you need to build simdjson with the CMake option `SIMDJSON_AVX512_ALLOWED` set to `ON` (e.g., `cmake -D SIMDJSON_AVX512_ALLOWED=ON -B build && cmake --build build`).
 
 You may still be worried about which SIMD instruction set is used by simdjson.  Thankfully,  [you can always determine and change which architecture-specific implementation is used](implementation-selection.md) by simdjson. Thus even if your CPU supports AVX2, you do not need to use AVX2. You are in control.

include/simdjson/implementations.h

@@ -17,7 +17,7 @@
 // Default Cascadelake to on if this is x86-64. Even if we're not compiled for it, it could be selected
 // at runtime.
 #ifndef SIMDJSON_IMPLEMENTATION_CASCADELAKE
-#define SIMDJSON_IMPLEMENTATION_CASCADELAKE SIMDJSON_IS_X86_64
+#define SIMDJSON_IMPLEMENTATION_CASCADELAKE ((SIMDJSON_IS_X86_64) && (SIMDJSON_AVX512_ALLOWED))
 #endif
 
 

src/cascadelake/dom_parser_implementation.cpp

@@ -103,7 +103,51 @@ simdjson_really_inline simd8<bool> must_be_2_3_continuation(const simd8<uint8_t>
 } // namespace simdjson
 
 #include "generic/stage1/utf8_lookup4_algorithm.h"
+// defining SIMDJSON_CUSTOM_BIT_INDEXER allows us to provide our own bit_indexer::write
+#define SIMDJSON_CUSTOM_BIT_INDEXER
 #include "generic/stage1/json_structural_indexer.h"
+// We must not forget to undefine it now:
+#undef SIMDJSON_CUSTOM_BIT_INDEXER
+
+/**
+ * We provide a custom version of bit_indexer::write using
+ * naked intrinsics.
+ * TODO: make this code more elegant.
+ */
+namespace simdjson { namespace SIMDJSON_IMPLEMENTATION { namespace { namespace stage1 {
+simdjson_really_inline void bit_indexer::write(uint32_t idx, uint64_t bits) {
+    // In some instances, the next branch is expensive because it is mispredicted.
+    // Unfortunately, in other cases,
+    // it helps tremendously.
+    if (bits == 0) { return; }
+    __m512i start_index = _mm512_set1_epi32(idx);
+    __m512i base_index = _mm512_setr_epi32(0,1,2,3,4,5,
+      6,7,8,9,10,11,12,13,14,15);
+    base_index = _mm512_add_epi32(base_index, start_index);
+    uint16_t mask;
+    mask = bits & 0xFFFF;
+    _mm512_mask_compressstoreu_epi32(this->tail,
+      mask, base_index);
+    this->tail += count_ones(mask);
+    const __m512i constant16 = _mm512_set1_epi32(16);
+    base_index = _mm512_add_epi32(base_index, constant16);
+    mask = (bits>>16) & 0xFFFF;
+    _mm512_mask_compressstoreu_epi32(this->tail,
+      mask, base_index);
+    this->tail += count_ones(mask);
+    base_index = _mm512_add_epi32(base_index, constant16);
+    mask = (bits>>32) & 0xFFFF;
+    _mm512_mask_compressstoreu_epi32(this->tail,
+      mask, base_index);
+    this->tail += count_ones(mask);
+    base_index = _mm512_add_epi32(base_index, constant16);
+    mask = bits>>48;
+    _mm512_mask_compressstoreu_epi32(this->tail,
+      mask, base_index);
+    this->tail += count_ones(mask);
+}
+}}}}
+
 #include "generic/stage1/utf8_validator.h"
 
 //

src/generic/stage1/json_structural_indexer.h

@@ -25,6 +25,12 @@ public:
   // base_ptr[base] incrementing base as we go
   // will potentially store extra values beyond end of valid bits, so base_ptr
   // needs to be large enough to handle this
+  //
+  // If the kernel sets SIMDJSON_CUSTOM_BIT_INDEXER, then it will provide its own
+  // version of the code.
+#ifdef SIMDJSON_CUSTOM_BIT_INDEXER
+  simdjson_really_inline void write(uint32_t idx, uint64_t bits);
+#else
   simdjson_really_inline void write(uint32_t idx, uint64_t bits) {
     // In some instances, the next branch is expensive because it is mispredicted.
     // Unfortunately, in other cases,
@@ -117,6 +123,8 @@ public:
     this->tail += cnt;
 #endif
   }
+#endif // SIMDJSON_CUSTOM_BIT_INDEXER
+
 };
 
 class json_structural_indexer {