Add ability to handle 8 UTF-8 SIMD blocks at a time

.clangd

@@ -45,3 +45,11 @@ Diagnostics:
     Suppress:
         - pragma_attribute_no_pop_eof
         - pragma_attribute_stack_mismatch
+---
+# clang 18
+If:
+    PathMatch:
+        - dependencies/.cache/json11/json11.cpp
+CompileFlags:
+    Add:
+        - -Wno-unqualified-std-cast-call

include/simdjson/arm64/simd.h

@@ -210,6 +210,7 @@ namespace {
 
     // Bit-specific operations
     simdjson_inline simd8<bool> any_bits_set(simd8<uint8_t> bits) const { return vtstq_u8(*this, bits); }
+    simdjson_inline bool is_ascii() const { return this->max_val() < 0x80u; }
     simdjson_inline bool any_bits_set_anywhere() const { return this->max_val() != 0; }
     simdjson_inline bool any_bits_set_anywhere(simd8<uint8_t> bits) const { return (*this & bits).any_bits_set_anywhere(); }
     template<int N>
@@ -412,6 +413,66 @@ namespace {
     }
   };
 
+#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
+      static const uint8x16_t BITMASK64_BUILDER_MASK = simdjson_make_uint8x16_t(
+        0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80,
+        0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80
+      );
+#else
+      static const uint8x16_t BITMASK64_BUILDER_MASK = {
+        0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80,
+        0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80
+      };
+#endif
+
+  template <int N = 0>
+  struct simd_bitmask64_builder;
+
+  template<>
+  struct simd_bitmask64_builder<4> {
+    const uint64_t mask;
+    operator uint64_t() && { return mask; }
+  }; // struct simd_bitmask64_builder<4>
+
+  template<>
+  struct simd_bitmask64_builder<3> {
+    const uint8x16_t sum01;
+    const simd8<bool> val2;
+    simdjson_inline simd_bitmask64_builder<4> next(simd8<bool> val3) {
+      // Add each of the elements next to each other, successively, to stuff each 8 byte mask into one.
+      uint8x16_t sum23 = vpaddq_u8(val2 & BITMASK64_BUILDER_MASK, val3 & BITMASK64_BUILDER_MASK);
+      uint8x16_t sum0123 = vpaddq_u8(sum01, sum23);
+
+      // This algorithm is actually designed to create a 128-bit mask from 8 16-byte simd masks,
+      // but since we only want 64 bits, we add the mask to itself (creating the final mask twice).
+      uint8x16_t sum01230123 = vpaddq_u8(sum0123, sum0123);
+      return { vgetq_lane_u64(vreinterpretq_u64_u8(sum01230123), 0) };
+    }
+  }; // struct simd_bitmask64_builder<3>
+
+  template<>
+  struct simd_bitmask64_builder<2> {
+    const uint8x16_t sum01;
+    simdjson_inline simd_bitmask64_builder<3> next(simd8<bool> val2) {
+      return { sum01, val2 };
+    }
+  }; // struct simd_bitmask64_builder<2>
+
+  template<>
+  struct simd_bitmask64_builder<1> {
+    const simd8<bool> val0;
+    simdjson_inline simd_bitmask64_builder<2> next(simd8<bool> val1) {
+      return { vpaddq_u8(val0 & BITMASK64_BUILDER_MASK, val1 & BITMASK64_BUILDER_MASK) };
+    }
+  }; // struct simd_bitmask64_builder<1>
+
+  template<>
+  struct simd_bitmask64_builder<0> {
+    simdjson_inline simd_bitmask64_builder<1> next(simd8<bool> val) {
+      return { val };
+    }
+  }; // struct simd_bitmask64_builder<0>
+
   template<typename T>
   struct simd8x64 {
     static constexpr int NUM_CHUNKS = 64 / sizeof(simd8<T>);
@@ -449,23 +510,11 @@ namespace {
     }
 
     simdjson_inline uint64_t to_bitmask() const {
-#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
-      const uint8x16_t bit_mask = simdjson_make_uint8x16_t(
-        0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80,
-        0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80
-      );
-#else
-      const uint8x16_t bit_mask = {
-        0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80,
-        0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80
-      };
-#endif
-      // Add each of the elements next to each other, successively, to stuff each 8 byte mask into one.
-      uint8x16_t sum0 = vpaddq_u8(this->chunks[0] & bit_mask, this->chunks[1] & bit_mask);
-      uint8x16_t sum1 = vpaddq_u8(this->chunks[2] & bit_mask, this->chunks[3] & bit_mask);
-      sum0 = vpaddq_u8(sum0, sum1);
-      sum0 = vpaddq_u8(sum0, sum0);
-      return vgetq_lane_u64(vreinterpretq_u64_u8(sum0), 0);
+      return simd_bitmask64_builder<0>()
+        .next(this->chunks[0])
+        .next(this->chunks[1])
+        .next(this->chunks[2])
+        .next(this->chunks[3]);
     }
 
     simdjson_inline uint64_t eq(const T m) const {

include/simdjson/haswell/simd.h

@@ -295,6 +295,30 @@ namespace simd {
     simdjson_inline int get_bit() const { return _mm256_movemask_epi8(_mm256_slli_epi16(*this, 7-N)); }
   };
 
+  template <int N = 0>
+  struct simd_bitmask64_builder;
+
+  template<>
+  struct simd_bitmask64_builder<2> {
+    const uint64_t bitmask;
+    operator uint64_t() && { return bitmask; }
+  }; // struct simd_bitmask64_builder<2>
+
+  template<>
+  struct simd_bitmask64_builder<1> {
+    const int bitmask;
+    simdjson_inline simd_bitmask64_builder<2> next(simd8<bool> val) {
+      uint64_t r_lo = uint32_t(bitmask);
+      uint64_t r_hi = val.to_bitmask();
+      return { r_lo | (r_hi << 32) };
+    }
+  }; // struct simd_bitmask64_builder<1>
+
+  template<>
+  struct simd_bitmask64_builder<0> {
+    simdjson_inline simd_bitmask64_builder<1> next(simd8<bool> val) { return { val.to_bitmask() }; }
+  }; // struct simd_bitmask64_builder<0>
+
   template<typename T>
   struct simd8x64 {
     static constexpr int NUM_CHUNKS = 64 / sizeof(simd8<T>);
@@ -322,9 +346,9 @@ namespace simd {
     }
 
     simdjson_inline uint64_t to_bitmask() const {
-      uint64_t r_lo = uint32_t(this->chunks[0].to_bitmask());
-      uint64_t r_hi =                       this->chunks[1].to_bitmask();
-      return r_lo | (r_hi << 32);
+      return simd_bitmask64_builder<0>()
+        .next(this->chunks[0])
+        .next(this->chunks[1]);
     }
 
     simdjson_inline simd8<T> reduce_or() const {

include/simdjson/icelake/simd.h

@@ -314,6 +314,20 @@ namespace simd {
     simdjson_inline uint64_t get_bit() const { return _mm512_movepi8_mask(_mm512_slli_epi16(*this, 7-N)); }
   };
 
+  template <int N = 0>
+  struct simd_bitmask64_builder;
+
+  template<>
+  struct simd_bitmask64_builder<1> {
+    const __mmask64 bitmask;
+    operator __mmask64() && { return bitmask; }
+  }; // struct simd_bitmask64_builder<2>
+
+  template<>
+  struct simd_bitmask64_builder<0> {
+    simdjson_inline simd_bitmask64_builder<1> next(simd8<bool> val) { return { _mm512_movepi8_mask(val) }; }
+  }; // struct simd_bitmask64_builder<0>
+
   template<typename T>
   struct simd8x64 {
     static constexpr int NUM_CHUNKS = 64 / sizeof(simd8<T>);

include/simdjson/lasx/simd.h

@@ -298,6 +298,35 @@ namespace simd {
     simdjson_inline simd8<uint8_t> shl() const { return simd8<uint8_t>(__lasx_xvslli_b(*this, N)); }
   };
 
+  template <int N = 0>
+  struct simd_bitmask64_builder;
+
+  template<>
+  struct simd_bitmask64_builder<2> {
+    const unsigned long int mask01;
+    operator uint64_t() { return mask01; }
+  }; // struct simd_bitmask64_builder<2>
+
+  template<>
+  struct simd_bitmask64_builder<1> {
+    const __m256i mask0;
+    simdjson_inline simd_bitmask64_builder<2> next(simd8<bool> val1) {
+      __m256i mask1 = __lasx_xvmskltz_b(val1);
+      __m256i mask_tmp = __lasx_xvpickve_w(mask0, 4);
+      __m256i tmp = __lasx_xvpickve_w(mask1, 4);
+      __m256i mask01 = __lasx_xvinsve0_w(mask0, mask1, 1);
+      __m256i mask01_tmp = __lasx_xvinsve0_w(mask_tmp, tmp, 1);
+      return { __lasx_xvpickve2gr_du(__lasx_xvpackev_h(mask01_tmp, mask01), 0) };
+    }
+  }; // struct simd_bitmask64_builder<1>
+
+  template<>
+  struct simd_bitmask64_builder<0> {
+    simdjson_inline simd_bitmask64_builder<1> next(simd8<bool> val) {
+      return { __lasx_xvmskltz_b(val0) };
+    }
+  }; // struct simd_bitmask64_builder<0>
+
   template<typename T>
   struct simd8x64 {
     static constexpr int NUM_CHUNKS = 64 / sizeof(simd8<T>);
@@ -331,13 +360,9 @@ namespace simd {
     }
 
     simdjson_inline uint64_t to_bitmask() const {
-      __m256i mask0 = __lasx_xvmskltz_b(this->chunks[0]);
-      __m256i mask1 = __lasx_xvmskltz_b(this->chunks[1]);
-      __m256i mask_tmp = __lasx_xvpickve_w(mask0, 4);
-      __m256i tmp = __lasx_xvpickve_w(mask1, 4);
-      mask0 = __lasx_xvinsve0_w(mask0, mask1, 1);
-      mask_tmp = __lasx_xvinsve0_w(mask_tmp, tmp, 1);
-      return __lasx_xvpickve2gr_du(__lasx_xvpackev_h(mask_tmp, mask0), 0);
+      return simd_bitmask64_builder<0>()
+        .next(this->chunks[0])
+        .next(this->chunks[1]);
     }
 
     simdjson_inline simd8<T> reduce_or() const {

include/simdjson/lsx/simd.h

@@ -255,6 +255,50 @@ namespace simd {
     simdjson_inline simd8<uint8_t> shl() const { return simd8<uint8_t>(__lsx_vslli_b(*this, N)); }
   };
 
+  template <int N = 0>
+  struct simd_bitmask64_builder;
+
+  template<>
+  struct simd_bitmask64_builder<4> {
+    const unsigned long int result;
+    operator uint64_t() { return result; }
+  }; // struct simd_bitmask64_builder<4>
+
+  template<>
+  struct simd_bitmask64_builder<3> {
+    const __m128i mask01;
+    const __m128i mask2;
+    simdjson_inline simd_bitmask64_builder<4> next(simd8<bool> val3) {
+      __m128i mask3 = __lsx_vmskltz_b(val3);
+      __m128i mask23 = __lsx_vilvl_h(mask3, mask2);
+      return { __lsx_vpickve2gr_du(__lsx_vilvl_w(mask23, mask01), 0) };
+    }
+  }; // struct simd_bitmask64_builder<3>
+
+  template<>
+  struct simd_bitmask64_builder<2> {
+    const __m128i mask01;
+    simdjson_inline simd_bitmask64_builder<3> next(simd8<bool> val2) {
+      return { mask01, __lsx_vmskltz_b(val2) };
+    }
+  }; // struct simd_bitmask64_builder<2>
+
+  template<>
+  struct simd_bitmask64_builder<1> {
+    const __m128i mask0;
+    simdjson_inline simd_bitmask64_builder<2> next(simd8<bool> val1) {
+      __m128i mask1 = __lsx_vmskltz_b(val1);
+      return { __lsx_vilvl_h(mask1, mask0) };
+    }
+  }; // struct simd_bitmask64_builder<1>
+
+  template<>
+  struct simd_bitmask64_builder<0> {
+    simdjson_inline simd_bitmask64_builder<1> next(simd8<bool> val0) {
+      return { __lsx_vmskltz_b(val0) };
+    }
+  }; // struct simd_bitmask64_builder<0>
+
   template<typename T>
   struct simd8x64 {
     static constexpr int NUM_CHUNKS = 64 / sizeof(simd8<T>);
@@ -303,13 +347,11 @@ namespace simd {
     }
 
     simdjson_inline uint64_t to_bitmask() const {
-      __m128i mask1 = __lsx_vmskltz_b(this->chunks[0]);
-      __m128i mask2 = __lsx_vmskltz_b(this->chunks[1]);
-      __m128i mask3 = __lsx_vmskltz_b(this->chunks[2]);
-      __m128i mask4 = __lsx_vmskltz_b(this->chunks[3]);
-      mask1 = __lsx_vilvl_h(mask2, mask1);
-      mask2 = __lsx_vilvl_h(mask4, mask3);
-      return __lsx_vpickve2gr_du(__lsx_vilvl_w(mask2, mask1), 0);
+      return simd_bitmask64_builder<0>()
+        .next(this->chunks[0])
+        .next(this->chunks[1])
+        .next(this->chunks[2])
+        .next(this->chunks[3]);
     }
 
     simdjson_inline simd8<T> reduce_or() const {

include/simdjson/ppc64/simd.h

@@ -392,6 +392,51 @@ template <> struct simd8<uint8_t> : base8_numeric<uint8_t> {
   }
 };
 
+template <int N = 0>
+struct simd_bitmask64_builder;
+
+template<>
+struct simd_bitmask64_builder<4> {
+  const uint64_t bitmask;
+  simdjson_inline operator uint64_t() && { return bitmask; }
+}; // struct simd_bitmask64_builder<4>
+
+template<>
+struct simd_bitmask64_builder<3> {
+  const int bitmask01;
+  const int bitmask2;
+  simdjson_inline simd_bitmask64_builder<4> next(simd8<bool> val) {
+    return {
+      uint64_t(this->bitmask01) |
+      (uint64_t(this->bitmask2) << 32) |
+      (uint64_t(val.to_bitmask()) << 48)
+    };
+  }
+}; // struct simd_bitmask64_builder<3>
+
+template<>
+struct simd_bitmask64_builder<2> {
+  const int bitmask01;
+  simdjson_inline simd_bitmask64_builder<3> next(simd8<bool> val) {
+    return { bitmask01, val.to_bitmask() };
+  }
+}; // struct simd_bitmask64_builder<2>
+
+template<>
+struct simd_bitmask64_builder<1> {
+  const int bitmask0;
+  simdjson_inline simd_bitmask64_builder<2> next(simd8<bool> val) {
+    return { bitmask0 | (val.to_bitmask() << 16) };
+  }
+}; // struct simd_bitmask64_builder<1>
+
+template<>
+struct simd_bitmask64_builder<0> {
+  simdjson_inline simd_bitmask64_builder<1> next(simd8<bool> val) {
+    return { val.to_bitmask() };
+  }
+}; // struct simd_bitmask64_builder<0>
+
 template <typename T> struct simd8x64 {
   static constexpr int NUM_CHUNKS = 64 / sizeof(simd8<T>);
   static_assert(NUM_CHUNKS == 4,
@@ -434,11 +479,11 @@ template <typename T> struct simd8x64 {
   }
 
   simdjson_inline uint64_t to_bitmask() const {
-    uint64_t r0 = uint32_t(this->chunks[0].to_bitmask());
-    uint64_t r1 = this->chunks[1].to_bitmask();
-    uint64_t r2 = this->chunks[2].to_bitmask();
-    uint64_t r3 = this->chunks[3].to_bitmask();
-    return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48);
+      return simd_bitmask64_builder<0>()
+        .next(this->chunks[0])
+        .next(this->chunks[1])
+        .next(this->chunks[2])
+        .next(this->chunks[3]);
   }
 
   simdjson_inline uint64_t eq(const T m) const {

include/simdjson/westmere/simd.h

@@ -260,6 +260,51 @@ namespace simd {
     simdjson_inline int get_bit() const { return _mm_movemask_epi8(_mm_slli_epi16(*this, 7-N)); }
   };
 
+  template <int N = 0>
+  struct simd_bitmask64_builder;
+
+  template<>
+  struct simd_bitmask64_builder<4> {
+    const uint64_t bitmask;
+    operator uint64_t() && { return bitmask; }
+  }; // struct simd_bitmask64_builder<4>
+
+  template<>
+  struct simd_bitmask64_builder<3> {
+    uint32_t bitmask01;
+    const int bitmask2;
+    simdjson_inline simd_bitmask64_builder<4> next(simd8<bool> val3) {
+      return {
+        uint64_t(this->bitmask01) |
+        (uint64_t(this->bitmask2) << 32) |
+        (uint64_t(val3.to_bitmask()) << 48)
+      };
+    }
+  }; // struct simd_bitmask64_builder<3>
+
+  template<>
+  struct simd_bitmask64_builder<2> {
+    uint32_t bitmask01;
+    simdjson_inline simd_bitmask64_builder<3> next(simd8<bool> val2) {
+      return { bitmask01, val2.to_bitmask() };
+    }
+  }; // struct simd_bitmask64_builder<2>
+
+  template<>
+  struct simd_bitmask64_builder<1> {
+    const int bitmask0;
+    simdjson_inline simd_bitmask64_builder<2> next(simd8<bool> val1) {
+      return { uint32_t(bitmask0) | (uint32_t(val1.to_bitmask()) << 16) };
+    }
+  }; // struct simd_bitmask64_builder<1>
+
+  template<>
+  struct simd_bitmask64_builder<0> {
+    simdjson_inline simd_bitmask64_builder<1> next(simd8<bool> val0) {
+      return { val0.to_bitmask() };
+    }
+  }; // struct simd_bitmask64_builder<0>
+
   template<typename T>
   struct simd8x64 {
     static constexpr int NUM_CHUNKS = 64 / sizeof(simd8<T>);
@@ -293,11 +338,11 @@ namespace simd {
     }
 
     simdjson_inline uint64_t to_bitmask() const {
-      uint64_t r0 = uint32_t(this->chunks[0].to_bitmask() );
-      uint64_t r1 =          this->chunks[1].to_bitmask() ;
-      uint64_t r2 =          this->chunks[2].to_bitmask() ;
-      uint64_t r3 =          this->chunks[3].to_bitmask() ;
-      return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48);
+      return simd_bitmask64_builder<0>()
+        .next(this->chunks[0])
+        .next(this->chunks[1])
+        .next(this->chunks[2])
+        .next(this->chunks[3]);
     }
 
     simdjson_inline uint64_t eq(const T m) const {

src/generic/stage1/amalgamated.h

@@ -1,6 +1,7 @@
 // Stuff other things depend on
 #include <generic/stage1/base.h>
 #include <generic/stage1/buf_block_reader.h>
+#include <generic/stage1/simd_reducer.h>
 #include <generic/stage1/json_escape_scanner.h>
 #include <generic/stage1/json_string_scanner.h>
 #include <generic/stage1/utf8_lookup4_algorithm.h>

src/generic/stage1/json_structural_indexer.h

@@ -239,7 +239,7 @@ simdjson_inline void json_structural_indexer::step<64>(const uint8_t *block, buf
 simdjson_inline void json_structural_indexer::next(const simd::simd8x64<uint8_t>& in, const json_block& block, size_t idx) {
   uint64_t unescaped = in.lteq(0x1F);
 #if SIMDJSON_UTF8VALIDATION
-  checker.check_next_input(in);
+  checker.next(in);
 #endif
   indexer.write(uint32_t(idx-64), prev_structurals); // Output *last* iteration's structurals to the parser
   prev_structurals = block.structural_start();
@@ -343,8 +343,7 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa
         return EMPTY;
     }
   }
-  checker.check_eof();
-  return checker.errors();
+  return std::move(checker).eof();
 }
 
 } // namespace stage1

src/generic/stage1/simd_reducer.h

@@ -0,0 +1,80 @@
+#ifndef SIMDJSON_SRC_GENERIC_STAGE1_SIMD_REDUCER_H
+
+#ifndef SIMDJSON_CONDITIONAL_INCLUDE
+#define SIMDJSON_SRC_GENERIC_STAGE1_SIMD_REDUCER_H
+#include <generic/stage1/base.h>
+#endif // SIMDJSON_CONDITIONAL_INCLUDE
+
+namespace simdjson {
+namespace SIMDJSON_IMPLEMENTATION {
+namespace {
+namespace simd {
+
+/** Incrementally performs a reduce OR operation to accumulate an error. */
+template <int N>
+struct simd_or_reducer;
+
+template<> struct simd_or_reducer<8> {
+  simd8<uint8_t> error01234567;
+  operator simd8<uint8_t>() && { return error01234567; }
+};
+template<> struct simd_or_reducer<7> {
+  simd8<uint8_t> error0123;
+  simd8<uint8_t> error45;
+  simd8<uint8_t> error6;
+  simd_or_reducer<8> next(simd8<uint8_t> error7) && { return { error0123 | error45 | error6 | error7 }; }
+  operator simd8<uint8_t>() && { return error0123 | error45 | error6; }
+};
+template<> struct simd_or_reducer<6> {
+  simd8<uint8_t>    error0123;
+  simd8<uint8_t>    error45;
+  // simd8<uint8_t> error_;
+  simd_or_reducer<7> next(simd8<uint8_t> error6) && { return { error0123, error45, error6 }; }
+  operator simd8<uint8_t>() && { return error0123 | error45; }
+};
+template<> struct simd_or_reducer<5> {
+  simd8<uint8_t>    error0123;
+  // simd8<uint8_t> error__;
+  simd8<uint8_t>    error4;
+  simd_or_reducer<6> next(simd8<uint8_t> error5) && { return { error0123, error4 | error5 }; }
+  operator simd8<uint8_t>() && { return error0123 | error4; }
+};
+template<> struct simd_or_reducer<4> {
+  simd8<uint8_t>    error0123;
+  // simd8<uint8_t> error__;
+  // simd8<uint8_t> error_;
+  operator simd8<uint8_t>() && { return error0123; }
+};
+template<> struct simd_or_reducer<3> {
+  // simd8<uint8_t> error____;
+  simd8<uint8_t>    error01;
+  simd8<uint8_t>    error2;
+  simd_or_reducer<4> next(simd8<uint8_t> error3) && { return { error01 | error2 | error3 }; }
+  operator simd8<uint8_t>() && { return error01 | error2; }
+};
+template<> struct simd_or_reducer<2> {
+  // simd8<uint8_t> error____;
+  simd8<uint8_t>    error01;
+  // simd8<uint8_t> error_;
+  simd_or_reducer<3> next(simd8<uint8_t> error2) && { return { error01, error2 }; }
+  operator simd8<uint8_t>() && { return error01; }
+};
+template<> struct simd_or_reducer<1> {
+  // simd8<uint8_t> error____;
+  // simd8<uint8_t> error__;
+  simd8<uint8_t>    error0;
+  simd_or_reducer<2> next(simd8<uint8_t> error1) && { return { error0 | error1 }; }
+  operator simd8<uint8_t>() && { return error0; }
+};
+template<> struct simd_or_reducer<0> {
+  simd8<uint8_t> prev_error{};
+  simd_or_reducer<1> next(simd8<uint8_t> error0) && { return { prev_error | error0 }; }
+  operator simd8<uint8_t>() && { return prev_error; }
+};
+
+} // namespace simd
+} // unnamed namespace
+} // namespace SIMDJSON_IMPLEMENTATION
+} // namespace simdjson
+
+#endif // SIMDJSON_SRC_GENERIC_STAGE1_SIMD_REDUCER_H

src/generic/stage1/utf8_lookup4_algorithm.h

@@ -4,6 +4,7 @@
 #define SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H
 #include <generic/stage1/base.h>
 #include <generic/dom_parser_implementation.h>
+#include <generic/stage1/simd_reducer.h>
 #endif // SIMDJSON_CONDITIONAL_INCLUDE
 
 namespace simdjson {
@@ -11,15 +12,84 @@ namespace SIMDJSON_IMPLEMENTATION {
 namespace {
 namespace utf8_validation {
 
-using namespace simd;
+  using namespace simd;
 
-  simdjson_inline simd8<uint8_t> check_special_cases(const simd8<uint8_t> input, const simd8<uint8_t> prev1) {
-// Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII)
-// Bit 1 = Too Long (ASCII followed by continuation)
-// Bit 2 = Overlong 3-byte
-// Bit 4 = Surrogate
-// Bit 5 = Overlong 2-byte
-// Bit 7 = Two Continuations
+  // At its height, this will keep 5 registers around.
+  template <int N>
+  struct simd_utf8_checker {
+    /**
+     * The current error.
+     */
+    simd_or_reducer<N> error;
+    /**
+     * Whether the previous input had incomplete UTF-8 characters at the end.
+     */
+    simd8<uint8_t> prev_incomplete;
+
+    /**
+     * Check the next simd input block.
+     */
+    simdjson_inline simd_utf8_checker<N+1> next(simd8<uint8_t> input, simd8<uint8_t> prev_input) &&;
+
+    /**
+     * Used to wrap back around to the beginning, starting with error and prev_incomplete
+     * from the end of this block
+     */
+    simdjson_inline simd_utf8_checker<0> next_cycle() && {
+      return { std::move(error), std::move(prev_incomplete) };
+    }
+
+    /**
+     * Extracts the validation error from the block.
+     */
+    simdjson_inline error_code eof() && {
+      if ((prev_incomplete | std::move(error)).any_bits_set_anywhere()) {
+        return error_code::UTF8_ERROR;
+      } else {
+        return error_code::SUCCESS;
+      }
+    }
+
+  private:
+
+    simdjson_inline simd8<uint8_t> check_utf8_bytes(simd8<uint8_t> input, simd8<uint8_t> prev_input) const;
+    simdjson_inline simd8<uint8_t> is_incomplete(simd8<uint8_t> input) const;
+    simdjson_inline simd8<uint8_t> check_special_cases(simd8<uint8_t> input, simd8<uint8_t> prev1) const;
+    simdjson_inline simd8<uint8_t> check_multibyte_lengths(simd8<uint8_t> input, simd8<uint8_t> prev_input, simd8<uint8_t> sc) const;
+  };
+
+  template <int N>
+  simdjson_inline simd_utf8_checker<N+1> simd_utf8_checker<N>::next(simd8<uint8_t> input, simd8<uint8_t> prev_input) && {
+    if (simdjson_likely(input.is_ascii())) {
+      return {
+        std::move(this->error).next(this->prev_incomplete),
+        simd8<uint8_t>::zero()
+      };
+    } else {
+      return {
+        std::move(this->error).next(check_utf8_bytes(input, prev_input)),
+        is_incomplete(input)
+      };
+    }
+  }
+
+  template <int N>
+  simdjson_inline simd8<uint8_t> simd_utf8_checker<N>::check_utf8_bytes(simd8<uint8_t> input, simd8<uint8_t> prev_input) const {
+    // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ lead bytes
+    // (2, 3, 4-byte leads become large positive numbers instead of small negative numbers)
+    simd8<uint8_t> prev1 = input.prev<1>(prev_input);
+    simd8<uint8_t> sc = check_special_cases(input, prev1);
+    return check_multibyte_lengths(input, prev_input, sc);
+  }
+
+  template <int N>
+  simdjson_inline simd8<uint8_t> simd_utf8_checker<N>::check_special_cases(simd8<uint8_t> input, simd8<uint8_t> prev1) const {
+    // Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII)
+    // Bit 1 = Too Long (ASCII followed by continuation)
+    // Bit 2 = Overlong 3-byte
+    // Bit 4 = Surrogate
+    // Bit 5 = Overlong 2-byte
+    // Bit 7 = Two Continuations
     constexpr const uint8_t TOO_SHORT   = 1<<0; // 11______ 0_______
                                                 // 11______ 11______
     constexpr const uint8_t TOO_LONG    = 1<<1; // 0_______ 10______
@@ -103,8 +173,13 @@ using namespace simd;
     );
     return (byte_1_high & byte_1_low & byte_2_high);
   }
-  simdjson_inline simd8<uint8_t> check_multibyte_lengths(const simd8<uint8_t> input,
-      const simd8<uint8_t> prev_input, const simd8<uint8_t> sc) {
+
+  template <int N>
+  simdjson_inline simd8<uint8_t> simd_utf8_checker<N>::check_multibyte_lengths(
+    simd8<uint8_t> input,
+    simd8<uint8_t> prev_input,
+    simd8<uint8_t> sc
+  ) const {
     simd8<uint8_t> prev2 = input.prev<2>(prev_input);
     simd8<uint8_t> prev3 = input.prev<3>(prev_input);
     simd8<uint8_t> must23 = must_be_2_3_continuation(prev2, prev3);
@@ -116,7 +191,8 @@ using namespace simd;
   // Return nonzero if there are incomplete multibyte characters at the end of the block:
   // e.g. if there is a 4-byte character, but it's 3 bytes from the end.
   //
-  simdjson_inline simd8<uint8_t> is_incomplete(const simd8<uint8_t> input) {
+  template <int N>
+  simdjson_inline simd8<uint8_t> simd_utf8_checker<N>::is_incomplete(simd8<uint8_t> input) const {
     // If the previous input's last 3 bytes match this, they're too short (they ended at EOF):
     // ... 1111____ 111_____ 11______
 #if SIMDJSON_IMPLEMENTATION_ICELAKE
@@ -144,59 +220,43 @@ using namespace simd;
 
   struct utf8_checker {
     // If this is nonzero, there has been a UTF-8 error.
-    simd8<uint8_t> error;
-    // The last input we received
-    simd8<uint8_t> prev_input_block;
-    // Whether the last input we received was incomplete (used for ASCII fast path)
-    simd8<uint8_t> prev_incomplete;
+    simd_utf8_checker<0> checker;
+    simd8<uint8_t> prev_input;
 
-    //
-    // Check whether the current bytes are valid UTF-8.
-    //
-    simdjson_inline void check_utf8_bytes(const simd8<uint8_t> input, const simd8<uint8_t> prev_input) {
-      // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ lead bytes
-      // (2, 3, 4-byte leads become large positive numbers instead of small negative numbers)
-      simd8<uint8_t> prev1 = input.prev<1>(prev_input);
-      simd8<uint8_t> sc = check_special_cases(input, prev1);
-      this->error |= check_multibyte_lengths(input, prev_input, sc);
-    }
+    simdjson_inline void next(const simd8x64<uint8_t>& input) & {
+      // you might think that a for-loop would work, but under Visual Studio, it is not good enough.
+      static_assert(
+        (simd8x64<uint8_t>::NUM_CHUNKS == 1) ||
+        (simd8x64<uint8_t>::NUM_CHUNKS == 2) ||
+        (simd8x64<uint8_t>::NUM_CHUNKS == 4),
+        "We support one, two or four chunks per 64-byte block."
+      );
 
-    // The only problem that can happen at EOF is that a multibyte character is too short
-    // or a byte value too large in the last bytes: check_special_cases only checks for bytes
-    // too large in the first of two bytes.
-    simdjson_inline void check_eof() {
-      // If the previous block had incomplete UTF-8 characters at the end, an ASCII block can't
-      // possibly finish them.
-      this->error |= this->prev_incomplete;
-    }
-
-    simdjson_inline void check_next_input(const simd8x64<uint8_t>& input) {
-      if(simdjson_likely(is_ascii(input))) {
-        this->error |= this->prev_incomplete;
-      } else {
-        // you might think that a for-loop would work, but under Visual Studio, it is not good enough.
-        static_assert((simd8x64<uint8_t>::NUM_CHUNKS == 1)
-                ||(simd8x64<uint8_t>::NUM_CHUNKS == 2)
-                || (simd8x64<uint8_t>::NUM_CHUNKS == 4),
-                "We support one, two or four chunks per 64-byte block.");
-        SIMDJSON_IF_CONSTEXPR (simd8x64<uint8_t>::NUM_CHUNKS == 1) {
-          this->check_utf8_bytes(input.chunks[0], this->prev_input_block);
-        } else SIMDJSON_IF_CONSTEXPR (simd8x64<uint8_t>::NUM_CHUNKS == 2) {
-          this->check_utf8_bytes(input.chunks[0], this->prev_input_block);
-          this->check_utf8_bytes(input.chunks[1], input.chunks[0]);
-        } else SIMDJSON_IF_CONSTEXPR (simd8x64<uint8_t>::NUM_CHUNKS == 4) {
-          this->check_utf8_bytes(input.chunks[0], this->prev_input_block);
-          this->check_utf8_bytes(input.chunks[1], input.chunks[0]);
-          this->check_utf8_bytes(input.chunks[2], input.chunks[1]);
-          this->check_utf8_bytes(input.chunks[3], input.chunks[2]);
-        }
-        this->prev_incomplete = is_incomplete(input.chunks[simd8x64<uint8_t>::NUM_CHUNKS-1]);
-        this->prev_input_block = input.chunks[simd8x64<uint8_t>::NUM_CHUNKS-1];
+      SIMDJSON_IF_CONSTEXPR (simd8x64<uint8_t>::NUM_CHUNKS == 1) {
+        this->checker = std::move(this->checker)
+          .next(input.chunks[0], this->prev_input)
+          .next_cycle();
+      } else SIMDJSON_IF_CONSTEXPR (simd8x64<uint8_t>::NUM_CHUNKS == 2) {
+        this->checker = std::move(this->checker)
+          .next(input.chunks[0], this->prev_input)
+          .next(input.chunks[1], input.chunks[0])
+          .next_cycle();
+      } else SIMDJSON_IF_CONSTEXPR (simd8x64<uint8_t>::NUM_CHUNKS == 4) {
+        this->checker = std::move(this->checker)
+          .next(input.chunks[0], this->prev_input)
+          .next(input.chunks[1], input.chunks[0])
+          .next(input.chunks[2], input.chunks[1])
+          .next(input.chunks[3], input.chunks[2])
+          .next_cycle();
       }
+      this->prev_input = input.chunks[simd8x64<uint8_t>::NUM_CHUNKS-1];
     }
-    // do not forget to call check_eof!
-    simdjson_inline error_code errors() {
-      return this->error.any_bits_set_anywhere() ? error_code::UTF8_ERROR : error_code::SUCCESS;
+
+    simdjson_inline error_code eof() && {
+      // The only problem that can happen at EOF is that a multibyte character is too short
+      // or a byte value too large in the last bytes: check_special_cases only checks for bytes
+      // too large in the first of two bytes.
+      return std::move(this->checker).eof() ? error_code::UTF8_ERROR : error_code::SUCCESS;
     }
 
   }; // struct utf8_checker

src/generic/stage1/utf8_validator.h

@@ -21,16 +21,16 @@ bool generic_validate_utf8(const uint8_t * input, size_t length) {
     buf_block_reader<64> reader(input, length);
     while (reader.has_full_block()) {
       simd::simd8x64<uint8_t> in(reader.full_block());
-      c.check_next_input(in);
+      c.next(in);
       reader.advance();
     }
+    // TODO parse the remainder in sub-blocks (16/32 bytes at a time depending on arch)
     uint8_t block[64]{};
     reader.get_remainder(block);
     simd::simd8x64<uint8_t> in(block);
-    c.check_next_input(in);
+    c.next(in);
     reader.advance();
-    c.check_eof();
-    return c.errors() == error_code::SUCCESS;
+    return std::move(c).eof() == error_code::SUCCESS;
 }
 
 bool generic_validate_utf8(const char * input, size_t length) {