extractor PR with clangcl tweaks

adding benchmark to extractor

.clangd

@@ -45,11 +45,3 @@ Diagnostics:
     Suppress:
         - pragma_attribute_no_pop_eof
         - pragma_attribute_stack_mismatch
----
-# clang 18
-If:
-    PathMatch:
-        - dependencies/.cache/json11/json11.cpp
-CompileFlags:
-    Add:
-        - -Wno-unqualified-std-cast-call

.github/workflows/vs17-clang-ci.yml

@@ -13,15 +13,15 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - {gen: Visual Studio 17 2022, arch: x64, build_type: Debug}
-          - {gen: Visual Studio 17 2022, arch: x64, build_type: Release}
-          - {gen: Visual Studio 17 2022, arch: x64, build_type: RelWithDebInfo}
+          - {gen: Visual Studio 17 2022, arch: x64, build_type: Debug, cxx: 17}
+          - {gen: Visual Studio 17 2022, arch: x64, build_type: Debug, cxx: 20}
+          - {gen: Visual Studio 17 2022, arch: x64, build_type: Release, cxx: 17}
     steps:
     - name: checkout
       uses: actions/checkout@v4
     - name: Configure
       run: |
-        cmake -G "${{matrix.gen}}" -A ${{matrix.arch}}  -T ClangCL -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -B build
+        cmake -G "${{matrix.gen}}" -A ${{matrix.arch}} -DSIMDJSON_CXX_STANDARD=${{matrix.cxx}} -T ClangCL -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -B build
     - name: Build
       run: cmake --build build --config ${{matrix.build_type}} --verbose
     - name: Run tests

CMakeLists.txt

@@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.14)
 project(
     simdjson
     # The version number is modified by tools/release.py
-    VERSION 3.10.0
+    VERSION 3.10.1
     DESCRIPTION "Parsing gigabytes of JSON per second"
     HOMEPAGE_URL "https://simdjson.org/"
     LANGUAGES CXX C
@@ -23,7 +23,11 @@ string(
 set(SIMDJSON_LIB_VERSION "23.0.0" CACHE STRING "simdjson library version")
 set(SIMDJSON_LIB_SOVERSION "23" CACHE STRING "simdjson library soversion")
 
-option(SIMDJSON_BUILD_STATIC_LIB "Build simdjson_static library along with simdjson" OFF)
+option(SIMDJSON_BUILD_STATIC_LIB "Build simdjson_static library along with simdjson (only makes sense if BUILD_SHARED_LIBS=ON)" OFF)
+if(SIMDJSON_BUILD_STATIC_LIB AND NOT BUILD_SHARED_LIBS)
+  message(WARNING "SIMDJSON_BUILD_STATIC_LIB only makes sense if BUILD_SHARED_LIBS is set to ON")
+  message(WARNING "You might be building and installing a two identical static libraries.")
+endif()
 
 option(SIMDJSON_ENABLE_THREADS "Link with thread support" ON)
 
@@ -206,6 +210,7 @@ if(SIMDJSON_BUILD_STATIC_LIB)
       TARGETS simdjson_static
       EXPORT simdjson_staticTargets
       ARCHIVE COMPONENT simdjson_Development
+      INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
   )
   install(
       EXPORT simdjson_staticTargets

Doxyfile

@@ -38,7 +38,7 @@ PROJECT_NAME           = simdjson
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = "3.10.0"
+PROJECT_NUMBER         = "3.10.1"
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a

benchmark/kostya/simdjson_ondemand.h

@@ -24,6 +24,35 @@ struct simdjson_ondemand {
 
 BENCHMARK_TEMPLATE(kostya, simdjson_ondemand)->UseManualTime();
 
+
+#if SIMDJSON_SUPPORTS_EXTRACT
+using namespace simdjson::ondemand;
+
+struct simdjson_ondemand_extract {
+  static constexpr diff_flags DiffFlags = diff_flags::NONE;
+
+  ondemand::parser parser{};
+
+  bool run(simdjson::padded_string &json, std::vector<point> &result) {
+    auto doc = parser.iterate(json);
+    for (ondemand::object object_point : doc.find_field("coordinates")) {
+      point p;
+      auto error = object_point.extract(
+        to{"x", p.x},
+        to{"y", p.y},
+        to{"z", p.z}
+      );
+      if(error) { return false; }
+      result.push_back(p);
+    }
+    return true;
+  }
+};
+
+BENCHMARK_TEMPLATE(kostya, simdjson_ondemand_extract)->UseManualTime();
+
+#endif // SIMDJSON_SUPPORTS_EXTRACT
+
 } // namespace kostya
 
 #endif // SIMDJSON_EXCEPTIONS

benchmark/partial_tweets/simdjson_ondemand.h

@@ -9,7 +9,7 @@ namespace partial_tweets {
 using namespace simdjson;
 
 struct simdjson_ondemand {
-  using StringType=std::string_view;
+  using StringType = std::string_view;
 
   ondemand::parser parser{};
 
@@ -43,6 +43,48 @@ struct simdjson_ondemand {
 
 BENCHMARK_TEMPLATE(partial_tweets, simdjson_ondemand)->UseManualTime();
 
+
+#if SIMDJSON_SUPPORTS_EXTRACT
+
+using namespace simdjson::ondemand;
+
+struct simdjson_ondemand_extract {
+  using StringType = std::string_view;
+  ondemand::parser parser{};
+  bool run(simdjson::padded_string &json, std::vector<tweet<std::string_view>> &result) {
+    // Walk the document, parsing the tweets as we go
+    auto doc = parser.iterate(json);
+    for (ondemand::object tweet_object : doc.find_field("statuses")) {
+      tweet<std::string_view> t;
+      auto error = tweet_object.extract(
+        to{"created_at", t.created_at},
+        to{"id", t.id},
+        to{"text", t.result},
+        to{"in_reply_to_status_id", [&t](auto val) {
+          if(val.is_null()) {
+            t.in_reply_to_status_id = 0;
+          } else {
+            t.in_reply_to_status_id = val;
+          }
+         }},
+        to{"user", sub{
+          to{"id", t.user.id},
+          to{"screen_name", t.user.screen_name},
+        }},
+        to{"retweet_count", t.retweet_count},
+        to{"favorite_count", t.favorite_count});
+        if(error) { return false; }
+        result.push_back(t);
+    }
+    return true;
+  }
+};
+
+BENCHMARK_TEMPLATE(partial_tweets, simdjson_ondemand_extract)->UseManualTime();
+
+
+#endif
+
 } // namespace partial_tweets
 
 #endif // SIMDJSON_EXCEPTIONS

include/simdjson/arm64/simd.h

@@ -210,7 +210,6 @@ namespace {
 
     // Bit-specific operations
     simdjson_inline simd8<bool> any_bits_set(simd8<uint8_t> bits) const { return vtstq_u8(*this, bits); }
-    simdjson_inline bool is_ascii() const { return this->max_val() < 0x80u; }
     simdjson_inline bool any_bits_set_anywhere() const { return this->max_val() != 0; }
     simdjson_inline bool any_bits_set_anywhere(simd8<uint8_t> bits) const { return (*this & bits).any_bits_set_anywhere(); }
     template<int N>
@@ -413,66 +412,6 @@ namespace {
     }
   };
 
-#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
-      static const uint8x16_t BITMASK64_BUILDER_MASK = simdjson_make_uint8x16_t(
-        0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80,
-        0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80
-      );
-#else
-      static const uint8x16_t BITMASK64_BUILDER_MASK = {
-        0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80,
-        0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80
-      };
-#endif
-
-  template <int N = 0>
-  struct simd_bitmask64_builder;
-
-  template<>
-  struct simd_bitmask64_builder<4> {
-    const uint64_t mask;
-    operator uint64_t() && { return mask; }
-  }; // struct simd_bitmask64_builder<4>
-
-  template<>
-  struct simd_bitmask64_builder<3> {
-    const uint8x16_t sum01;
-    const simd8<bool> val2;
-    simdjson_inline simd_bitmask64_builder<4> next(simd8<bool> val3) {
-      // Add each of the elements next to each other, successively, to stuff each 8 byte mask into one.
-      uint8x16_t sum23 = vpaddq_u8(val2 & BITMASK64_BUILDER_MASK, val3 & BITMASK64_BUILDER_MASK);
-      uint8x16_t sum0123 = vpaddq_u8(sum01, sum23);
-
-      // This algorithm is actually designed to create a 128-bit mask from 8 16-byte simd masks,
-      // but since we only want 64 bits, we add the mask to itself (creating the final mask twice).
-      uint8x16_t sum01230123 = vpaddq_u8(sum0123, sum0123);
-      return { vgetq_lane_u64(vreinterpretq_u64_u8(sum01230123), 0) };
-    }
-  }; // struct simd_bitmask64_builder<3>
-
-  template<>
-  struct simd_bitmask64_builder<2> {
-    const uint8x16_t sum01;
-    simdjson_inline simd_bitmask64_builder<3> next(simd8<bool> val2) {
-      return { sum01, val2 };
-    }
-  }; // struct simd_bitmask64_builder<2>
-
-  template<>
-  struct simd_bitmask64_builder<1> {
-    const simd8<bool> val0;
-    simdjson_inline simd_bitmask64_builder<2> next(simd8<bool> val1) {
-      return { vpaddq_u8(val0 & BITMASK64_BUILDER_MASK, val1 & BITMASK64_BUILDER_MASK) };
-    }
-  }; // struct simd_bitmask64_builder<1>
-
-  template<>
-  struct simd_bitmask64_builder<0> {
-    simdjson_inline simd_bitmask64_builder<1> next(simd8<bool> val) {
-      return { val };
-    }
-  }; // struct simd_bitmask64_builder<0>
-
   template<typename T>
   struct simd8x64 {
     static constexpr int NUM_CHUNKS = 64 / sizeof(simd8<T>);
@@ -510,11 +449,23 @@ namespace {
     }
 
     simdjson_inline uint64_t to_bitmask() const {
-      return simd_bitmask64_builder<0>()
-        .next(this->chunks[0])
-        .next(this->chunks[1])
-        .next(this->chunks[2])
-        .next(this->chunks[3]);
+#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
+      const uint8x16_t bit_mask = simdjson_make_uint8x16_t(
+        0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80,
+        0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80
+      );
+#else
+      const uint8x16_t bit_mask = {
+        0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80,
+        0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80
+      };
+#endif
+      // Add each of the elements next to each other, successively, to stuff each 8 byte mask into one.
+      uint8x16_t sum0 = vpaddq_u8(this->chunks[0] & bit_mask, this->chunks[1] & bit_mask);
+      uint8x16_t sum1 = vpaddq_u8(this->chunks[2] & bit_mask, this->chunks[3] & bit_mask);
+      sum0 = vpaddq_u8(sum0, sum1);
+      sum0 = vpaddq_u8(sum0, sum0);
+      return vgetq_lane_u64(vreinterpretq_u64_u8(sum0), 0);
     }
 
     simdjson_inline uint64_t eq(const T m) const {

include/simdjson/fallback/bitmanipulation.h

@@ -9,7 +9,7 @@ namespace simdjson {
 namespace fallback {
 namespace {
 
-#if defined(_MSC_VER) && !defined(_M_ARM64) && !defined(_M_X64)
+#if SIMDJSON_REGULAR_VISUAL_STUDIO && !defined(_M_ARM64) && !defined(_M_X64)
 static inline unsigned char _BitScanForward64(unsigned long* ret, uint64_t x) {
   unsigned long x0 = (unsigned long)x, top, bottom;
   _BitScanForward(&top, (unsigned long)(x >> 32));
@@ -28,7 +28,7 @@ static unsigned char _BitScanReverse64(unsigned long* ret, uint64_t x) {
 
 /* result might be undefined when input_num is zero */
 simdjson_inline int leading_zeroes(uint64_t input_num) {
-#ifdef _MSC_VER
+#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
   unsigned long leading_zero = 0;
   // Search the mask data from most significant bit (MSB)
   // to least significant bit (LSB) for a set bit (1).
@@ -38,7 +38,7 @@ simdjson_inline int leading_zeroes(uint64_t input_num) {
     return 64;
 #else
   return __builtin_clzll(input_num);
-#endif// _MSC_VER
+#endif// SIMDJSON_REGULAR_VISUAL_STUDIO
 }
 
 } // unnamed namespace

include/simdjson/generic/ondemand/object-inl.h

@@ -15,6 +15,141 @@ namespace simdjson {
 namespace SIMDJSON_IMPLEMENTATION {
 namespace ondemand {
 
+
+#ifdef SIMDJSON_SUPPORTS_EXTRACT
+
+
+#if SIMDJSON_REGULAR_VISUAL_STUDIO
+template <endpoint ...Funcs>
+simdjson_inline error_code object::extract(Funcs&&... endpoints) {
+  return iter.on_field_raw([&, eps = std::make_tuple(std::forward<Funcs>(endpoints)...)](auto field_key, error_code& error) mutable {
+    std::apply([&](auto &...endpoints) {
+       std::ignore = ((field_key.unsafe_is_equal(endpoints.key()) ? (error = endpoints(value(iter.child()))) == SUCCESS : true) && ...);
+    }, eps);
+    if (error) {
+      return true;
+    }
+    return false;
+  });
+}
+#else
+template <endpoint ...Funcs>
+simdjson_inline error_code object::extract(Funcs&&... endpoints) noexcept((nothrow_endpoint<Funcs> && ...)) {
+  return iter.on_field_raw([&](auto field_key, error_code& error) noexcept((nothrow_endpoint<Funcs> && ...)) {
+    std::ignore = ((field_key.unsafe_is_equal(endpoints.key()) ? (error = endpoints(value(iter.child()))) == SUCCESS : true) && ...);
+    if (error) {
+      return true;
+    }
+    return false;
+  });
+}
+#endif
+
+template <typename T>
+struct to {
+private:
+  T *pointer;
+  std::string_view m_key;
+
+public:
+  constexpr explicit(false)
+      to(std::string_view const inp_key, T &obj_ref) noexcept
+      : pointer{std::addressof(obj_ref)}, m_key{inp_key} {}
+
+  constexpr to(to const &) = default;
+  constexpr to(to &&) noexcept = default;
+  constexpr to &operator=(to const &) = default;
+  constexpr to &operator=(to &&) noexcept = default;
+  constexpr ~to() = default;
+
+  [[nodiscard]] constexpr std::string_view key() const noexcept {
+    return m_key;
+  }
+
+  [[nodiscard]] constexpr error_code operator()(simdjson_result<value> val) noexcept(
+      std::is_nothrow_assignable_v<T, simdjson_result<value>>) {
+    return val.get<T>(*pointer);
+  }
+};
+
+template <typename Func>
+  requires(std::is_invocable_v<Func, simdjson_result<value>>)
+struct to<Func> {
+private:
+  Func func;
+  std::string_view m_key;
+
+public:
+  constexpr explicit(false)
+      to(std::string_view const inp_key,
+         Func &&inp_func) noexcept(std::is_nothrow_copy_assignable_v<Func>)
+      : func{std::forward<Func>(inp_func)}, m_key{inp_key} {}
+
+  constexpr to(to const &) = default;
+  constexpr to(to &&) noexcept = default;
+  constexpr to& operator=(to const &) = default;
+  constexpr to& operator=(to &&) noexcept = default;
+  constexpr ~to() = default;
+
+  [[nodiscard]] constexpr std::string_view key() const noexcept {
+    return m_key;
+  }
+
+  [[nodiscard]] constexpr error_code operator()(simdjson_result<value> val) noexcept(
+      std::is_nothrow_invocable_v<Func, simdjson_result<value>>) {
+    if constexpr (std::is_invocable_r_v<error_code, Func, simdjson_result<value>>) {
+      return func(val);
+    } else {
+      static_cast<void>(func(val));
+      return SUCCESS;
+    }
+  }
+};
+
+template <typename Func>
+  requires(std::is_invocable_v<Func, simdjson_result<value>>)
+to(std::string_view, Func &&) -> to<Func>;
+
+template <typename T>
+to(std::string_view, T&) -> to<T>;
+
+template <endpoint... Tos>
+struct sub {
+private:
+  using tuple_type = std::tuple<Tos...>;
+  tuple_type tos;
+
+public:
+
+  // double templating to make perfect forwarding work
+  template <typename ...T>
+    requires ((std::same_as<T, Tos> && ...))
+  explicit constexpr sub(T&&...inp_tos) noexcept(std::is_nothrow_constructible_v<tuple_type, T...>)
+      : tos{std::forward<T>(inp_tos)...} {}
+
+  constexpr sub(sub const &) = default;
+  constexpr sub(sub &&) noexcept = default;
+  constexpr sub &operator=(sub const &) = default;
+  constexpr sub &operator=(sub &&) = default;
+  constexpr ~sub() = default;
+
+  [[nodiscard]] constexpr error_code operator()(simdjson_result<value> val) noexcept((nothrow_endpoint<Tos> && ...)) {
+    object obj;
+    if (auto const err = val.get_object().get(obj); err) {
+      return err;
+    }
+    return std::apply([&obj]<typename... T>(T &&...app_tos) {
+      return obj.extract(std::forward<T>(app_tos)...);
+    }, tos);
+  }
+};
+
+template <endpoint... Tos>
+sub(Tos&&...) -> sub<Tos...>;
+
+#endif
+
+
 simdjson_inline simdjson_result<value> object::find_field_unordered(const std::string_view key) & noexcept {
   bool has_value;
   SIMDJSON_TRY( iter.find_field_unordered_raw(key).get(has_value) );

include/simdjson/generic/ondemand/object.h

@@ -11,6 +11,20 @@ namespace simdjson {
 namespace SIMDJSON_IMPLEMENTATION {
 namespace ondemand {
 
+#if defined(__cpp_concepts) && defined(__cpp_consteval)
+#define SIMDJSON_SUPPORTS_EXTRACT 1
+
+template <typename T>
+concept endpoint = std::is_invocable_r_v<error_code, T, simdjson_result<value>> &&
+                   std::is_copy_constructible_v<T> && requires(T to) {
+                     { to.key() } noexcept -> std::convertible_to<std::string_view>;
+                   };
+
+template <typename T>
+concept nothrow_endpoint = endpoint<T> && std::is_nothrow_invocable_r_v<error_code, T, simdjson_result<value>>;
+
+#endif
+
 /**
  * A forward-only JSON object field iterator.
  */
@@ -65,6 +79,21 @@ public:
   /** @overload simdjson_inline simdjson_result<value> find_field(std::string_view key) & noexcept; */
   simdjson_inline simdjson_result<value> find_field(std::string_view key) && noexcept;
 
+#ifdef SIMDJSON_SUPPORTS_EXTRACT
+  /**
+   * Extract all the fields in one go
+   * Funcs are invocables that take a simdjson_result<value> as input.
+   */
+  template <endpoint ...Funcs>
+  simdjson_inline error_code extract(Funcs&&... endpoints)
+#ifndef SIMDJSON_REGULAR_VISUAL_STUDIO // msvc thinks noexcept is not the same in definition
+ noexcept((nothrow_endpoint<Funcs> && ...))
+#endif
+ ;
+#endif
+
+
+
   /**
    * Look up a field by name on an object, without regard to key order.
    *

include/simdjson/generic/ondemand/value_iterator-inl.h

@@ -1,3 +1,4 @@
+#include <type_traits>
 #ifndef SIMDJSON_GENERIC_ONDEMAND_VALUE_ITERATOR_INL_H
 
 #ifndef SIMDJSON_CONDITIONAL_INCLUDE
@@ -109,6 +110,108 @@ simdjson_warn_unused simdjson_inline simdjson_result<bool> value_iterator::has_n
   }
 }
 
+template <typename Func>
+simdjson_warn_unused simdjson_inline error_code value_iterator::on_field_raw(Func&& func)
+#ifdef __cpp_lib_is_invocable
+  noexcept(std::is_nothrow_invocable_r_v<bool, Func, raw_json_string, error_code&>)
+#else
+  noexcept(false)
+#endif
+{
+#ifdef __cpp_lib_is_invocable
+  static_assert(std::is_invocable_r_v<bool, Func, raw_json_string, error_code&>, "Invalid function provided.");
+#endif
+
+  error_code error = SUCCESS;
+  bool has_value;
+  //
+  // Initially, the object can be in one of a few different places:
+  //
+  // 1. The start of the object, at the first field:
+  //
+  //    ```
+  //    { "a": [ 1, 2 ], "b": [ 3, 4 ] }
+  //      ^ (depth 2, index 1)
+  //    ```
+  if (at_first_field()) {
+    has_value = true;
+
+  //
+  // 2. When a previous search did not yield a value or the object is empty:
+  //
+  //    ```
+  //    { "a": [ 1, 2 ], "b": [ 3, 4 ] }
+  //                                     ^ (depth 0)
+  //    { }
+  //        ^ (depth 0, index 2)
+  //    ```
+  //
+  } else if (!is_open()) {
+#if SIMDJSON_DEVELOPMENT_CHECKS
+    // If we're past the end of the object, we're being iterated out of order.
+    // Note: this is not perfect detection. It's possible the user is inside some other object; if so,
+    // this object iterator will blithely scan that object for fields.
+    if (_json_iter->depth() < depth() - 1) { return OUT_OF_ORDER_ITERATION; }
+#endif
+    return EMPTY;
+
+  // 3. When a previous search found a field or an iterator yielded a value:
+  //
+  //    ```
+  //    // When a field was not fully consumed (or not even touched at all)
+  //    { "a": [ 1, 2 ], "b": [ 3, 4 ] }
+  //           ^ (depth 2)
+  //    // When a field was fully consumed
+  //    { "a": [ 1, 2 ], "b": [ 3, 4 ] }
+  //                   ^ (depth 1)
+  //    // When the last field was fully consumed
+  //    { "a": [ 1, 2 ], "b": [ 3, 4 ] }
+  //                                   ^ (depth 1)
+  //    ```
+  //
+  } else {
+    if ((error = skip_child() )) { abandon(); return error; }
+    if ((error = has_next_field().get(has_value) )) { abandon(); return error; }
+#if SIMDJSON_DEVELOPMENT_CHECKS
+    if (_json_iter->start_position(_depth) != start_position()) { return OUT_OF_ORDER_ITERATION; }
+#endif
+  }
+  while (has_value) {
+    // Get the key and colon, stopping at the value.
+    raw_json_string actual_key;
+    // size_t max_key_length = _json_iter->peek_length() - 2; // -2 for the two quotes
+    // Note: _json_iter->peek_length() - 2 might overflow if _json_iter->peek_length() < 2.
+    // field_key() advances the pointer and checks that '"' is found (corresponding to a key).
+    // The depth is left unchanged by field_key().
+    if ((error = field_key().get(actual_key) )) { abandon(); return error; };
+    // field_value() will advance and check that we find a ':' separating the
+    // key and the value. It will also increment the depth by one.
+    if ((error = field_value() )) { abandon(); return error; }
+    // If it matches, stop and return
+    // We could do it this way if we wanted to allow arbitrary
+    // key content (including escaped quotes).
+    //if (actual_key.unsafe_is_equal(max_key_length, key)) {
+    // Instead we do the following which may trigger buffer overruns if the
+    // user provides an adversarial key (containing a well placed unescaped quote
+    // character and being longer than the number of bytes remaining in the JSON
+    // input).
+    if (func(actual_key, error)) {
+      break;
+    }
+
+    // The call to skip_child is meant to skip over the value corresponding to the key.
+    // After skip_child(), we are right before the next comma (',') or the final brace ('}').
+    SIMDJSON_TRY( skip_child() ); // Skip the value entirely
+    // The has_next_field() advances the pointer and check that either ',' or '}' is found.
+    // It returns true if ',' is found, false otherwise. If anything other than ',' or '}' is found,
+    // then we are in error and we abort.
+    if ((error = has_next_field().get(has_value) )) { abandon(); return error; }
+  }
+
+  // If the loop ended, we're out of fields to look at.
+  return error;
+}
+
 simdjson_warn_unused simdjson_inline simdjson_result<bool> value_iterator::find_field_raw(const std::string_view key) noexcept {
   error_code error;
   bool has_value;
@@ -1088,4 +1191,4 @@ simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value_iterato
 
 } // namespace simdjson
 
-#endif // SIMDJSON_GENERIC_ONDEMAND_VALUE_ITERATOR_INL_H
+#endif // SIMDJSON_GENERIC_ONDEMAND_VALUE_ITERATOR_INL_H

include/simdjson/generic/ondemand/value_iterator.h

@@ -6,6 +6,12 @@
 #include "simdjson/generic/implementation_simdjson_result_base.h"
 #endif // SIMDJSON_CONDITIONAL_INCLUDE
 
+#ifdef __has_include
+#if __has_include (<version>)
+#include <version>
+#endif
+#endif
+
 namespace simdjson {
 namespace SIMDJSON_IMPLEMENTATION {
 namespace ondemand {
@@ -198,6 +204,22 @@ public:
    */
   simdjson_warn_unused simdjson_inline simdjson_result<bool> find_field_raw(const std::string_view key) noexcept;
 
+
+  /**
+   * Runs Func on each key found.
+   * Almost same as `find_field_raw` but it runs `func` instead of checking the key ourselves.
+   *
+   * @param Func func(raw_json_string key, error_code& error) noexcept
+   */
+  template <typename Func>
+  simdjson_warn_unused simdjson_inline error_code on_field_raw(Func&& func)
+#ifdef __cpp_lib_is_invocable
+  noexcept(std::is_nothrow_invocable_r_v<bool, Func, raw_json_string, error_code&>)
+#else
+  noexcept(false)
+#endif
+  ;
+
   /**
    * Find the field with the given key without regard to order, and *without* unescaping.
    *
@@ -489,4 +511,4 @@ public:
 
 } // namespace simdjson
 
-#endif // SIMDJSON_GENERIC_ONDEMAND_VALUE_ITERATOR_H
+#endif // SIMDJSON_GENERIC_ONDEMAND_VALUE_ITERATOR_H

include/simdjson/haswell/simd.h

@@ -295,30 +295,6 @@ namespace simd {
     simdjson_inline int get_bit() const { return _mm256_movemask_epi8(_mm256_slli_epi16(*this, 7-N)); }
   };
 
-  template <int N = 0>
-  struct simd_bitmask64_builder;
-
-  template<>
-  struct simd_bitmask64_builder<2> {
-    const uint64_t bitmask;
-    operator uint64_t() && { return bitmask; }
-  }; // struct simd_bitmask64_builder<2>
-
-  template<>
-  struct simd_bitmask64_builder<1> {
-    const int bitmask;
-    simdjson_inline simd_bitmask64_builder<2> next(simd8<bool> val) {
-      uint64_t r_lo = uint32_t(bitmask);
-      uint64_t r_hi = val.to_bitmask();
-      return { r_lo | (r_hi << 32) };
-    }
-  }; // struct simd_bitmask64_builder<1>
-
-  template<>
-  struct simd_bitmask64_builder<0> {
-    simdjson_inline simd_bitmask64_builder<1> next(simd8<bool> val) { return { val.to_bitmask() }; }
-  }; // struct simd_bitmask64_builder<0>
-
   template<typename T>
   struct simd8x64 {
     static constexpr int NUM_CHUNKS = 64 / sizeof(simd8<T>);
@@ -346,9 +322,9 @@ namespace simd {
     }
 
     simdjson_inline uint64_t to_bitmask() const {
-      return simd_bitmask64_builder<0>()
-        .next(this->chunks[0])
-        .next(this->chunks[1]);
+      uint64_t r_lo = uint32_t(this->chunks[0].to_bitmask());
+      uint64_t r_hi =                       this->chunks[1].to_bitmask();
+      return r_lo | (r_hi << 32);
     }
 
     simdjson_inline simd8<T> reduce_or() const {

include/simdjson/icelake/simd.h

@@ -314,20 +314,6 @@ namespace simd {
     simdjson_inline uint64_t get_bit() const { return _mm512_movepi8_mask(_mm512_slli_epi16(*this, 7-N)); }
   };
 
-  template <int N = 0>
-  struct simd_bitmask64_builder;
-
-  template<>
-  struct simd_bitmask64_builder<1> {
-    const __mmask64 bitmask;
-    operator __mmask64() && { return bitmask; }
-  }; // struct simd_bitmask64_builder<2>
-
-  template<>
-  struct simd_bitmask64_builder<0> {
-    simdjson_inline simd_bitmask64_builder<1> next(simd8<bool> val) { return { _mm512_movepi8_mask(val) }; }
-  }; // struct simd_bitmask64_builder<0>
-
   template<typename T>
   struct simd8x64 {
     static constexpr int NUM_CHUNKS = 64 / sizeof(simd8<T>);

include/simdjson/lasx/simd.h

@@ -298,35 +298,6 @@ namespace simd {
     simdjson_inline simd8<uint8_t> shl() const { return simd8<uint8_t>(__lasx_xvslli_b(*this, N)); }
   };
 
-  template <int N = 0>
-  struct simd_bitmask64_builder;
-
-  template<>
-  struct simd_bitmask64_builder<2> {
-    const unsigned long int mask01;
-    operator uint64_t() { return mask01; }
-  }; // struct simd_bitmask64_builder<2>
-
-  template<>
-  struct simd_bitmask64_builder<1> {
-    const __m256i mask0;
-    simdjson_inline simd_bitmask64_builder<2> next(simd8<bool> val1) {
-      __m256i mask1 = __lasx_xvmskltz_b(val1);
-      __m256i mask_tmp = __lasx_xvpickve_w(mask0, 4);
-      __m256i tmp = __lasx_xvpickve_w(mask1, 4);
-      __m256i mask01 = __lasx_xvinsve0_w(mask0, mask1, 1);
-      __m256i mask01_tmp = __lasx_xvinsve0_w(mask_tmp, tmp, 1);
-      return { __lasx_xvpickve2gr_du(__lasx_xvpackev_h(mask01_tmp, mask01), 0) };
-    }
-  }; // struct simd_bitmask64_builder<1>
-
-  template<>
-  struct simd_bitmask64_builder<0> {
-    simdjson_inline simd_bitmask64_builder<1> next(simd8<bool> val) {
-      return { __lasx_xvmskltz_b(val0) };
-    }
-  }; // struct simd_bitmask64_builder<0>
-
   template<typename T>
   struct simd8x64 {
     static constexpr int NUM_CHUNKS = 64 / sizeof(simd8<T>);
@@ -360,9 +331,13 @@ namespace simd {
     }
 
     simdjson_inline uint64_t to_bitmask() const {
-      return simd_bitmask64_builder<0>()
-        .next(this->chunks[0])
-        .next(this->chunks[1]);
+      __m256i mask0 = __lasx_xvmskltz_b(this->chunks[0]);
+      __m256i mask1 = __lasx_xvmskltz_b(this->chunks[1]);
+      __m256i mask_tmp = __lasx_xvpickve_w(mask0, 4);
+      __m256i tmp = __lasx_xvpickve_w(mask1, 4);
+      mask0 = __lasx_xvinsve0_w(mask0, mask1, 1);
+      mask_tmp = __lasx_xvinsve0_w(mask_tmp, tmp, 1);
+      return __lasx_xvpickve2gr_du(__lasx_xvpackev_h(mask_tmp, mask0), 0);
     }
 
     simdjson_inline simd8<T> reduce_or() const {

include/simdjson/lsx/simd.h

@@ -255,50 +255,6 @@ namespace simd {
     simdjson_inline simd8<uint8_t> shl() const { return simd8<uint8_t>(__lsx_vslli_b(*this, N)); }
   };
 
-  template <int N = 0>
-  struct simd_bitmask64_builder;
-
-  template<>
-  struct simd_bitmask64_builder<4> {
-    const unsigned long int result;
-    operator uint64_t() { return result; }
-  }; // struct simd_bitmask64_builder<4>
-
-  template<>
-  struct simd_bitmask64_builder<3> {
-    const __m128i mask01;
-    const __m128i mask2;
-    simdjson_inline simd_bitmask64_builder<4> next(simd8<bool> val3) {
-      __m128i mask3 = __lsx_vmskltz_b(val3);
-      __m128i mask23 = __lsx_vilvl_h(mask3, mask2);
-      return { __lsx_vpickve2gr_du(__lsx_vilvl_w(mask23, mask01), 0) };
-    }
-  }; // struct simd_bitmask64_builder<3>
-
-  template<>
-  struct simd_bitmask64_builder<2> {
-    const __m128i mask01;
-    simdjson_inline simd_bitmask64_builder<3> next(simd8<bool> val2) {
-      return { mask01, __lsx_vmskltz_b(val2) };
-    }
-  }; // struct simd_bitmask64_builder<2>
-
-  template<>
-  struct simd_bitmask64_builder<1> {
-    const __m128i mask0;
-    simdjson_inline simd_bitmask64_builder<2> next(simd8<bool> val1) {
-      __m128i mask1 = __lsx_vmskltz_b(val1);
-      return { __lsx_vilvl_h(mask1, mask0) };
-    }
-  }; // struct simd_bitmask64_builder<1>
-
-  template<>
-  struct simd_bitmask64_builder<0> {
-    simdjson_inline simd_bitmask64_builder<1> next(simd8<bool> val0) {
-      return { __lsx_vmskltz_b(val0) };
-    }
-  }; // struct simd_bitmask64_builder<0>
-
   template<typename T>
   struct simd8x64 {
     static constexpr int NUM_CHUNKS = 64 / sizeof(simd8<T>);
@@ -347,11 +303,13 @@ namespace simd {
     }
 
     simdjson_inline uint64_t to_bitmask() const {
-      return simd_bitmask64_builder<0>()
-        .next(this->chunks[0])
-        .next(this->chunks[1])
-        .next(this->chunks[2])
-        .next(this->chunks[3]);
+      __m128i mask1 = __lsx_vmskltz_b(this->chunks[0]);
+      __m128i mask2 = __lsx_vmskltz_b(this->chunks[1]);
+      __m128i mask3 = __lsx_vmskltz_b(this->chunks[2]);
+      __m128i mask4 = __lsx_vmskltz_b(this->chunks[3]);
+      mask1 = __lsx_vilvl_h(mask2, mask1);
+      mask2 = __lsx_vilvl_h(mask4, mask3);
+      return __lsx_vpickve2gr_du(__lsx_vilvl_w(mask2, mask1), 0);
     }
 
     simdjson_inline simd8<T> reduce_or() const {

include/simdjson/padded_string-inl.h

@@ -53,6 +53,9 @@ inline padded_string::padded_string(const char *data, size_t length) noexcept
   if ((data != nullptr) && (data_ptr != nullptr)) {
     std::memcpy(data_ptr, data, length);
   }
+  if (data_ptr == nullptr) {
+    viable_size = 0;
+  }
 }
 #ifdef __cpp_char8_t
 inline padded_string::padded_string(const char8_t *data, size_t length) noexcept
@@ -60,12 +63,17 @@ inline padded_string::padded_string(const char8_t *data, size_t length) noexcept
   if ((data != nullptr) && (data_ptr != nullptr)) {
     std::memcpy(data_ptr, reinterpret_cast<const char *>(data), length);
   }
+  if (data_ptr == nullptr) {
+    viable_size = 0;
+  }
 }
 #endif
 // note: do not pass std::string arguments by value
 inline padded_string::padded_string(const std::string & str_ ) noexcept
     : viable_size(str_.size()), data_ptr(internal::allocate_padded_buffer(str_.size())) {
-  if (data_ptr != nullptr) {
+  if (data_ptr == nullptr) {
+    viable_size = 0;
+  } else {
     std::memcpy(data_ptr, str_.data(), str_.size());
   }
 }

include/simdjson/ppc64/simd.h

@@ -392,51 +392,6 @@ template <> struct simd8<uint8_t> : base8_numeric<uint8_t> {
   }
 };
 
-template <int N = 0>
-struct simd_bitmask64_builder;
-
-template<>
-struct simd_bitmask64_builder<4> {
-  const uint64_t bitmask;
-  simdjson_inline operator uint64_t() && { return bitmask; }
-}; // struct simd_bitmask64_builder<4>
-
-template<>
-struct simd_bitmask64_builder<3> {
-  const int bitmask01;
-  const int bitmask2;
-  simdjson_inline simd_bitmask64_builder<4> next(simd8<bool> val) {
-    return {
-      uint64_t(this->bitmask01) |
-      (uint64_t(this->bitmask2) << 32) |
-      (uint64_t(val.to_bitmask()) << 48)
-    };
-  }
-}; // struct simd_bitmask64_builder<3>
-
-template<>
-struct simd_bitmask64_builder<2> {
-  const int bitmask01;
-  simdjson_inline simd_bitmask64_builder<3> next(simd8<bool> val) {
-    return { bitmask01, val.to_bitmask() };
-  }
-}; // struct simd_bitmask64_builder<2>
-
-template<>
-struct simd_bitmask64_builder<1> {
-  const int bitmask0;
-  simdjson_inline simd_bitmask64_builder<2> next(simd8<bool> val) {
-    return { bitmask0 | (val.to_bitmask() << 16) };
-  }
-}; // struct simd_bitmask64_builder<1>
-
-template<>
-struct simd_bitmask64_builder<0> {
-  simdjson_inline simd_bitmask64_builder<1> next(simd8<bool> val) {
-    return { val.to_bitmask() };
-  }
-}; // struct simd_bitmask64_builder<0>
-
 template <typename T> struct simd8x64 {
   static constexpr int NUM_CHUNKS = 64 / sizeof(simd8<T>);
   static_assert(NUM_CHUNKS == 4,
@@ -479,11 +434,11 @@ template <typename T> struct simd8x64 {
   }
 
   simdjson_inline uint64_t to_bitmask() const {
-      return simd_bitmask64_builder<0>()
-        .next(this->chunks[0])
-        .next(this->chunks[1])
-        .next(this->chunks[2])
-        .next(this->chunks[3]);
+    uint64_t r0 = uint32_t(this->chunks[0].to_bitmask());
+    uint64_t r1 = this->chunks[1].to_bitmask();
+    uint64_t r2 = this->chunks[2].to_bitmask();
+    uint64_t r3 = this->chunks[3].to_bitmask();
+    return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48);
   }
 
   simdjson_inline uint64_t eq(const T m) const {

include/simdjson/simdjson_version.h

@@ -4,7 +4,7 @@
 #define SIMDJSON_SIMDJSON_VERSION_H
 
 /** The version of simdjson being used (major.minor.revision) */
-#define SIMDJSON_VERSION "3.10.0"
+#define SIMDJSON_VERSION "3.10.1"
 
 namespace simdjson {
 enum {
@@ -19,7 +19,7 @@ enum {
   /**
    * The revision (major.minor.REVISION) of simdjson being used.
    */
-  SIMDJSON_VERSION_REVISION = 0
+  SIMDJSON_VERSION_REVISION = 1
 };
 } // namespace simdjson
 

include/simdjson/westmere/simd.h

@@ -260,51 +260,6 @@ namespace simd {
     simdjson_inline int get_bit() const { return _mm_movemask_epi8(_mm_slli_epi16(*this, 7-N)); }
   };
 
-  template <int N = 0>
-  struct simd_bitmask64_builder;
-
-  template<>
-  struct simd_bitmask64_builder<4> {
-    const uint64_t bitmask;
-    operator uint64_t() && { return bitmask; }
-  }; // struct simd_bitmask64_builder<4>
-
-  template<>
-  struct simd_bitmask64_builder<3> {
-    uint32_t bitmask01;
-    const int bitmask2;
-    simdjson_inline simd_bitmask64_builder<4> next(simd8<bool> val3) {
-      return {
-        uint64_t(this->bitmask01) |
-        (uint64_t(this->bitmask2) << 32) |
-        (uint64_t(val3.to_bitmask()) << 48)
-      };
-    }
-  }; // struct simd_bitmask64_builder<3>
-
-  template<>
-  struct simd_bitmask64_builder<2> {
-    uint32_t bitmask01;
-    simdjson_inline simd_bitmask64_builder<3> next(simd8<bool> val2) {
-      return { bitmask01, val2.to_bitmask() };
-    }
-  }; // struct simd_bitmask64_builder<2>
-
-  template<>
-  struct simd_bitmask64_builder<1> {
-    const int bitmask0;
-    simdjson_inline simd_bitmask64_builder<2> next(simd8<bool> val1) {
-      return { uint32_t(bitmask0) | (uint32_t(val1.to_bitmask()) << 16) };
-    }
-  }; // struct simd_bitmask64_builder<1>
-
-  template<>
-  struct simd_bitmask64_builder<0> {
-    simdjson_inline simd_bitmask64_builder<1> next(simd8<bool> val0) {
-      return { val0.to_bitmask() };
-    }
-  }; // struct simd_bitmask64_builder<0>
-
   template<typename T>
   struct simd8x64 {
     static constexpr int NUM_CHUNKS = 64 / sizeof(simd8<T>);
@@ -338,11 +293,11 @@ namespace simd {
     }
 
     simdjson_inline uint64_t to_bitmask() const {
-      return simd_bitmask64_builder<0>()
-        .next(this->chunks[0])
-        .next(this->chunks[1])
-        .next(this->chunks[2])
-        .next(this->chunks[3]);
+      uint64_t r0 = uint32_t(this->chunks[0].to_bitmask() );
+      uint64_t r1 =          this->chunks[1].to_bitmask() ;
+      uint64_t r2 =          this->chunks[2].to_bitmask() ;
+      uint64_t r3 =          this->chunks[3].to_bitmask() ;
+      return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48);
     }
 
     simdjson_inline uint64_t eq(const T m) const {

singleheader/simdjson.cpp

@@ -1,4 +1,4 @@
-/* auto-generated on 2024-08-01 09:31:50 -0400. Do not edit! */
+/* auto-generated on 2024-08-26 09:37:03 -0400. Do not edit! */
 /* including simdjson.cpp:  */
 /* begin file simdjson.cpp */
 #define SIMDJSON_SRC_SIMDJSON_CPP
@@ -332,6 +332,8 @@ double from_chars(const char *first, const char* end) noexcept;
 #define SIMDJSON_ISALIGNED_N(ptr, n) (((uintptr_t)(ptr) & ((n)-1)) == 0)
 
 #if SIMDJSON_REGULAR_VISUAL_STUDIO
+  // We could use [[deprecated]] but it requires C++14
+  #define simdjson_deprecated __declspec(deprecated)
 
   #define simdjson_really_inline __forceinline
   #define simdjson_never_inline __declspec(noinline)
@@ -370,6 +372,8 @@ double from_chars(const char *first, const char* end) noexcept;
   #define SIMDJSON_POP_DISABLE_UNUSED_WARNINGS
 
 #else // SIMDJSON_REGULAR_VISUAL_STUDIO
+  // We could use [[deprecated]] but it requires C++14
+  #define simdjson_deprecated __attribute__((deprecated))
 
   #define simdjson_really_inline inline __attribute__((always_inline))
   #define simdjson_never_inline inline __attribute__((noinline))

src/generic/stage1/amalgamated.h

@@ -1,7 +1,6 @@
 // Stuff other things depend on
 #include <generic/stage1/base.h>
 #include <generic/stage1/buf_block_reader.h>
-#include <generic/stage1/simd_reducer.h>
 #include <generic/stage1/json_escape_scanner.h>
 #include <generic/stage1/json_string_scanner.h>
 #include <generic/stage1/utf8_lookup4_algorithm.h>

src/generic/stage1/json_structural_indexer.h

@@ -239,7 +239,7 @@ simdjson_inline void json_structural_indexer::step<64>(const uint8_t *block, buf
 simdjson_inline void json_structural_indexer::next(const simd::simd8x64<uint8_t>& in, const json_block& block, size_t idx) {
   uint64_t unescaped = in.lteq(0x1F);
 #if SIMDJSON_UTF8VALIDATION
-  checker.next(in);
+  checker.check_next_input(in);
 #endif
   indexer.write(uint32_t(idx-64), prev_structurals); // Output *last* iteration's structurals to the parser
   prev_structurals = block.structural_start();
@@ -343,7 +343,8 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa
         return EMPTY;
     }
   }
-  return std::move(checker).eof();
+  checker.check_eof();
+  return checker.errors();
 }
 
 } // namespace stage1

src/generic/stage1/simd_reducer.h

@@ -1,80 +0,0 @@
-#ifndef SIMDJSON_SRC_GENERIC_STAGE1_SIMD_REDUCER_H
-
-#ifndef SIMDJSON_CONDITIONAL_INCLUDE
-#define SIMDJSON_SRC_GENERIC_STAGE1_SIMD_REDUCER_H
-#include <generic/stage1/base.h>
-#endif // SIMDJSON_CONDITIONAL_INCLUDE
-
-namespace simdjson {
-namespace SIMDJSON_IMPLEMENTATION {
-namespace {
-namespace simd {
-
-/** Incrementally performs a reduce OR operation to accumulate an error. */
-template <int N>
-struct simd_or_reducer;
-
-template<> struct simd_or_reducer<8> {
-  simd8<uint8_t> error01234567;
-  operator simd8<uint8_t>() && { return error01234567; }
-};
-template<> struct simd_or_reducer<7> {
-  simd8<uint8_t> error0123;
-  simd8<uint8_t> error45;
-  simd8<uint8_t> error6;
-  simd_or_reducer<8> next(simd8<uint8_t> error7) && { return { error0123 | error45 | error6 | error7 }; }
-  operator simd8<uint8_t>() && { return error0123 | error45 | error6; }
-};
-template<> struct simd_or_reducer<6> {
-  simd8<uint8_t>    error0123;
-  simd8<uint8_t>    error45;
-  // simd8<uint8_t> error_;
-  simd_or_reducer<7> next(simd8<uint8_t> error6) && { return { error0123, error45, error6 }; }
-  operator simd8<uint8_t>() && { return error0123 | error45; }
-};
-template<> struct simd_or_reducer<5> {
-  simd8<uint8_t>    error0123;
-  // simd8<uint8_t> error__;
-  simd8<uint8_t>    error4;
-  simd_or_reducer<6> next(simd8<uint8_t> error5) && { return { error0123, error4 | error5 }; }
-  operator simd8<uint8_t>() && { return error0123 | error4; }
-};
-template<> struct simd_or_reducer<4> {
-  simd8<uint8_t>    error0123;
-  // simd8<uint8_t> error__;
-  // simd8<uint8_t> error_;
-  operator simd8<uint8_t>() && { return error0123; }
-};
-template<> struct simd_or_reducer<3> {
-  // simd8<uint8_t> error____;
-  simd8<uint8_t>    error01;
-  simd8<uint8_t>    error2;
-  simd_or_reducer<4> next(simd8<uint8_t> error3) && { return { error01 | error2 | error3 }; }
-  operator simd8<uint8_t>() && { return error01 | error2; }
-};
-template<> struct simd_or_reducer<2> {
-  // simd8<uint8_t> error____;
-  simd8<uint8_t>    error01;
-  // simd8<uint8_t> error_;
-  simd_or_reducer<3> next(simd8<uint8_t> error2) && { return { error01, error2 }; }
-  operator simd8<uint8_t>() && { return error01; }
-};
-template<> struct simd_or_reducer<1> {
-  // simd8<uint8_t> error____;
-  // simd8<uint8_t> error__;
-  simd8<uint8_t>    error0;
-  simd_or_reducer<2> next(simd8<uint8_t> error1) && { return { error0 | error1 }; }
-  operator simd8<uint8_t>() && { return error0; }
-};
-template<> struct simd_or_reducer<0> {
-  simd8<uint8_t> prev_error{};
-  simd_or_reducer<1> next(simd8<uint8_t> error0) && { return { prev_error | error0 }; }
-  operator simd8<uint8_t>() && { return prev_error; }
-};
-
-} // namespace simd
-} // unnamed namespace
-} // namespace SIMDJSON_IMPLEMENTATION
-} // namespace simdjson
-
-#endif // SIMDJSON_SRC_GENERIC_STAGE1_SIMD_REDUCER_H

src/generic/stage1/utf8_lookup4_algorithm.h

@@ -4,7 +4,6 @@
 #define SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H
 #include <generic/stage1/base.h>
 #include <generic/dom_parser_implementation.h>
-#include <generic/stage1/simd_reducer.h>
 #endif // SIMDJSON_CONDITIONAL_INCLUDE
 
 namespace simdjson {
@@ -12,84 +11,15 @@ namespace SIMDJSON_IMPLEMENTATION {
 namespace {
 namespace utf8_validation {
 
-  using namespace simd;
+using namespace simd;
 
-  // At its height, this will keep 5 registers around.
-  template <int N>
-  struct simd_utf8_checker {
-    /**
-     * The current error.
-     */
-    simd_or_reducer<N> error;
-    /**
-     * Whether the previous input had incomplete UTF-8 characters at the end.
-     */
-    simd8<uint8_t> prev_incomplete;
-
-    /**
-     * Check the next simd input block.
-     */
-    simdjson_inline simd_utf8_checker<N+1> next(simd8<uint8_t> input, simd8<uint8_t> prev_input) &&;
-
-    /**
-     * Used to wrap back around to the beginning, starting with error and prev_incomplete
-     * from the end of this block
-     */
-    simdjson_inline simd_utf8_checker<0> next_cycle() && {
-      return { std::move(error), std::move(prev_incomplete) };
-    }
-
-    /**
-     * Extracts the validation error from the block.
-     */
-    simdjson_inline error_code eof() && {
-      if ((prev_incomplete | std::move(error)).any_bits_set_anywhere()) {
-        return error_code::UTF8_ERROR;
-      } else {
-        return error_code::SUCCESS;
-      }
-    }
-
-  private:
-
-    simdjson_inline simd8<uint8_t> check_utf8_bytes(simd8<uint8_t> input, simd8<uint8_t> prev_input) const;
-    simdjson_inline simd8<uint8_t> is_incomplete(simd8<uint8_t> input) const;
-    simdjson_inline simd8<uint8_t> check_special_cases(simd8<uint8_t> input, simd8<uint8_t> prev1) const;
-    simdjson_inline simd8<uint8_t> check_multibyte_lengths(simd8<uint8_t> input, simd8<uint8_t> prev_input, simd8<uint8_t> sc) const;
-  };
-
-  template <int N>
-  simdjson_inline simd_utf8_checker<N+1> simd_utf8_checker<N>::next(simd8<uint8_t> input, simd8<uint8_t> prev_input) && {
-    if (simdjson_likely(input.is_ascii())) {
-      return {
-        std::move(this->error).next(this->prev_incomplete),
-        simd8<uint8_t>::zero()
-      };
-    } else {
-      return {
-        std::move(this->error).next(check_utf8_bytes(input, prev_input)),
-        is_incomplete(input)
-      };
-    }
-  }
-
-  template <int N>
-  simdjson_inline simd8<uint8_t> simd_utf8_checker<N>::check_utf8_bytes(simd8<uint8_t> input, simd8<uint8_t> prev_input) const {
-    // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ lead bytes
-    // (2, 3, 4-byte leads become large positive numbers instead of small negative numbers)
-    simd8<uint8_t> prev1 = input.prev<1>(prev_input);
-    simd8<uint8_t> sc = check_special_cases(input, prev1);
-    return check_multibyte_lengths(input, prev_input, sc);
-  }
-
-  template <int N>
-  simdjson_inline simd8<uint8_t> simd_utf8_checker<N>::check_special_cases(simd8<uint8_t> input, simd8<uint8_t> prev1) const {
-    // Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII)
-    // Bit 1 = Too Long (ASCII followed by continuation)
-    // Bit 2 = Overlong 3-byte
-    // Bit 4 = Surrogate
-    // Bit 5 = Overlong 2-byte
-    // Bit 7 = Two Continuations
+  simdjson_inline simd8<uint8_t> check_special_cases(const simd8<uint8_t> input, const simd8<uint8_t> prev1) {
+// Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII)
+// Bit 1 = Too Long (ASCII followed by continuation)
+// Bit 2 = Overlong 3-byte
+// Bit 4 = Surrogate
+// Bit 5 = Overlong 2-byte
+// Bit 7 = Two Continuations
     constexpr const uint8_t TOO_SHORT   = 1<<0; // 11______ 0_______
                                                 // 11______ 11______
     constexpr const uint8_t TOO_LONG    = 1<<1; // 0_______ 10______
@@ -173,13 +103,8 @@ namespace utf8_validation {
     );
     return (byte_1_high & byte_1_low & byte_2_high);
   }
-
-  template <int N>
-  simdjson_inline simd8<uint8_t> simd_utf8_checker<N>::check_multibyte_lengths(
-    simd8<uint8_t> input,
-    simd8<uint8_t> prev_input,
-    simd8<uint8_t> sc
-  ) const {
+  simdjson_inline simd8<uint8_t> check_multibyte_lengths(const simd8<uint8_t> input,
+      const simd8<uint8_t> prev_input, const simd8<uint8_t> sc) {
     simd8<uint8_t> prev2 = input.prev<2>(prev_input);
     simd8<uint8_t> prev3 = input.prev<3>(prev_input);
     simd8<uint8_t> must23 = must_be_2_3_continuation(prev2, prev3);
@@ -191,8 +116,7 @@ namespace utf8_validation {
   // Return nonzero if there are incomplete multibyte characters at the end of the block:
   // e.g. if there is a 4-byte character, but it's 3 bytes from the end.
   //
-  template <int N>
-  simdjson_inline simd8<uint8_t> simd_utf8_checker<N>::is_incomplete(simd8<uint8_t> input) const {
+  simdjson_inline simd8<uint8_t> is_incomplete(const simd8<uint8_t> input) {
     // If the previous input's last 3 bytes match this, they're too short (they ended at EOF):
     // ... 1111____ 111_____ 11______
 #if SIMDJSON_IMPLEMENTATION_ICELAKE
@@ -220,43 +144,59 @@ namespace utf8_validation {
 
   struct utf8_checker {
     // If this is nonzero, there has been a UTF-8 error.
-    simd_utf8_checker<0> checker;
-    simd8<uint8_t> prev_input;
+    simd8<uint8_t> error;
+    // The last input we received
+    simd8<uint8_t> prev_input_block;
+    // Whether the last input we received was incomplete (used for ASCII fast path)
+    simd8<uint8_t> prev_incomplete;
 
-    simdjson_inline void next(const simd8x64<uint8_t>& input) & {
-      // you might think that a for-loop would work, but under Visual Studio, it is not good enough.
-      static_assert(
-        (simd8x64<uint8_t>::NUM_CHUNKS == 1) ||
-        (simd8x64<uint8_t>::NUM_CHUNKS == 2) ||
-        (simd8x64<uint8_t>::NUM_CHUNKS == 4),
-        "We support one, two or four chunks per 64-byte block."
-      );
-
-      SIMDJSON_IF_CONSTEXPR (simd8x64<uint8_t>::NUM_CHUNKS == 1) {
-        this->checker = std::move(this->checker)
-          .next(input.chunks[0], this->prev_input)
-          .next_cycle();
-      } else SIMDJSON_IF_CONSTEXPR (simd8x64<uint8_t>::NUM_CHUNKS == 2) {
-        this->checker = std::move(this->checker)
-          .next(input.chunks[0], this->prev_input)
-          .next(input.chunks[1], input.chunks[0])
-          .next_cycle();
-      } else SIMDJSON_IF_CONSTEXPR (simd8x64<uint8_t>::NUM_CHUNKS == 4) {
-        this->checker = std::move(this->checker)
-          .next(input.chunks[0], this->prev_input)
-          .next(input.chunks[1], input.chunks[0])
-          .next(input.chunks[2], input.chunks[1])
-          .next(input.chunks[3], input.chunks[2])
-          .next_cycle();
-      }
-      this->prev_input = input.chunks[simd8x64<uint8_t>::NUM_CHUNKS-1];
+    //
+    // Check whether the current bytes are valid UTF-8.
+    //
+    simdjson_inline void check_utf8_bytes(const simd8<uint8_t> input, const simd8<uint8_t> prev_input) {
+      // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ lead bytes
+      // (2, 3, 4-byte leads become large positive numbers instead of small negative numbers)
+      simd8<uint8_t> prev1 = input.prev<1>(prev_input);
+      simd8<uint8_t> sc = check_special_cases(input, prev1);
+      this->error |= check_multibyte_lengths(input, prev_input, sc);
     }
 
-    simdjson_inline error_code eof() && {
-      // The only problem that can happen at EOF is that a multibyte character is too short
-      // or a byte value too large in the last bytes: check_special_cases only checks for bytes
-      // too large in the first of two bytes.
-      return std::move(this->checker).eof() ? error_code::UTF8_ERROR : error_code::SUCCESS;
+    // The only problem that can happen at EOF is that a multibyte character is too short
+    // or a byte value too large in the last bytes: check_special_cases only checks for bytes
+    // too large in the first of two bytes.
+    simdjson_inline void check_eof() {
+      // If the previous block had incomplete UTF-8 characters at the end, an ASCII block can't
+      // possibly finish them.
+      this->error |= this->prev_incomplete;
+    }
+
+    simdjson_inline void check_next_input(const simd8x64<uint8_t>& input) {
+      if(simdjson_likely(is_ascii(input))) {
+        this->error |= this->prev_incomplete;
+      } else {
+        // you might think that a for-loop would work, but under Visual Studio, it is not good enough.
+        static_assert((simd8x64<uint8_t>::NUM_CHUNKS == 1)
+                ||(simd8x64<uint8_t>::NUM_CHUNKS == 2)
+                || (simd8x64<uint8_t>::NUM_CHUNKS == 4),
+                "We support one, two or four chunks per 64-byte block.");
+        SIMDJSON_IF_CONSTEXPR (simd8x64<uint8_t>::NUM_CHUNKS == 1) {
+          this->check_utf8_bytes(input.chunks[0], this->prev_input_block);
+        } else SIMDJSON_IF_CONSTEXPR (simd8x64<uint8_t>::NUM_CHUNKS == 2) {
+          this->check_utf8_bytes(input.chunks[0], this->prev_input_block);
+          this->check_utf8_bytes(input.chunks[1], input.chunks[0]);
+        } else SIMDJSON_IF_CONSTEXPR (simd8x64<uint8_t>::NUM_CHUNKS == 4) {
+          this->check_utf8_bytes(input.chunks[0], this->prev_input_block);
+          this->check_utf8_bytes(input.chunks[1], input.chunks[0]);
+          this->check_utf8_bytes(input.chunks[2], input.chunks[1]);
+          this->check_utf8_bytes(input.chunks[3], input.chunks[2]);
+        }
+        this->prev_incomplete = is_incomplete(input.chunks[simd8x64<uint8_t>::NUM_CHUNKS-1]);
+        this->prev_input_block = input.chunks[simd8x64<uint8_t>::NUM_CHUNKS-1];
+      }
+    }
+    // do not forget to call check_eof!
+    simdjson_inline error_code errors() {
+      return this->error.any_bits_set_anywhere() ? error_code::UTF8_ERROR : error_code::SUCCESS;
     }
 
   }; // struct utf8_checker

src/generic/stage1/utf8_validator.h

@@ -21,16 +21,16 @@ bool generic_validate_utf8(const uint8_t * input, size_t length) {
     buf_block_reader<64> reader(input, length);
     while (reader.has_full_block()) {
       simd::simd8x64<uint8_t> in(reader.full_block());
-      c.next(in);
+      c.check_next_input(in);
       reader.advance();
     }
-    // TODO parse the remainder in sub-blocks (16/32 bytes at a time depending on arch)
     uint8_t block[64]{};
     reader.get_remainder(block);
     simd::simd8x64<uint8_t> in(block);
-    c.next(in);
+    c.check_next_input(in);
     reader.advance();
-    return std::move(c).eof() == error_code::SUCCESS;
+    c.check_eof();
+    return c.errors() == error_code::SUCCESS;
 }
 
 bool generic_validate_utf8(const char * input, size_t length) {

tests/ondemand/CMakeLists.txt

@@ -28,6 +28,7 @@ add_cpp_test(ondemand_to_string              LABELS ondemand acceptance per_impl
 add_cpp_test(ondemand_twitter_tests          LABELS ondemand acceptance per_implementation)
 add_cpp_test(ondemand_wrong_type_error_tests LABELS ondemand acceptance per_implementation)
 add_cpp_test(ondemand_iterate_many_csv       LABELS ondemand acceptance per_implementation)
+add_cpp_test(ondemand_extract_tests          LABELS ondemand acceptance per_implementation)
 if(NOT SIMDJSON_SANITIZE)
   add_cpp_test(ondemand_cacheline              LABELS ondemand acceptance per_implementation)
 endif()

tests/ondemand/ondemand_extract_tests.cpp

@@ -0,0 +1,116 @@
+#include "simdjson.h"
+#include "test_ondemand.h"
+
+using namespace simdjson;
+
+namespace multi_get_tests {
+#ifdef __cpp_concepts
+struct Car {
+  std::string_view make;
+  std::string_view model;
+  std::int64_t year = 0;
+  struct wheel_size_type {
+    std::int64_t front = 0;
+    std::int64_t back = 0;
+  } wheels;
+
+  static simdjson_result<Car> create(auto& value) {
+    using ondemand::to;
+    using ondemand::sub;
+
+    simdjson::ondemand::object obj;
+    auto error = value.get_object().get(obj);
+    if (error) {
+      return error;
+    }
+    Car car{};
+    // Instead of this:
+    //     for (auto field : obj) {
+    //       simdjson::ondemand::raw_json_string key;
+    //       error = field.key().get(key);
+    //       if (error) {
+    //         return error;
+    //       }
+    //       if (key == "make") {
+    //         error = field.value().get_string(car.make);
+    //         if (error) {
+    //           return error;
+    //         }
+    //       } else if (key == "model") {
+    //         error = field.value().get_string(car.model);
+    //         if (error) {
+    //           return error;
+    //         }
+    //       } else if (key == "year") {
+    //         error = field.value().get(car.year);
+    //         if (error) {
+    //           return error;
+    //         }
+    //     }
+    //
+    // we can do this now:
+    error = obj.extract(
+      to{"wheels", sub{
+        to{"front", car.wheels.front},
+        to{"back", car.wheels.back},
+      }},
+      to{"make", car.make},
+      to{"model", car.model},
+      to{"year", [&car](auto val) {
+        car.year = val;
+      }});
+    if (error) {
+      return error;
+    }
+    return car;
+  }
+};
+
+bool car_example() {
+  TEST_START();
+  simdjson::padded_string json =
+      R"( [ { "make": "Toyota", "model": "Camry",  "year": 2018,
+       "tire_pressure": [ 40.1, 39.9 ], "wheels": { "front": 10, "back": 10 } },
+  { "make": "Kia",    "model": "Soul",   "year": 2012,
+       "tire_pressure": [ 30.1, 31.0 ], "wheels": { "front": 10, "back": 10 } },
+  { "make": "Toyota", "model": "Tercel", "year": 1999,
+       "tire_pressure": [ 29.8, 30.0 ], "wheels": { "front": 10, "back": 10 } }
+])"_padded;
+
+  simdjson::ondemand::parser parser;
+  simdjson::ondemand::document doc = parser.iterate(json);
+  for (auto val : doc) {
+    auto car = Car::create(val);
+    ASSERT_EQUAL(car.error(), SUCCESS);
+    Car const c(car.value());
+    if (c.make != "Toyota" && c.make != "Kia") {
+      return false;
+    }
+    if (c.model != "Camry" && c.model != "Soul" && c.model != "Tercel") {
+      return false;
+    }
+    if (c.year != 2018 && c.year != 2012 && c.year != 1999) {
+      return false;
+    }
+    if (c.wheels.front != 10 || c.wheels.back != 10) {
+      return false;
+    }
+  }
+
+  TEST_SUCCEED();
+}
+
+#endif
+bool run() {
+  return
+#ifdef __cpp_concepts
+      car_example() &&
+#endif
+      true;
+}
+
+} // namespace multi_get_tests
+
+int main(int argc, char *argv[]) {
+  return test_main(argc, argv, multi_get_tests::run);
+}