Optimize reflection-based deserialization with hybrid algorithm

- Add threshold-based algorithm selection (N<=8: per-field, N>=9: single-pass)
- Per-field lookup leverages simdjson's SIMD-optimized find_field_unordered
- Single-pass uses bitmask tracking, first-char filter, and early termination
- Add fields_in_order<T> trait for user-defined ordered field optimization

Performance improvement on benchmarks:
- Twitter: +5.3% (4167 -> 4389 MB/s)
- CITM: +14.6% (2600 -> 2994 MB/s)

Threshold=8 is based on crossover analysis: at N=9 fields, per-field lookup
causes ~35% regression due to quadratic scan cost vs single-pass linear cost.

CMakeLists.txt

@@ -12,7 +12,7 @@ endif()
 project(
     simdjson
     # The version number is modified by tools/release.py
-    VERSION 4.3.1
+    VERSION 4.2.4
     DESCRIPTION "Parsing gigabytes of JSON per second"
     HOMEPAGE_URL "https://simdjson.org/"
     LANGUAGES CXX C
@@ -29,8 +29,8 @@ string(
 # ---- Options, variables ----
 
 # These version numbers are modified by tools/release.py
-set(SIMDJSON_LIB_VERSION "30.0.0" CACHE STRING "simdjson library version")
-set(SIMDJSON_LIB_SOVERSION "30" CACHE STRING "simdjson library soversion")
+set(SIMDJSON_LIB_VERSION "29.0.0" CACHE STRING "simdjson library version")
+set(SIMDJSON_LIB_SOVERSION "29" CACHE STRING "simdjson library soversion")
 
 option(SIMDJSON_BUILD_STATIC_LIB "Build simdjson_static library along with simdjson (only makes sense if BUILD_SHARED_LIBS=ON)" OFF)
 if(SIMDJSON_BUILD_STATIC_LIB AND NOT BUILD_SHARED_LIBS)

Doxyfile

@@ -38,7 +38,7 @@ PROJECT_NAME           = simdjson
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = "4.3.1"
+PROJECT_NUMBER         = "4.2.4"
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a

benchmark/static_reflect/citm_catalog_benchmark/benchmark_parsing_citm.cpp

@@ -8,6 +8,7 @@
 #include <simdjson.h>
 #include <string>
 #include "citm_catalog_data.h"
+// NOTE: citm_traits.h NOT included because CITM JSON fields are NOT in struct order
 #include "nlohmann_citm_catalog_data.h"
 #include "../benchmark_utils/benchmark_helper.h"
 

benchmark/static_reflect/twitter_benchmark/benchmark_parsing_twitter.cpp

@@ -8,6 +8,7 @@
 #include <simdjson.h>
 #include <string>
 #include "twitter_data.h"
+// NOTE: twitter_traits.h NOT included because Twitter JSON fields are NOT in struct order
 #include "nlohmann_twitter_data.h"
 #include "../benchmark_utils/benchmark_helper.h"
 #ifdef SIMDJSON_COMPETITION_RAPIDJSON

include/simdjson/generic/builder/json_string_builder-inl.h

@@ -146,7 +146,7 @@ simdjson_inline bool fast_needs_escaping(std::string_view view) {
 #endif
 
 // Scalar fallback for finding next quotable character
-SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline size_t
+SIMDJSON_CONSTEXPR_LAMBDA inline size_t
 find_next_json_quotable_character_scalar(const std::string_view view,
                                          size_t location) noexcept {
   for (auto pos = view.begin() + location; pos != view.end(); ++pos) {
@@ -258,7 +258,7 @@ find_next_json_quotable_character(const std::string_view view,
   return find_next_json_quotable_character_scalar(view, current);
 }
 #else
-SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline size_t
+SIMDJSON_CONSTEXPR_LAMBDA inline size_t
 find_next_json_quotable_character(const std::string_view view,
                                   size_t location) noexcept {
   return find_next_json_quotable_character_scalar(view, location);
@@ -277,7 +277,7 @@ SIMDJSON_CONSTEXPR_LAMBDA static std::string_view control_chars[] = {
 // control characters (U+0000 through U+001F). There are two-character sequence
 // escape representations of some popular characters:
 // \", \\, \b, \f, \n, \r, \t.
-SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline void escape_json_char(char c, char *&out) {
+SIMDJSON_CONSTEXPR_LAMBDA void escape_json_char(char c, char *&out) {
   if (c == '"') {
     memcpy(out, "\\\"", 2);
     out += 2;
@@ -336,7 +336,7 @@ simdjson_inline bool string_builder::capacity_check(size_t upcoming_bytes) {
     return true;
   }
   // check for overflow, most of the time there is no overflow
-  if (simdjson_unlikely(position + upcoming_bytes < position)) {
+  if (simdjson_likely(position + upcoming_bytes < position)) {
     return false;
   }
   // We will rarely get here.

include/simdjson/generic/ondemand/std_deserialize.h

@@ -266,6 +266,13 @@ constexpr bool user_defined_type = (std::is_class_v<T>
 && !std::is_same_v<T, std::string> && !std::is_same_v<T, std::string_view> && !concepts::optional_type<T> &&
 !concepts::appendable_containers<T>);
 
+// Trait to indicate JSON fields arrive in struct declaration order.
+// Users can specialize this for their types to enable faster ordered field lookup.
+// Example:
+//   template<> struct simdjson::fields_in_order<MyStruct> : std::true_type {};
+template <typename T>
+struct fields_in_order : std::false_type {};
+
 
 template <typename T, typename ValT>
   requires(user_defined_type<T> && std::is_class_v<T>)
@@ -276,25 +283,112 @@ error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept {
   } else {
     SIMDJSON_TRY(val.get_object().get(obj));
   }
-  template for (constexpr auto mem : std::define_static_array(std::meta::nonstatic_data_members_of(^^T, std::meta::access_context::unchecked()))) {
-    if constexpr (!std::meta::is_const(mem) && std::meta::is_public(mem)) {
-      constexpr std::string_view key = std::define_static_string(std::meta::identifier_of(mem));
-      if constexpr (concepts::optional_type<decltype(out.[:mem:])>) {
-        // for optional members, it's ok if the key is missing
-        auto error = obj[key].get(out.[:mem:]);
-        if (error && error != NO_SUCH_FIELD) {
-          if(error == NO_SUCH_FIELD) {
-            out.[:mem:].reset();
-            continue;
+
+  // For ordered fields, use fast ordered lookup
+  if constexpr (fields_in_order<T>::value) {
+    template for (constexpr auto mem : std::define_static_array(std::meta::nonstatic_data_members_of(^^T, std::meta::access_context::unchecked()))) {
+      if constexpr (!std::meta::is_const(mem) && std::meta::is_public(mem)) {
+        constexpr std::string_view key = std::define_static_string(std::meta::identifier_of(mem));
+        if constexpr (concepts::optional_type<decltype(out.[:mem:])>) {
+          error_code error = obj.find_field(key).get(out.[:mem:]);
+          if (error && error != NO_SUCH_FIELD) {
+            return error;
           }
-          return error;
+        } else {
+          SIMDJSON_TRY(obj.find_field(key).get(out.[:mem:]));
         }
-      } else {
-        // for non-optional members, the key must be present
-        SIMDJSON_TRY(obj[key].get(out.[:mem:]));
       }
     }
-  };
+    return simdjson::SUCCESS;
+  }
+
+  // Algorithm selection based on struct size:
+  // - Per-field lookup: calls find_field_unordered() for each struct field (N scans)
+  // - Single-pass: iterates JSON once, checking each field against all struct fields
+  //
+  // Crossover analysis: per-field does N object scans, single-pass does 1 scan with N compares.
+  // Empirically, per-field wins for N<=8, single-pass wins dramatically for N>=9.
+  // At N=9, per-field causes ~35% performance regression vs single-pass.
+  constexpr size_t num_fields = []() consteval {
+    size_t count = 0;
+    template for (constexpr auto mem : std::define_static_array(std::meta::nonstatic_data_members_of(^^T, std::meta::access_context::unchecked()))) {
+      if constexpr (!std::meta::is_const(mem) && std::meta::is_public(mem)) {
+        ++count;
+      }
+    }
+    return count;
+  }();
+
+  if constexpr (num_fields <= 8) {
+    // Per-field lookup: efficient for small structs, leverages simdjson's SIMD-optimized find_field
+    template for (constexpr auto mem : std::define_static_array(std::meta::nonstatic_data_members_of(^^T, std::meta::access_context::unchecked()))) {
+      if constexpr (!std::meta::is_const(mem) && std::meta::is_public(mem)) {
+        constexpr std::string_view key = std::define_static_string(std::meta::identifier_of(mem));
+        if constexpr (concepts::optional_type<decltype(out.[:mem:])>) {
+          error_code error = obj.find_field_unordered(key).get(out.[:mem:]);
+          if (error && error != NO_SUCH_FIELD) {
+            return error;
+          }
+        } else {
+          SIMDJSON_TRY(obj.find_field_unordered(key).get(out.[:mem:]));
+        }
+      }
+    }
+  } else {
+    // Single-pass with all optimizations for larger structs
+    constexpr uint64_t required_mask = []() consteval {
+      uint64_t mask = 0;
+      size_t idx = 0;
+      template for (constexpr auto mem : std::define_static_array(std::meta::nonstatic_data_members_of(^^T, std::meta::access_context::unchecked()))) {
+        if constexpr (!std::meta::is_const(mem) && std::meta::is_public(mem)) {
+          if constexpr (!concepts::optional_type<decltype(std::declval<T&>().[:mem:])>) {
+            mask |= (uint64_t(1) << idx);
+          }
+          ++idx;
+        }
+      }
+      return mask;
+    }();
+
+    uint64_t found_mask = 0;
+    constexpr uint64_t all_fields_mask = (num_fields < 64) ? ((uint64_t(1) << num_fields) - 1) : ~uint64_t(0);
+
+    for (auto field : obj) {
+      if (found_mask == all_fields_mask) break;
+
+      SIMDJSON_IMPLEMENTATION::ondemand::raw_json_string json_key = field.key();
+      const char* raw = json_key.raw();
+      char first_char = raw[0];
+      bool matched = false;
+      size_t field_idx = 0;
+
+      template for (constexpr auto mem : std::define_static_array(std::meta::nonstatic_data_members_of(^^T, std::meta::access_context::unchecked()))) {
+        if constexpr (!std::meta::is_const(mem) && std::meta::is_public(mem)) {
+          constexpr std::string_view expected_key = std::define_static_string(std::meta::identifier_of(mem));
+          if (!matched && first_char == expected_key[0] && !(found_mask & (uint64_t(1) << field_idx))) {
+            if (json_key.unsafe_is_equal(expected_key)) {
+              auto field_val = field.value();
+              error_code err = field_val.get(out.[:mem:]);
+              if (err) {
+                if constexpr (concepts::optional_type<decltype(out.[:mem:])>) {
+                  if (err != INCORRECT_TYPE) { return err; }
+                } else {
+                  return err;
+                }
+              }
+              found_mask |= (uint64_t(1) << field_idx);
+              matched = true;
+            }
+          }
+          ++field_idx;
+        }
+      }
+    }
+
+    if ((found_mask & required_mask) != required_mask) {
+      return NO_SUCH_FIELD;
+    }
+  }
   return simdjson::SUCCESS;
 }
 

include/simdjson/simdjson_version.h

@@ -4,7 +4,7 @@
 #define SIMDJSON_SIMDJSON_VERSION_H
 
 /** The version of simdjson being used (major.minor.revision) */
-#define SIMDJSON_VERSION "4.3.1"
+#define SIMDJSON_VERSION "4.2.4"
 
 namespace simdjson {
 enum {
@@ -15,11 +15,11 @@ enum {
   /**
    * The minor version (major.MINOR.revision) of simdjson being used.
    */
-  SIMDJSON_VERSION_MINOR = 3,
+  SIMDJSON_VERSION_MINOR = 2,
   /**
    * The revision (major.minor.REVISION) of simdjson being used.
    */
-  SIMDJSON_VERSION_REVISION = 1
+  SIMDJSON_VERSION_REVISION = 4
 };
 } // namespace simdjson
 

tests/compilation_failure_tests/CMakeLists.txt

@@ -25,8 +25,3 @@ if (SIMDJSON_EXCEPTIONS)
   add_dual_compile_test(dangling_parser_parse_padstring)
   add_dual_compile_test(unsafe_parse_many)
 endif()
-
-if(NOT BUILD_SHARED_LIBS)
-  # We only check that it builds
-  add_subdirectory(multiple_include)
-endif()

tests/compilation_failure_tests/multiple_include/CMakeLists.txt

@@ -1,5 +0,0 @@
-add_library(mylib mylib.cpp)
-target_link_libraries(mylib PUBLIC simdjson::simdjson)
-target_include_directories(mylib PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
-add_executable(myexe main.cpp)
-target_link_libraries(myexe PRIVATE mylib)

tests/compilation_failure_tests/multiple_include/main.cpp

@@ -1,22 +0,0 @@
-#include "mylib.h"
-#include <simdjson.h>
-#include <iostream>
-#include <memory>
-
-int main() {
-    simdjson::padded_string json = R"([ 1, 2, 3, 4 ])"_padded;
-    simdjson::padded_string minified = minify_json(json);
-    std::cout << "Minified: " << std::string_view(minified) << std::endl;
-
-    // Also directly use minify
-    size_t length = json.size();
-    std::unique_ptr<char[]> buffer{new char[length]};
-    size_t new_length{};
-    auto error = simdjson::minify(json.data(), length, buffer.get(), new_length);
-    if (error) {
-        std::cerr << "Error: " << simdjson::error_message(error) << std::endl;
-    } else {
-        std::cout << "Direct minified: " << std::string(buffer.get(), new_length) << std::endl;
-    }
-    return 0;
-}

tests/compilation_failure_tests/multiple_include/mylib.cpp

@@ -1,14 +0,0 @@
-#include "mylib.h"
-#include <simdjson.h>
-#include <memory>
-
-simdjson::padded_string minify_json(const simdjson::padded_string& json) {
-    size_t length = json.size();
-    std::unique_ptr<char[]> buffer{new char[length]};
-    size_t new_length{};
-    auto error = simdjson::minify(json.data(), length, buffer.get(), new_length);
-    if (error) {
-        return simdjson::padded_string();
-    }
-    return simdjson::padded_string(std::string_view(buffer.get(), new_length));
-}

tests/compilation_failure_tests/multiple_include/mylib.h

@@ -1,9 +0,0 @@
-#ifndef MYLIB_H
-#define MYLIB_H
-
-#include <simdjson.h>
-#include <string>
-
-simdjson::padded_string minify_json(const simdjson::padded_string& json);
-
-#endif // MYLIB_H