Fix wrong branch hint and add missing consteval guard in extract_from (#2609)

Two fixes to the ablation logic:

1. Fix incorrect branch hint in capacity_check() overflow detection
   (json_string_builder-inl.h:365): simdjson_likely was used on the
   overflow check `position + upcoming_bytes < position`, but overflow
   is rare so this should be simdjson_unlikely. The comment itself says
   "most of the time there is no overflow". This was causing the branch
   predictor to optimize for the wrong path.

2. Add missing ABLATION_NO_CONSTEVAL guard in extract_from()
   (json_builder.h:332): consteval_to_quoted_escaped was being used
   without an ablation guard, for consistency with all other call sites.

Updated ablation results with both fixes:
| Configuration    | CITM (MB/s) | Impact | Twitter (MB/s) | Impact |
|------------------|-------------|--------|----------------|--------|
| Baseline         | 3001.10     | -      | 5651.89        | -      |
| NO_BRANCH_HINTS  | 2443.67     | -19%   | 4651.78        | -18%   |
| NO_SIMD_ESCAPING | 2904.34     | -3%    | 1403.36        | -75%   |
| NO_CONSTEVAL     | 1796.05     | -40%   | 3542.77        | -37%   |

ablation.sh

@@ -0,0 +1,23 @@
+CXX=clang++ cmake -B buildreflect -D SIMDJSON_STATIC_REFLECTION=ON -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_ABLATION_NO_BRANCH_HINTS=ON
+cmake --build buildreflect --target benchmark_serialization_citm_catalog benchmark_serialization_twitter
+./buildreflect/benchmark/static_reflect/citm_catalog_benchmark/benchmark_serialization_citm_catalog
+./buildreflect/benchmark/static_reflect/twitter_benchmark/benchmark_serialization_twitter
+
+
+CXX=clang++ cmake -B buildreflecthints -D SIMDJSON_STATIC_REFLECTION=ON -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_ABLATION_NO_BRANCH_HINTS=ON
+cmake --build buildreflecthints --target benchmark_serialization_citm_catalog benchmark_serialization_twitter
+./buildreflecthints/benchmark/static_reflect/citm_catalog_benchmark/benchmark_serialization_citm_catalog
+./buildreflecthints/benchmark/static_reflect/twitter_benchmark/benchmark_serialization_twitter
+
+
+CXX=clang++ cmake -B buildreflectescaping -D SIMDJSON_STATIC_REFLECTION=ON -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_ABLATION_NO_SIMD_ESCAPING=ON
+cmake --build buildreflectescaping --target benchmark_serialization_citm_catalog benchmark_serialization_twitter
+./buildreflectescaping/benchmark/static_reflect/citm_catalog_benchmark/benchmark_serialization_citm_catalog
+./buildreflectescaping/benchmark/static_reflect/twitter_benchmark/benchmark_serialization_twitter
+
+
+
+CXX=clang++ cmake -B buildreflectconsteval -D SIMDJSON_STATIC_REFLECTION=ON -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_ABLATION_NO_CONSTEVAL=ON
+cmake --build buildreflectconsteval --target benchmark_serialization_citm_catalog benchmark_serialization_twitter
+./buildreflectconsteval/benchmark/static_reflect/citm_catalog_benchmark/benchmark_serialization_citm_catalog
+./buildreflectconsteval/benchmark/static_reflect/twitter_benchmark/benchmark_serialization_twitter

benchmark/static_reflect/citm_catalog_benchmark/benchmark_parsing_citm.cpp

@@ -8,7 +8,6 @@
 #include <simdjson.h>
 #include <string>
 #include "citm_catalog_data.h"
-// NOTE: citm_traits.h NOT included because CITM JSON fields are NOT in struct order
 #include "nlohmann_citm_catalog_data.h"
 #include "../benchmark_utils/benchmark_helper.h"
 

benchmark/static_reflect/twitter_benchmark/benchmark_parsing_twitter.cpp

@@ -8,7 +8,6 @@
 #include <simdjson.h>
 #include <string>
 #include "twitter_data.h"
-// NOTE: twitter_traits.h NOT included because Twitter JSON fields are NOT in struct order
 #include "nlohmann_twitter_data.h"
 #include "../benchmark_utils/benchmark_helper.h"
 #ifdef SIMDJSON_COMPETITION_RAPIDJSON

cmake/developer-options.cmake

@@ -240,6 +240,36 @@ else()
   add_compile_definitions(SIMDJSON_UTF8VALIDATION=1)
 endif()
 
+option(
+    SIMDJSON_ABLATION_NO_BRANCH_HINTS
+    "Disable branch hints for ablation testing."
+    OFF
+)
+if(SIMDJSON_ABLATION_NO_BRANCH_HINTS)
+  add_compile_definitions(ABLATION_NO_BRANCH_HINTS=1)
+  message(STATUS "ABLATION_NO_BRANCH_HINTS enabled")
+endif()
+
+option(
+    SIMDJSON_ABLATION_NO_SIMD_ESCAPING
+    "Disable SIMD escaping for ablation testing."
+    OFF
+)
+if(SIMDJSON_ABLATION_NO_SIMD_ESCAPING)
+  add_compile_definitions(ABLATION_NO_SIMD_ESCAPING=1)
+  message(STATUS "ABLATION_NO_SIMD_ESCAPING enabled")
+endif()
+
+option(
+    SIMDJSON_ABLATION_NO_CONSTEVAL
+    "Disable consteval for ablation testing."
+    OFF
+)
+if(SIMDJSON_ABLATION_NO_CONSTEVAL)
+  add_compile_definitions(ABLATION_NO_CONSTEVAL=1)
+  message(STATUS "ABLATION_NO_CONSTEVAL enabled")
+endif()
+
 include(CheckSymbolExists)
 check_symbol_exists(fork unistd.h HAVE_POSIX_FORK)
 check_symbol_exists(wait sys/wait.h HAVE_POSIX_WAIT)

include/simdjson/generic/builder/json_builder.h

@@ -94,8 +94,12 @@ constexpr void atom(string_builder &b, const T &t) {
   template for (constexpr auto dm : std::define_static_array(std::meta::nonstatic_data_members_of(^^T, std::meta::access_context::unchecked()))) {
     if (i != 0)
       b.append(',');
+#if ABLATION_NO_CONSTEVAL
+    b.escape_and_append_with_quotes(std::meta::identifier_of(dm));
+#else
     constexpr auto key = std::define_static_string(constevalutil::consteval_to_quoted_escaped(std::meta::identifier_of(dm)));
     b.append_raw(key);
+#endif
     b.append(':');
     atom(b, t.[:dm:]);
     i++;
@@ -132,9 +136,13 @@ void atom(string_builder &b, const T &e) {
 #if SIMDJSON_STATIC_REFLECTION
   constexpr auto enumerators = std::define_static_array(std::meta::enumerators_of(^^T));
   template for (constexpr auto enum_val : enumerators) {
-    constexpr auto enum_str = std::define_static_string(constevalutil::consteval_to_quoted_escaped(std::meta::identifier_of(enum_val)));
     if (e == [:enum_val:]) {
+#if ABLATION_NO_CONSTEVAL
+      b.escape_and_append_with_quotes(std::meta::identifier_of(enum_val));
+#else
+      constexpr auto enum_str = std::define_static_string(constevalutil::consteval_to_quoted_escaped(std::meta::identifier_of(enum_val)));
       b.append_raw(enum_str);
+#endif
       return;
     }
   };
@@ -229,8 +237,12 @@ void append(string_builder &b, const Z &z) {
   template for (constexpr auto dm : std::define_static_array(std::meta::nonstatic_data_members_of(^^Z, std::meta::access_context::unchecked()))) {
     if (i != 0)
       b.append(',');
+#if ABLATION_NO_CONSTEVAL
+    b.escape_and_append_with_quotes(std::meta::identifier_of(dm));
+#else
     constexpr auto key = std::define_static_string(constevalutil::consteval_to_quoted_escaped(std::meta::identifier_of(dm)));
     b.append_raw(key);
+#endif
     b.append(':');
     atom(b, z.[:dm:]);
     i++;
@@ -317,8 +329,12 @@ void extract_from(string_builder &b, const T &obj) {
         first = false;
 
         // Serialize the key
+#if ABLATION_NO_CONSTEVAL
+        b.escape_and_append_with_quotes(std::meta::identifier_of(mem));
+#else
         constexpr auto quoted_key = std::define_static_string(constevalutil::consteval_to_quoted_escaped(std::meta::identifier_of(mem)));
         b.append_raw(quoted_key);
+#endif
         b.append(':');
 
         // Serialize the value

include/simdjson/generic/builder/json_string_builder-inl.h

@@ -146,7 +146,7 @@ simdjson_inline bool fast_needs_escaping(std::string_view view) {
 #endif
 
 // Scalar fallback for finding next quotable character
-SIMDJSON_CONSTEXPR_LAMBDA inline size_t
+SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline size_t
 find_next_json_quotable_character_scalar(const std::string_view view,
                                          size_t location) noexcept {
   for (auto pos = view.begin() + location; pos != view.end(); ++pos) {
@@ -258,7 +258,7 @@ find_next_json_quotable_character(const std::string_view view,
   return find_next_json_quotable_character_scalar(view, current);
 }
 #else
-SIMDJSON_CONSTEXPR_LAMBDA inline size_t
+SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline size_t
 find_next_json_quotable_character(const std::string_view view,
                                   size_t location) noexcept {
   return find_next_json_quotable_character_scalar(view, location);
@@ -277,7 +277,7 @@ SIMDJSON_CONSTEXPR_LAMBDA static std::string_view control_chars[] = {
 // control characters (U+0000 through U+001F). There are two-character sequence
 // escape representations of some popular characters:
 // \", \\, \b, \f, \n, \r, \t.
-SIMDJSON_CONSTEXPR_LAMBDA void escape_json_char(char c, char *&out) {
+SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline void escape_json_char(char c, char *&out) {
   if (c == '"') {
     memcpy(out, "\\\"", 2);
     out += 2;
@@ -295,9 +295,22 @@ SIMDJSON_CONSTEXPR_LAMBDA void escape_json_char(char c, char *&out) {
 // written. Uses SIMD position finding to locate quotable characters efficiently.
 inline size_t write_string_escaped(const std::string_view input, char *out) {
   size_t mysize = input.size();
+#if ABLATION_NO_SIMD_ESCAPING
+  auto naive_find_next_json_quotable_character = [](std::string_view input, size_t start) -> size_t {
+    for (size_t i = start; i < input.size(); ++i) {
+        auto c = static_cast<unsigned char>(input[i]);
+        if (c == '"' || c == '\\' || c < 32) return i;
+    }
+    return input.size();
+  };
+#endif
 
   // Use SIMD position finder directly - it returns mysize if no escape needed
+#if ABLATION_NO_SIMD_ESCAPING
+  size_t location = naive_find_next_json_quotable_character(input, 0);
+#else
   size_t location = find_next_json_quotable_character(input, 0);
+#endif
   if (location == mysize) {
     // Fast path: no escaping needed
     memcpy(out, input.data(), input.size());
@@ -310,7 +323,11 @@ inline size_t write_string_escaped(const std::string_view input, char *out) {
   escape_json_char(input[location], out);
   location += 1;
   while (location < mysize) {
+    #if ABLATION_NO_SIMD_ESCAPING
+    size_t newlocation = naive_find_next_json_quotable_character(input, location);
+    #else
     size_t newlocation = find_next_json_quotable_character(input, location);
+    #endif
     memcpy(out, input.data() + location, newlocation - location);
     out += newlocation - location;
     location = newlocation;
@@ -332,13 +349,23 @@ simdjson_inline bool string_builder::capacity_check(size_t upcoming_bytes) {
   // We use the convention that when is_valid is false, then the capacity and
   // the position are 0.
   // Most of the time, this function will return true.
+#if ABLATION_NO_BRANCH_HINTS
+  if (upcoming_bytes <= capacity - position) {
+    return true;
+  }
+  // check for overflow, most of the time there is no overflow
+  if (position + upcoming_bytes < position) {
+    return false;
+  }
+#else
   if (simdjson_likely(upcoming_bytes <= capacity - position)) {
     return true;
   }
   // check for overflow, most of the time there is no overflow
-  if (simdjson_likely(position + upcoming_bytes < position)) {
+  if (simdjson_unlikely(position + upcoming_bytes < position)) {
     return false;
   }
+#endif
   // We will rarely get here.
   grow_buffer((std::max)(capacity * 2, position + upcoming_bytes));
   // If the buffer allocation failed, we set is_valid to false.

include/simdjson/generic/ondemand/std_deserialize.h

@@ -266,13 +266,6 @@ constexpr bool user_defined_type = (std::is_class_v<T>
 && !std::is_same_v<T, std::string> && !std::is_same_v<T, std::string_view> && !concepts::optional_type<T> &&
 !concepts::appendable_containers<T>);
 
-// Trait to indicate JSON fields arrive in struct declaration order.
-// Users can specialize this for their types to enable faster ordered field lookup.
-// Example:
-//   template<> struct simdjson::fields_in_order<MyStruct> : std::true_type {};
-template <typename T>
-struct fields_in_order : std::false_type {};
-
 
 template <typename T, typename ValT>
   requires(user_defined_type<T> && std::is_class_v<T>)
@@ -283,112 +276,25 @@ error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept {
   } else {
     SIMDJSON_TRY(val.get_object().get(obj));
   }
-
-  // For ordered fields, use fast ordered lookup
-  if constexpr (fields_in_order<T>::value) {
-    template for (constexpr auto mem : std::define_static_array(std::meta::nonstatic_data_members_of(^^T, std::meta::access_context::unchecked()))) {
-      if constexpr (!std::meta::is_const(mem) && std::meta::is_public(mem)) {
-        constexpr std::string_view key = std::define_static_string(std::meta::identifier_of(mem));
-        if constexpr (concepts::optional_type<decltype(out.[:mem:])>) {
-          error_code error = obj.find_field(key).get(out.[:mem:]);
-          if (error && error != NO_SUCH_FIELD) {
-            return error;
+  template for (constexpr auto mem : std::define_static_array(std::meta::nonstatic_data_members_of(^^T, std::meta::access_context::unchecked()))) {
+    if constexpr (!std::meta::is_const(mem) && std::meta::is_public(mem)) {
+      constexpr std::string_view key = std::define_static_string(std::meta::identifier_of(mem));
+      if constexpr (concepts::optional_type<decltype(out.[:mem:])>) {
+        // for optional members, it's ok if the key is missing
+        auto error = obj[key].get(out.[:mem:]);
+        if (error && error != NO_SUCH_FIELD) {
+          if(error == NO_SUCH_FIELD) {
+            out.[:mem:].reset();
+            continue;
           }
-        } else {
-          SIMDJSON_TRY(obj.find_field(key).get(out.[:mem:]));
+          return error;
         }
+      } else {
+        // for non-optional members, the key must be present
+        SIMDJSON_TRY(obj[key].get(out.[:mem:]));
       }
     }
-    return simdjson::SUCCESS;
-  }
-
-  // Algorithm selection based on struct size:
-  // - Per-field lookup: calls find_field_unordered() for each struct field (N scans)
-  // - Single-pass: iterates JSON once, checking each field against all struct fields
-  //
-  // Crossover analysis: per-field does N object scans, single-pass does 1 scan with N compares.
-  // Empirically, per-field wins for N<=8, single-pass wins dramatically for N>=9.
-  // At N=9, per-field causes ~35% performance regression vs single-pass.
-  constexpr size_t num_fields = []() consteval {
-    size_t count = 0;
-    template for (constexpr auto mem : std::define_static_array(std::meta::nonstatic_data_members_of(^^T, std::meta::access_context::unchecked()))) {
-      if constexpr (!std::meta::is_const(mem) && std::meta::is_public(mem)) {
-        ++count;
-      }
-    }
-    return count;
-  }();
-
-  if constexpr (num_fields <= 8) {
-    // Per-field lookup: efficient for small structs, leverages simdjson's SIMD-optimized find_field
-    template for (constexpr auto mem : std::define_static_array(std::meta::nonstatic_data_members_of(^^T, std::meta::access_context::unchecked()))) {
-      if constexpr (!std::meta::is_const(mem) && std::meta::is_public(mem)) {
-        constexpr std::string_view key = std::define_static_string(std::meta::identifier_of(mem));
-        if constexpr (concepts::optional_type<decltype(out.[:mem:])>) {
-          error_code error = obj.find_field_unordered(key).get(out.[:mem:]);
-          if (error && error != NO_SUCH_FIELD) {
-            return error;
-          }
-        } else {
-          SIMDJSON_TRY(obj.find_field_unordered(key).get(out.[:mem:]));
-        }
-      }
-    }
-  } else {
-    // Single-pass with all optimizations for larger structs
-    constexpr uint64_t required_mask = []() consteval {
-      uint64_t mask = 0;
-      size_t idx = 0;
-      template for (constexpr auto mem : std::define_static_array(std::meta::nonstatic_data_members_of(^^T, std::meta::access_context::unchecked()))) {
-        if constexpr (!std::meta::is_const(mem) && std::meta::is_public(mem)) {
-          if constexpr (!concepts::optional_type<decltype(std::declval<T&>().[:mem:])>) {
-            mask |= (uint64_t(1) << idx);
-          }
-          ++idx;
-        }
-      }
-      return mask;
-    }();
-
-    uint64_t found_mask = 0;
-    constexpr uint64_t all_fields_mask = (num_fields < 64) ? ((uint64_t(1) << num_fields) - 1) : ~uint64_t(0);
-
-    for (auto field : obj) {
-      if (found_mask == all_fields_mask) break;
-
-      SIMDJSON_IMPLEMENTATION::ondemand::raw_json_string json_key = field.key();
-      const char* raw = json_key.raw();
-      char first_char = raw[0];
-      bool matched = false;
-      size_t field_idx = 0;
-
-      template for (constexpr auto mem : std::define_static_array(std::meta::nonstatic_data_members_of(^^T, std::meta::access_context::unchecked()))) {
-        if constexpr (!std::meta::is_const(mem) && std::meta::is_public(mem)) {
-          constexpr std::string_view expected_key = std::define_static_string(std::meta::identifier_of(mem));
-          if (!matched && first_char == expected_key[0] && !(found_mask & (uint64_t(1) << field_idx))) {
-            if (json_key.unsafe_is_equal(expected_key)) {
-              auto field_val = field.value();
-              error_code err = field_val.get(out.[:mem:]);
-              if (err) {
-                if constexpr (concepts::optional_type<decltype(out.[:mem:])>) {
-                  if (err != INCORRECT_TYPE) { return err; }
-                } else {
-                  return err;
-                }
-              }
-              found_mask |= (uint64_t(1) << field_idx);
-              matched = true;
-            }
-          }
-          ++field_idx;
-        }
-      }
-    }
-
-    if ((found_mask & required_mask) != required_mask) {
-      return NO_SUCH_FIELD;
-    }
-  }
+  };
   return simdjson::SUCCESS;
 }
 

tests/compilation_failure_tests/CMakeLists.txt

@@ -25,3 +25,8 @@ if (SIMDJSON_EXCEPTIONS)
   add_dual_compile_test(dangling_parser_parse_padstring)
   add_dual_compile_test(unsafe_parse_many)
 endif()
+
+if(NOT BUILD_SHARED_LIBS)
+  # We only check that it builds
+  add_subdirectory(multiple_include)
+endif()

tests/compilation_failure_tests/multiple_include/CMakeLists.txt

@@ -0,0 +1,5 @@
+add_library(mylib mylib.cpp)
+target_link_libraries(mylib PUBLIC simdjson::simdjson)
+target_include_directories(mylib PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
+add_executable(myexe main.cpp)
+target_link_libraries(myexe PRIVATE mylib)

tests/compilation_failure_tests/multiple_include/main.cpp

@@ -0,0 +1,22 @@
+#include "mylib.h"
+#include <simdjson.h>
+#include <iostream>
+#include <memory>
+
+int main() {
+    simdjson::padded_string json = R"([ 1, 2, 3, 4 ])"_padded;
+    simdjson::padded_string minified = minify_json(json);
+    std::cout << "Minified: " << std::string_view(minified) << std::endl;
+
+    // Also directly use minify
+    size_t length = json.size();
+    std::unique_ptr<char[]> buffer{new char[length]};
+    size_t new_length{};
+    auto error = simdjson::minify(json.data(), length, buffer.get(), new_length);
+    if (error) {
+        std::cerr << "Error: " << simdjson::error_message(error) << std::endl;
+    } else {
+        std::cout << "Direct minified: " << std::string(buffer.get(), new_length) << std::endl;
+    }
+    return 0;
+}

tests/compilation_failure_tests/multiple_include/mylib.cpp

@@ -0,0 +1,14 @@
+#include "mylib.h"
+#include <simdjson.h>
+#include <memory>
+
+simdjson::padded_string minify_json(const simdjson::padded_string& json) {
+    size_t length = json.size();
+    std::unique_ptr<char[]> buffer{new char[length]};
+    size_t new_length{};
+    auto error = simdjson::minify(json.data(), length, buffer.get(), new_length);
+    if (error) {
+        return simdjson::padded_string();
+    }
+    return simdjson::padded_string(std::string_view(buffer.get(), new_length));
+}

tests/compilation_failure_tests/multiple_include/mylib.h

@@ -0,0 +1,9 @@
+#ifndef MYLIB_H
+#define MYLIB_H
+
+#include <simdjson.h>
+#include <string>
+
+simdjson::padded_string minify_json(const simdjson::padded_string& json);
+
+#endif // MYLIB_H