bump

extractor PR with clangcl tweaks

.github/workflows/cifuzz.yml

@@ -17,7 +17,7 @@ jobs:
         fuzz-seconds: 600
         dry-run: false
     - name: Upload Crash
-      uses: actions/upload-artifact@v1
+      uses: actions/upload-artifact@v3
       if: failure() && steps.build.outcome == 'success'
       with:
         name: artifacts

.github/workflows/fix-trailing-whitespace.yml

@@ -24,7 +24,7 @@ jobs:
              echo "no trailing whitespace found, good!"
           fi
       - name: Archive whitespace patch
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         if: always()
         with:
           name: whitespace-patch

.github/workflows/vs17-clang-ci.yml

@@ -13,15 +13,15 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - {gen: Visual Studio 17 2022, arch: x64, build_type: Debug}
-          - {gen: Visual Studio 17 2022, arch: x64, build_type: Release}
-          - {gen: Visual Studio 17 2022, arch: x64, build_type: RelWithDebInfo}
+          - {gen: Visual Studio 17 2022, arch: x64, build_type: Debug, cxx: 17}
+          - {gen: Visual Studio 17 2022, arch: x64, build_type: Debug, cxx: 20}
+          - {gen: Visual Studio 17 2022, arch: x64, build_type: Release, cxx: 17}
     steps:
     - name: checkout
       uses: actions/checkout@v4
     - name: Configure
       run: |
-        cmake -G "${{matrix.gen}}" -A ${{matrix.arch}}  -T ClangCL -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -B build
+        cmake -G "${{matrix.gen}}" -A ${{matrix.arch}} -DSIMDJSON_CXX_STANDARD=${{matrix.cxx}} -T ClangCL -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -B build
     - name: Build
       run: cmake --build build --config ${{matrix.build_type}} --verbose
     - name: Run tests

benchmark/partial_tweets/simdjson_ondemand.h

@@ -45,7 +45,6 @@ BENCHMARK_TEMPLATE(partial_tweets, simdjson_ondemand)->UseManualTime();
 
 
 #if SIMDJSON_SUPPORTS_EXTRACT
-
 using namespace simdjson::ondemand;
 
 struct simdjson_ondemand_extract {

include/simdjson/compiler_check.h

@@ -50,4 +50,18 @@
 #endif
 #endif
 
+#ifdef __has_include
+#if __has_include(<version>)
+#include <version>
+#endif
+#endif
+
+
+
+#if defined(__cpp_concepts)
+#define SIMDJSON_SUPPORTS_EXTRACT 1
+#else
+#define SIMDJSON_SUPPORTS_EXTRACT 0
+#endif
+
 #endif // SIMDJSON_COMPILER_CHECK_H

include/simdjson/fallback/bitmanipulation.h

@@ -9,7 +9,7 @@ namespace simdjson {
 namespace fallback {
 namespace {
 
-#if defined(_MSC_VER) && !defined(_M_ARM64) && !defined(_M_X64)
+#if SIMDJSON_REGULAR_VISUAL_STUDIO && !defined(_M_ARM64) && !defined(_M_X64)
 static inline unsigned char _BitScanForward64(unsigned long* ret, uint64_t x) {
   unsigned long x0 = (unsigned long)x, top, bottom;
   _BitScanForward(&top, (unsigned long)(x >> 32));
@@ -28,7 +28,7 @@ static unsigned char _BitScanReverse64(unsigned long* ret, uint64_t x) {
 
 /* result might be undefined when input_num is zero */
 simdjson_inline int leading_zeroes(uint64_t input_num) {
-#ifdef _MSC_VER
+#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
   unsigned long leading_zero = 0;
   // Search the mask data from most significant bit (MSB)
   // to least significant bit (LSB) for a set bit (1).
@@ -38,7 +38,7 @@ simdjson_inline int leading_zeroes(uint64_t input_num) {
     return 64;
 #else
   return __builtin_clzll(input_num);
-#endif// _MSC_VER
+#endif// SIMDJSON_REGULAR_VISUAL_STUDIO
 }
 
 } // unnamed namespace

include/simdjson/generic/ondemand/object-inl.h

@@ -16,27 +16,34 @@ namespace SIMDJSON_IMPLEMENTATION {
 namespace ondemand {
 
 
-#ifdef SIMDJSON_SUPPORTS_EXTRACT
+#if SIMDJSON_SUPPORTS_EXTRACT
 
+
+#if SIMDJSON_REGULAR_VISUAL_STUDIO
 template <endpoint ...Funcs>
-simdjson_inline error_code object::extract(Funcs&&... endpoints)
-#ifndef _MSC_VER // msvc thinks noexcept is not the same in definition
- noexcept((nothrow_endpoint<Funcs> && ...))
-#endif
-{
-  raw_json_string field_key;
-  error_code error = SUCCESS;
-  for(auto pair : *this) {
-    if (error = pair.key().get(field_key); error) {
-      break;
-    }
-    std::ignore = ((field_key.unsafe_is_equal(endpoints.key()) ? (error = endpoints(pair.value())) == SUCCESS : true) && ...);
+simdjson_inline error_code object::extract(Funcs&&... endpoints) {
+  return iter.on_field_raw([&, eps = std::make_tuple(std::forward<Funcs>(endpoints)...)](auto field_key, error_code& error) mutable {
+    std::apply([&](auto &...endpoints) {
+       std::ignore = ((field_key.unsafe_is_equal(endpoints.key()) ? (error = endpoints(value(iter.child()))) == SUCCESS : true) && ...);
+    }, eps);
     if (error) {
-      break;
+      return true;
     }
-  }
-  return error;
+    return false;
+  });
 }
+#else
+template <endpoint ...Funcs>
+simdjson_inline error_code object::extract(Funcs&&... endpoints) noexcept((nothrow_endpoint<Funcs> && ...)) {
+  return iter.on_field_raw([&](auto field_key, error_code& error) noexcept((nothrow_endpoint<Funcs> && ...)) {
+    std::ignore = ((field_key.unsafe_is_equal(endpoints.key()) ? (error = endpoints(value(iter.child()))) == SUCCESS : true) && ...);
+    if (error) {
+      return true;
+    }
+    return false;
+  });
+}
+#endif
 
 template <typename T>
 struct to {

include/simdjson/generic/ondemand/object.h

@@ -11,8 +11,7 @@ namespace simdjson {
 namespace SIMDJSON_IMPLEMENTATION {
 namespace ondemand {
 
-#if defined(__cpp_concepts) && defined(__cpp_consteval)
-#define SIMDJSON_SUPPORTS_EXTRACT 1
+#if SIMDJSON_SUPPORTS_EXTRACT
 
 template <typename T>
 concept endpoint = std::is_invocable_r_v<error_code, T, simdjson_result<value>> &&
@@ -79,14 +78,14 @@ public:
   /** @overload simdjson_inline simdjson_result<value> find_field(std::string_view key) & noexcept; */
   simdjson_inline simdjson_result<value> find_field(std::string_view key) && noexcept;
 
-#ifdef SIMDJSON_SUPPORTS_EXTRACT
+#if SIMDJSON_SUPPORTS_EXTRACT
   /**
    * Extract all the fields in one go
    * Funcs are invocables that take a simdjson_result<value> as input.
    */
   template <endpoint ...Funcs>
   simdjson_inline error_code extract(Funcs&&... endpoints)
-#ifndef _MSC_VER // msvc thinks noexcept is not the same in definition
+#ifndef SIMDJSON_REGULAR_VISUAL_STUDIO // msvc thinks noexcept is not the same in definition
  noexcept((nothrow_endpoint<Funcs> && ...))
 #endif
  ;

include/simdjson/generic/ondemand/value_iterator-inl.h

@@ -1,3 +1,4 @@
+#include <type_traits>
 #ifndef SIMDJSON_GENERIC_ONDEMAND_VALUE_ITERATOR_INL_H
 
 #ifndef SIMDJSON_CONDITIONAL_INCLUDE
@@ -109,6 +110,108 @@ simdjson_warn_unused simdjson_inline simdjson_result<bool> value_iterator::has_n
   }
 }
 
+template <typename Func>
+simdjson_warn_unused simdjson_inline error_code value_iterator::on_field_raw(Func&& func)
+#ifdef __cpp_lib_is_invocable
+  noexcept(std::is_nothrow_invocable_r_v<bool, Func, raw_json_string, error_code&>)
+#else
+  noexcept(false)
+#endif
+{
+#ifdef __cpp_lib_is_invocable
+  static_assert(std::is_invocable_r_v<bool, Func, raw_json_string, error_code&>, "Invalid function provided.");
+#endif
+
+  error_code error = SUCCESS;
+  bool has_value;
+  //
+  // Initially, the object can be in one of a few different places:
+  //
+  // 1. The start of the object, at the first field:
+  //
+  //    ```
+  //    { "a": [ 1, 2 ], "b": [ 3, 4 ] }
+  //      ^ (depth 2, index 1)
+  //    ```
+  if (at_first_field()) {
+    has_value = true;
+
+  //
+  // 2. When a previous search did not yield a value or the object is empty:
+  //
+  //    ```
+  //    { "a": [ 1, 2 ], "b": [ 3, 4 ] }
+  //                                     ^ (depth 0)
+  //    { }
+  //        ^ (depth 0, index 2)
+  //    ```
+  //
+  } else if (!is_open()) {
+#if SIMDJSON_DEVELOPMENT_CHECKS
+    // If we're past the end of the object, we're being iterated out of order.
+    // Note: this is not perfect detection. It's possible the user is inside some other object; if so,
+    // this object iterator will blithely scan that object for fields.
+    if (_json_iter->depth() < depth() - 1) { return OUT_OF_ORDER_ITERATION; }
+#endif
+    return EMPTY;
+
+  // 3. When a previous search found a field or an iterator yielded a value:
+  //
+  //    ```
+  //    // When a field was not fully consumed (or not even touched at all)
+  //    { "a": [ 1, 2 ], "b": [ 3, 4 ] }
+  //           ^ (depth 2)
+  //    // When a field was fully consumed
+  //    { "a": [ 1, 2 ], "b": [ 3, 4 ] }
+  //                   ^ (depth 1)
+  //    // When the last field was fully consumed
+  //    { "a": [ 1, 2 ], "b": [ 3, 4 ] }
+  //                                   ^ (depth 1)
+  //    ```
+  //
+  } else {
+    if ((error = skip_child() )) { abandon(); return error; }
+    if ((error = has_next_field().get(has_value) )) { abandon(); return error; }
+#if SIMDJSON_DEVELOPMENT_CHECKS
+    if (_json_iter->start_position(_depth) != start_position()) { return OUT_OF_ORDER_ITERATION; }
+#endif
+  }
+  while (has_value) {
+    // Get the key and colon, stopping at the value.
+    raw_json_string actual_key;
+    // size_t max_key_length = _json_iter->peek_length() - 2; // -2 for the two quotes
+    // Note: _json_iter->peek_length() - 2 might overflow if _json_iter->peek_length() < 2.
+    // field_key() advances the pointer and checks that '"' is found (corresponding to a key).
+    // The depth is left unchanged by field_key().
+    if ((error = field_key().get(actual_key) )) { abandon(); return error; };
+    // field_value() will advance and check that we find a ':' separating the
+    // key and the value. It will also increment the depth by one.
+    if ((error = field_value() )) { abandon(); return error; }
+    // If it matches, stop and return
+    // We could do it this way if we wanted to allow arbitrary
+    // key content (including escaped quotes).
+    //if (actual_key.unsafe_is_equal(max_key_length, key)) {
+    // Instead we do the following which may trigger buffer overruns if the
+    // user provides an adversarial key (containing a well placed unescaped quote
+    // character and being longer than the number of bytes remaining in the JSON
+    // input).
+    if (func(actual_key, error)) {
+      break;
+    }
+
+    // The call to skip_child is meant to skip over the value corresponding to the key.
+    // After skip_child(), we are right before the next comma (',') or the final brace ('}').
+    SIMDJSON_TRY( skip_child() ); // Skip the value entirely
+    // The has_next_field() advances the pointer and check that either ',' or '}' is found.
+    // It returns true if ',' is found, false otherwise. If anything other than ',' or '}' is found,
+    // then we are in error and we abort.
+    if ((error = has_next_field().get(has_value) )) { abandon(); return error; }
+  }
+
+  // If the loop ended, we're out of fields to look at.
+  return error;
+}
+
 simdjson_warn_unused simdjson_inline simdjson_result<bool> value_iterator::find_field_raw(const std::string_view key) noexcept {
   error_code error;
   bool has_value;
@@ -1088,4 +1191,4 @@ simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value_iterato
 
 } // namespace simdjson
 
-#endif // SIMDJSON_GENERIC_ONDEMAND_VALUE_ITERATOR_INL_H
+#endif // SIMDJSON_GENERIC_ONDEMAND_VALUE_ITERATOR_INL_H

include/simdjson/generic/ondemand/value_iterator.h

@@ -6,6 +6,12 @@
 #include "simdjson/generic/implementation_simdjson_result_base.h"
 #endif // SIMDJSON_CONDITIONAL_INCLUDE
 
+#ifdef __has_include
+#if __has_include (<version>)
+#include <version>
+#endif
+#endif
+
 namespace simdjson {
 namespace SIMDJSON_IMPLEMENTATION {
 namespace ondemand {
@@ -198,6 +204,22 @@ public:
    */
   simdjson_warn_unused simdjson_inline simdjson_result<bool> find_field_raw(const std::string_view key) noexcept;
 
+
+  /**
+   * Runs Func on each key found.
+   * Almost same as `find_field_raw` but it runs `func` instead of checking the key ourselves.
+   *
+   * @param Func func(raw_json_string key, error_code& error) noexcept
+   */
+  template <typename Func>
+  simdjson_warn_unused simdjson_inline error_code on_field_raw(Func&& func)
+#ifdef __cpp_lib_is_invocable
+  noexcept(std::is_nothrow_invocable_r_v<bool, Func, raw_json_string, error_code&>)
+#else
+  noexcept(false)
+#endif
+  ;
+
   /**
    * Find the field with the given key without regard to order, and *without* unescaping.
    *
@@ -489,4 +511,4 @@ public:
 
 } // namespace simdjson
 
-#endif // SIMDJSON_GENERIC_ONDEMAND_VALUE_ITERATOR_H
+#endif // SIMDJSON_GENERIC_ONDEMAND_VALUE_ITERATOR_H