chore: add RelWithDebInfo to our CI tests (#2209)

* chore: add RelWithDebInfo to our tests

* fix

* chore: add various build types to ubuntu ci

.github/workflows/msys2-clang.yml

@@ -20,6 +20,9 @@ jobs:
           - msystem: "MINGW64"
             install: mingw-w64-x86_64-libxml2 mingw-w64-x86_64-cmake mingw-w64-x86_64-ninja mingw-w64-x86_64-clang
             type: Debug
+            - msystem: "MINGW64"
+            install: mingw-w64-x86_64-libxml2 mingw-w64-x86_64-cmake mingw-w64-x86_64-ninja mingw-w64-x86_64-clang
+            type: RelWithDebInfo
     env:
       CMAKE_GENERATOR: Ninja
 

.github/workflows/msys2.yml

@@ -22,6 +22,9 @@ jobs:
           - msystem: "MINGW64"
             install: mingw-w64-x86_64-cmake mingw-w64-x86_64-ninja mingw-w64-x86_64-gcc
             type: Debug
+            - msystem: "MINGW64"
+            install: mingw-w64-x86_64-cmake mingw-w64-x86_64-ninja mingw-w64-x86_64-gcc
+            type: RelWithDebInfo
     env:
       CMAKE_GENERATOR: Ninja
 

.github/workflows/ubuntu22-gcc13.yml

@@ -1,23 +0,0 @@
-name: Ubuntu 22.04 CI (GCC 13)
-
-on: [push, pull_request]
-
-jobs:
-  ubuntu-build:
-    if: >-
-      ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
-      ! contains(toJSON(github.event.commits.*.message), '[skip github]')
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/cache@v4
-        with:
-          path: dependencies/.cache
-          key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
-      - name: Use cmake
-        run: |
-          mkdir build &&
-          cd build &&
-          CXX=g++-13 cmake -DSIMDJSON_DEVELOPER_MODE=ON ..  &&
-          cmake --build .   &&
-          ctest --output-on-failure -LE explicitonly -j

.github/workflows/ubuntu24.yml

@@ -0,0 +1,25 @@
+name: Ubuntu 24.04 CI
+
+on: [push, pull_request]
+jobs:
+  ubuntu-build:
+    if: >-
+      ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
+      ! contains(toJSON(github.event.commits.*.message), '[skip github]')
+    runs-on: ubuntu-24.04
+    strategy:
+      matrix:
+        shared: [ON, OFF]
+        cxx: [g++-13, clang++-16]
+        sanitizer: [ON, OFF]
+        build_type: [RelWithDebInfo, Debug, Release]
+    steps:
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+      - name: Prepare
+        run: cmake -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_SANITIZE=${{matrix.sanitizer}} -DBUILD_SHARED_LIBS=${{matrix.shared}}  -B build
+        env:
+          CXX: ${{matrix.cxx}}
+      - name: Build
+        run: cmake --build build -j=2
+      - name: Test
+        run: ctest --output-on-failure --test-dir build

.github/workflows/vs17-ci.yml

@@ -13,10 +13,12 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - {gen: Visual Studio 17 2022, arch: Win32, shared: ON}
-          - {gen: Visual Studio 17 2022, arch: Win32, shared: OFF}
-          - {gen: Visual Studio 17 2022, arch: x64, shared: ON}
-          - {gen: Visual Studio 17 2022, arch: x64, shared: OFF}
+          - {gen: Visual Studio 17 2022, arch: Win32, shared: ON, build_type: Release}
+          - {gen: Visual Studio 17 2022, arch: Win32, shared: OFF, build_type: Release}
+          - {gen: Visual Studio 17 2022, arch: x64, shared: ON, build_type: Release}
+          - {gen: Visual Studio 17 2022, arch: x64, shared: OFF, build_type: Debug}
+          - {gen: Visual Studio 17 2022, arch: x64, shared: OFF, build_type: Release}
+          - {gen: Visual Studio 17 2022, arch: x64, shared: OFF, build_type: RelWithDebInfo}
     steps:
     - name: checkout
       uses: actions/checkout@v4
@@ -24,21 +26,15 @@ jobs:
       run: |
         cmake -G "${{matrix.gen}}" -A ${{matrix.arch}} -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -DBUILD_SHARED_LIBS=${{matrix.shared}} -B build
     - name: Build Debug
-      run: cmake --build build --config Debug --verbose
-    - name: Build Release
-      run: cmake --build build --config Release --verbose
-    - name: Run Release tests
+      run: cmake --build build --config ${{build_type}} --verbose
+    - name: Run tests
       run: |
         cd build
-        ctest -C Release  -LE explicitonly  --output-on-failure
-    - name: Run Debug tests
-      run: |
-        cd build
-        ctest -C Debug  -LE explicitonly  --output-on-failure
+        ctest -C ${{build_type}}  -LE explicitonly  --output-on-failure
     - name: Install
       run: |
-        cmake --install build --config Release
+        cmake --install build --config ${{build_type}}
     - name: Test Installation
       run: |
         cmake -G "${{matrix.gen}}" -A ${{matrix.arch}}  -B build_install_test tests/installation_tests/find
-        cmake --build build_install_test --config Release
+        cmake --build build_install_test --config ${{build_type}}

.github/workflows/vs17-clang-ci.yml

@@ -13,29 +13,25 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - {gen: Visual Studio 17 2022, arch: x64}
+          - {gen: Visual Studio 17 2022, arch: x64, build_type: Debug}
+          - {gen: Visual Studio 17 2022, arch: x64, build_type: Release}
+          - {gen: Visual Studio 17 2022, arch: x64, build_type: RelWithDebInfo}
     steps:
     - name: checkout
       uses: actions/checkout@v4
     - name: Configure
       run: |
         cmake -G "${{matrix.gen}}" -A ${{matrix.arch}}  -T ClangCL -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -B build
-    - name: Build Debug
-      run: cmake --build build --config Debug --verbose
-    - name: Build Release
-      run: cmake --build build --config Release --verbose
-    - name: Run Release tests
+    - name: Build
+      run: cmake --build build --config ${{matrix.build_type}} --verbose
+    - name: Run tests
       run: |
         cd build
-        ctest -C Release  -LE explicitonly  --output-on-failure
-    - name: Run Debug tests
-      run: |
-        cd build
-        ctest -C Debug  -LE explicitonly  --output-on-failure
+        ctest -C ${{matrix.build_type}}  -LE explicitonly  --output-on-failure
     - name: Install
       run: |
-        cmake --install build --config Release
+        cmake --install build --config ${{matrix.build_type}}
     - name: Test Installation
       run: |
         cmake -G "${{matrix.gen}}" -A ${{matrix.arch}}  -B build_install_test tests/installation_tests/find
-        cmake --build build_install_test --config Release
+        cmake --build build_install_test --config ${{matrix.build_type}}

CMakeLists.txt

@@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.14)
 project(
     simdjson
     # The version number is modified by tools/release.py
-    VERSION 3.9.2
+    VERSION 3.9.4
     DESCRIPTION "Parsing gigabytes of JSON per second"
     HOMEPAGE_URL "https://simdjson.org/"
     LANGUAGES CXX C

Doxyfile

@@ -38,7 +38,7 @@ PROJECT_NAME           = simdjson
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = "3.9.2"
+PROJECT_NUMBER         = "3.9.4"
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a

doc/basics.md

@@ -64,20 +64,23 @@ into your project. Then include it in your project with:
 using namespace simdjson; // optional
 ```
 
-You can compile with:
+Under most systems, you can compile with:
 
 ```
 c++ myproject.cpp simdjson.cpp
 ```
 
 Note:
+- We recommend that you use simdjson by copying the single-header `simdjson.h` file along with the source file `simdjson.cpp` directly in your project, as they are part of [every release](https://github.com/simdjson/simdjson/releases) as assets. In this manner, you only have to compile `simdjson.cpp` as any other source file: it works well in every development environment. However, you may also use simdjson as a git submodule ([example](https://github.com/simdjson/cmakedemo)), using FetchContent ([example](https://github.com/simdjson/cmake_demo_single_file)), with ExternalProject_Add ([example](https://github.com/simdjson/cmakedemo_externalproject)) or with CPM ([example](https://github.com/cpm-cmake/CPM.cmake/tree/master/examples/simdjson)).
 - Users on macOS and other platforms where default compilers do not provide C++11 compliant by default should request it with the appropriate flag (e.g., `c++ -std=c++11 myproject.cpp simdjson.cpp`).
 - The library relies on [runtime CPU detection](implementation-selection.md): avoid specifying an architecture at compile time (e.g., `-march-native`) if you want your binaries to run everywhere.
 
 Using simdjson with package managers
 ------------------
 
-You can install the simdjson library on your system or in your project using multiple package managers such as MSYS2, the conan package manager, vcpkg, brew, the apt package manager (debian-based Linux systems), the FreeBSD package manager (FreeBSD), and so on. [Visit our wiki for more details](https://github.com/simdjson/simdjson/wiki/Installing-simdjson-with-a-package-manager).
+You can install the simdjson library on your system or in your project using multiple package managers such as MSYS2, the conan package manager, vcpkg, brew, the apt package manager (debian-based Linux systems), the FreeBSD package manager (FreeBSD), and so on. E.g., [we provide an complete example with vcpkg](https://github.com/simdjson/simdjson-vcpkg) that works under Windows. [Visit our wiki for more details](https://github.com/simdjson/simdjson/wiki/Installing-simdjson-with-a-package-manager).
+
+
 
 The following Linux distributions provide simdjson packages: Alpine, RedHat, Rocky Linux, Debian, Fedora, and Ubuntu.
 
@@ -156,7 +159,8 @@ For efficiency reasons, simdjson requires a string with a few bytes (`simdjson::
 at the end, these bytes may be read but their content does not affect the parsing. In practice,
 it means that the JSON inputs should be stored in a memory region with `simdjson::SIMDJSON_PADDING`
 extra bytes at the end. You do not have to set these bytes to specific values though you may
-want to if you want to avoid runtime warnings with some sanitizers.
+want to if you want to avoid runtime warnings with some sanitizers. Advanced users may want to
+read the section Free Padding in [our performance notes](performance.md).
 
 The simdjson library offers a tree-like [API](https://en.wikipedia.org/wiki/API), which you can
 access by creating a `ondemand::parser` and calling the `iterate()` method. The iterate method
@@ -567,7 +571,7 @@ support for users who avoid exceptions. See [the simdjson error handling documen
   auto json = R"( { "test":{ "val1":1, "val2":2 } }   )"_padded;
   auto doc = parser.iterate(json);
   size_t count = doc.count_fields(); // requires simdjson 1.0 or better
-  std::cout << "Number of fields: " <<  new_count << std::endl; // Prints "Number of fields: 1"
+  std::cout << "Number of fields: " <<  count << std::endl; // Prints "Number of fields: 1"
   ```
   Similarly to `count_elements`, you should not let an object instance go out of scope before consuming it after calling
   the `count_fields` method. If you access an object inside a document, you can use the `count_fields` method as follow.
@@ -2211,12 +2215,39 @@ string representation.
 ```
 
 
+You can use `raw_json()` to capture the content of some JSON values as `std::string_view`
+instances which can be safely used later. The `std::string_view` instances point inside
+the original document and do not depend in any way on simdjson. In the following example,
+we store the `std::string_view` instances inside a `std::vector<std::string_view>` instance
+and print the out after the parsing is concluded:
+
+```cpp
+  padded_string json_padded = "{\"a\":[1,2,3], \"b\": 2, \"c\": \"hello\"}"_padded;
+  std::vector<std::string_view> fields;
+
+  ondemand::parser parser;
+  auto doc = parser.iterate(json_padded);
+  auto object = doc.get_object();
+  for (auto field : object) {
+    fields.push_back(field.value().raw_json());
+  }
+  // Output the fields
+  // Expected output:
+  // [1,2,3]
+  // 2
+  // "hello"
+  for (std::string_view field_ref : fields) {
+    std::cout << field_ref << std::endl;
+  }
+  ```
+
+
 Storing directly into an existing string instance
 -----------------------------------------------------
 
 The simdjson library favours  the use of `std::string_view` instances because
 it tends to lead to better performance due to causing fewer memory allocations.
-However, they are cases where you need to store a string result in a `std::string``
+However, they are cases where you need to store a string result in a `std::string`
 instance. You can do so with a templated version of the `to_string()` method which takes as
 a parameter a reference to a `std::string`.
 
@@ -2596,6 +2627,7 @@ Performance tips
 --------
 
 
+- Read [our performance notes](performance.md) for advanced topics.
 - The On Demand front-end works best when doing a single pass over the input: avoid calling `count_elements`, `rewind` and similar methods.
 - If you are familiar with assembly language, you may use the online tool godbolt to explore the compiled code. The following example may work: [https://godbolt.org/z/xE4GWs573](https://godbolt.org/z/xE4GWs573).
 - Given a field `field` in an object, calling `field.key()` is often faster than `field.unescaped_key()` so if you do not need an unescaped `std::string_view` instance, prefer `field.key()`. Similarly, we expect `field.escaped_key()` to be faster than `field.unescaped_key()` even though both return a `std::string_view` instance.
@@ -2618,8 +2650,10 @@ Performance tips
   ```
 - To better understand the operation of your On Demand parser, and whether it is performing as well as you think it should be, there is a  logger feature built in to simdjson! To use it, define the pre-processor directive `SIMDJSON_VERBOSE_LOGGING` prior to including the `simdjson.h` header, which enables logging in simdjson. Run your code. It may generate a lot of logging output; adding printouts from your application that show each section may be helpful. The log's output will show step-by-step information on state, buffer pointer position, depth, and key retrieval status. Importantly, unless `SIMDJSON_VERBOSE_LOGGING` is defined, logging is entirely disabled and thus carries no overhead.
 
+
+
 Further reading
 --------
 
 
-- John Keiser, Daniel Lemire, [On-Demand JSON: A Better Way to Parse Documents?](http://arxiv.org/abs/2312.17149), Software: Practice and Experience (to appear)
+- John Keiser, Daniel Lemire, [On-Demand JSON: A Better Way to Parse Documents?](http://arxiv.org/abs/2312.17149), Software: Practice and Experience 54 (6), 2024

doc/performance.md

@@ -14,6 +14,7 @@ testing and get the best performance.
 * [Number parsing](#number-parsing)
 * [Visual Studio](#visual-studio)
 * [Power Usage and Downclocking](#power-usage-and-downclocking)
+* [Free Padding](#free-padding)
 
 
 NDEBUG directive
@@ -179,3 +180,110 @@ The simdjson library does not generally make use of heavy 256-bit instructions.
 the macro `SIMDJSON_AVX512_ALLOWED` to `0` in C++ prior to importing the headers.
 
 You may still be worried about which SIMD instruction set is used by simdjson.  Thankfully,  [you can always determine and change which architecture-specific implementation is used](implementation-selection.md) by simdjson. Thus even if your CPU supports AVX2, you do not need to use AVX2. You are in control.
+
+
+Free Padding
+-------
+
+For performance reasons, the simdjson library requires that the JSON input contain at least
+`simdjson::SIMDJSON_PADDING` bytes at the end of the stream. The value `simdjson::SIMDJSON_PADDING` is
+small (e.g., 64 bytes). On modern systems, you can safely read beyond an allocated buffers,
+as long as you remain within an allocated page. Pages on modern systems span at least 4 kilobytes,
+but can be significantly larger. E.g., Apple systems favour pages spanning 16 kilobytes.
+
+In effect, it means that you can almost always read a few bytes beyond your current buffer---without
+allocating extra memory. However, tools such as valgrind or memory sanitizers will flag such behavior as unsafe.
+Nevertheless, you can still make sure of this capability in your code if you are an expert
+programmer and you are willing to silence sanitizer warnings. The following code provides
+a portable example.
+
+
+The conditional compilation checks for the `_MSC_VER` macro (indicating Microsoft Visual Studio)
+and includes platform-specific headers accordingly.
+The `page_size()` function determines the default size of a memory page in bytes on the system.
+On Windows (when `_WIN32` is defined), it uses `GetSystemInfo()` to retrieve system information and obtain the page size.
+On other platforms (non-Windows), it uses `sysconf(_SC_PAGESIZE)` to get the page size.
+The function returns the page size.
+The `need_allocation()` function checks whether the buffer (given by `buf`) plus the specified length (`len`) is near a page boundary.
+If the buffer extends beyond the current page when padded by `simdjson::SIMDJSON_PADDING`, it returns true, indicating that reallocation is needed.
+Otherwise, it returns false.
+The `get_padded_string_view()`  creates a `padded_string_view` from the input buffer.
+If reallocation is needed (unlikely case), it allocates a new padded_string and assigns it to `jsonbuffer`.
+Otherwise (very likely), it creates a `padded_string_view` directly from the buffer.
+The `simdjson::SIMDJSON_PADDING` ensures that there is additional padding for parsing efficiency.
+The calling code just needs to provide `jsonbuffer` (an instance of `simdjson::padded_string`)
+and pass `get_padded_string_view(buf, len, jsonbuffer)` to  `parser.iterate`. Most of the time,
+this code will not allocate new memory.
+
+
+```cpp
+#ifdef _WIN32
+#include <windows.h>
+#include <sysinfoapi.h>
+#else
+#include <unistd.h>
+#endif
+#include "simdjson.h"
+#include <cstdio>
+
+// Returns the default size of the page in bytes on this system.
+long page_size() {
+#ifdef _WIN32
+  SYSTEM_INFO sysInfo;
+  GetSystemInfo(&sysInfo);
+  long pagesize = sysInfo.dwPageSize;
+#else
+  long pagesize = sysconf(_SC_PAGESIZE);
+#endif
+  return pagesize;
+}
+
+// Returns true if the buffer + len + simdjson::SIMDJSON_PADDING crosses the
+// page boundary.
+bool need_allocation(const char *buf, size_t len) {
+  return ((reinterpret_cast<uintptr_t>(buf + len - 1) % page_size()) <
+          simdjson::SIMDJSON_PADDING);
+}
+
+simdjson::padded_string_view
+get_padded_string_view(const char *buf, size_t len,
+                       simdjson::padded_string &jsonbuffer) {
+  if (need_allocation(buf, len)) { // unlikely case
+    jsonbuffer = simdjson::padded_string(buf, len);
+    return jsonbuffer;
+  } else { // no reallcation needed (very likely)
+    return simdjson::padded_string_view(buf, len,
+                                            len + simdjson::SIMDJSON_PADDING);
+  }
+}
+
+int main() {
+  printf("page_size: %ld\n", page_size());
+  const char *jsonpoiner = R"(
+        {
+            "key": "value"
+        }
+    )";
+  size_t len = strlen(jsonpoiner);
+  simdjson::padded_string jsonbuffer; // only allocate if needed
+  simdjson::ondemand::parser parser;
+  simdjson::ondemand::document doc;
+  simdjson::error_code error =
+      parser.iterate(get_padded_string_view(jsonpoiner, len, jsonbuffer))
+          .get(doc);
+  if (error) {
+    printf("error: %s\n", simdjson::error_message(error));
+    return EXIT_FAILURE;
+  }
+  std::string_view value;
+  error = doc["key"].get_string().get(value);
+  if (error) {
+    return EXIT_FAILURE;
+  }
+  printf("Value: \"%.*s\"\n", (int)value.size(), value.data());
+  if (value != "value") {
+    return EXIT_FAILURE;
+  }
+  return EXIT_SUCCESS;
+}
+```

include/simdjson/dom/document_stream-inl.h

@@ -224,10 +224,10 @@ simdjson_inline std::string_view document_stream::iterator::source() const noexc
   } else {
     size_t next_doc_index = stream->batch_start + stream->parser->implementation->structural_indexes[stream->parser->implementation->next_structural_index];
     size_t svlen = next_doc_index - current_index();
-    if(svlen > 1) {
+    while(svlen > 1 && (std::isspace(start[svlen-1]) || start[svlen-1] == '\0')) {
       svlen--;
     }
-    return std::string_view(reinterpret_cast<const char*>(stream->buf) + current_index(), svlen);
+    return std::string_view(start, svlen);
   }
 }
 

include/simdjson/dom/element-inl.h

@@ -375,6 +375,23 @@ inline simdjson_result<element> element::operator[](const char *key) const noexc
   return at_key(key);
 }
 
+inline bool is_pointer_well_formed(std::string_view json_pointer) noexcept {
+  if (simdjson_unlikely(json_pointer[0] != '/')) {
+    return false;
+  }
+  size_t escape = json_pointer.find('~');
+  if (escape == std::string_view::npos) {
+    return true;
+  }
+  if (escape == json_pointer.size() - 1) {
+    return false;
+  }
+  if (json_pointer[escape + 1] != '0' && json_pointer[escape + 1] != '1') {
+    return false;
+  }
+  return true;
+}
+
 inline simdjson_result<element> element::at_pointer(std::string_view json_pointer) const noexcept {
   SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914
   switch (tape.tape_ref_type()) {
@@ -383,7 +400,10 @@ inline simdjson_result<element> element::at_pointer(std::string_view json_pointe
     case internal::tape_type::START_ARRAY:
       return array(tape).at_pointer(json_pointer);
     default: {
-      if(!json_pointer.empty()) { // a non-empty string is invalid on an atom
+      if (!json_pointer.empty()) { // a non-empty string can be invalid, or accessing a primitive (issue 2154)
+        if (is_pointer_well_formed(json_pointer)) {
+          return NO_SUCH_FIELD;
+        }
         return INVALID_JSON_POINTER;
       }
       // an empty string means that we return the current node

include/simdjson/dom/parser-inl.h

@@ -191,7 +191,7 @@ simdjson_inline size_t parser::capacity() const noexcept {
 simdjson_inline size_t parser::max_capacity() const noexcept {
   return _max_capacity;
 }
-simdjson_inline size_t parser::max_depth() const noexcept {
+simdjson_pure simdjson_inline size_t parser::max_depth() const noexcept {
   return implementation ? implementation->max_depth() : DEFAULT_MAX_DEPTH;
 }
 

include/simdjson/dom/parser.h

@@ -527,7 +527,7 @@ public:
    *
    * @return Maximum depth, in bytes.
    */
-  simdjson_inline size_t max_depth() const noexcept;
+  simdjson_pure simdjson_inline size_t max_depth() const noexcept;
 
   /**
    * Set max_capacity. This is the largest document this parser can automatically support.

include/simdjson/dom/serialization.h

@@ -57,15 +57,15 @@ public:
   simdjson_inline void one_char(char c);
 
   simdjson_inline void call_print_newline() {
-      this->print_newline();
+      static_cast<formatter*>(this)->print_newline();
   }
 
   simdjson_inline void call_print_indents(size_t depth) {
-      this->print_indents(depth);
+      static_cast<formatter*>(this)->print_indents(depth);
   }
 
   simdjson_inline void call_print_space() {
-      this->print_space();
+      static_cast<formatter*>(this)->print_space();
   }
 
 protected:

include/simdjson/error.h

@@ -39,7 +39,7 @@ enum error_code {
   INDEX_OUT_OF_BOUNDS,        ///< JSON array index too large
   NO_SUCH_FIELD,              ///< JSON field not found in object
   IO_ERROR,                   ///< Error reading a file
-  INVALID_JSON_POINTER,       ///< Invalid JSON pointer reference
+  INVALID_JSON_POINTER,       ///< Invalid JSON pointer syntax
   INVALID_URI_FRAGMENT,       ///< Invalid URI fragment
   UNEXPECTED_ERROR,           ///< indicative of a bug in simdjson
   PARSER_IN_USE,              ///< parser is already in use.

include/simdjson/generic/ondemand/document_stream-inl.h

@@ -348,10 +348,11 @@ simdjson_inline std::string_view document_stream::iterator::source() const noexc
           auto next_index = stream->parser->implementation->structural_indexes[++cur_struct_index];
           // normally the length would be next_index - current_index() - 1, except for the last document
           size_t svlen = next_index - current_index();
-          if(svlen > 1) {
+          const char *start = reinterpret_cast<const char*>(stream->buf) + current_index();
+          while(svlen > 1 && (std::isspace(start[svlen-1]) || start[svlen-1] == '\0')) {
             svlen--;
           }
-          return std::string_view(reinterpret_cast<const char*>(stream->buf) + current_index(), svlen);
+          return std::string_view(start, svlen);
         }
     }
     cur_struct_index++;

include/simdjson/generic/ondemand/json_iterator-inl.h

@@ -54,6 +54,23 @@ simdjson_inline json_iterator::json_iterator(const uint8_t *buf, ondemand::parse
 #endif
 }
 
+#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON
+simdjson_inline json_iterator::json_iterator(const uint8_t *buf, ondemand::parser *_parser, bool streaming) noexcept
+    : token(buf, &_parser->implementation->structural_indexes[0]),
+      parser{_parser},
+      _string_buf_loc{parser->string_buf.get()},
+      _depth{1},
+      _root{parser->implementation->structural_indexes.get()},
+      _streaming{streaming}
+
+{
+  logger::log_headers();
+#if SIMDJSON_CHECK_EOF
+  assert_more_tokens();
+#endif
+}
+#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON
+
 inline void json_iterator::rewind() noexcept {
   token.set_position( root_position() );
   logger::log_headers(); // We start again

include/simdjson/generic/ondemand/json_iterator.h

@@ -293,6 +293,9 @@ public:
   inline bool balanced() const noexcept;
 protected:
   simdjson_inline json_iterator(const uint8_t *buf, ondemand::parser *parser) noexcept;
+#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON
+  simdjson_inline json_iterator(const uint8_t *buf, ondemand::parser *parser, bool streaming) noexcept;
+#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON
   /// The last token before the end
   simdjson_inline token_position last_position() const noexcept;
   /// The token *at* the end. This points at gibberish and should only be used for comparison.

include/simdjson/generic/ondemand/parser-inl.h

@@ -58,6 +58,27 @@ simdjson_warn_unused simdjson_inline simdjson_result<document> parser::iterate(p
   return document::start({ reinterpret_cast<const uint8_t *>(json.data()), this });
 }
 
+#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON
+simdjson_warn_unused simdjson_inline simdjson_result<document> parser::iterate_allow_incomplete_json(padded_string_view json) & noexcept {
+  if (json.padding() < SIMDJSON_PADDING) { return INSUFFICIENT_PADDING; }
+
+  json.remove_utf8_bom();
+
+  // Allocate if needed
+  if (capacity() < json.length() || !string_buf) {
+    SIMDJSON_TRY( allocate(json.length(), max_depth()) );
+  }
+
+  // Run stage 1.
+  const simdjson::error_code err = implementation->stage1(reinterpret_cast<const uint8_t *>(json.data()), json.length(), stage1_mode::regular);
+  if (err) {
+    if (err != UNCLOSED_STRING)
+      return err;
+  }
+  return document::start({ reinterpret_cast<const uint8_t *>(json.data()), this, true });
+}
+#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON
+
 simdjson_warn_unused simdjson_inline simdjson_result<document> parser::iterate(const char *json, size_t len, size_t allocated) & noexcept {
   return iterate(padded_string_view(json, len, allocated));
 }
@@ -129,13 +150,13 @@ inline simdjson_result<document_stream> parser::iterate_many(const padded_string
   return iterate_many(s.data(), s.length(), batch_size, allow_comma_separated);
 }
 
-simdjson_inline size_t parser::capacity() const noexcept {
+simdjson_pure simdjson_inline size_t parser::capacity() const noexcept {
   return _capacity;
 }
-simdjson_inline size_t parser::max_capacity() const noexcept {
+simdjson_pure simdjson_inline size_t parser::max_capacity() const noexcept {
   return _max_capacity;
 }
-simdjson_inline size_t parser::max_depth() const noexcept {
+simdjson_pure simdjson_inline size_t parser::max_depth() const noexcept {
   return _max_depth;
 }
 

include/simdjson/generic/ondemand/parser.h

@@ -98,6 +98,9 @@ public:
    *         - UNCLOSED_STRING if there is an unclosed string in the document.
    */
   simdjson_warn_unused simdjson_result<document> iterate(padded_string_view json) & noexcept;
+#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON
+  simdjson_warn_unused simdjson_result<document> iterate_allow_incomplete_json(padded_string_view json) & noexcept;
+#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON
   /** @overload simdjson_result<document> iterate(padded_string_view json) & noexcept */
   simdjson_warn_unused simdjson_result<document> iterate(const char *json, size_t len, size_t capacity) & noexcept;
   /** @overload simdjson_result<document> iterate(padded_string_view json) & noexcept */
@@ -242,9 +245,9 @@ public:
   simdjson_result<document_stream> iterate_many(const char *buf, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept = delete;
 
   /** The capacity of this parser (the largest document it can process). */
-  simdjson_inline size_t capacity() const noexcept;
+  simdjson_pure simdjson_inline size_t capacity() const noexcept;
   /** The maximum capacity of this parser (the largest document it is allowed to process). */
-  simdjson_inline size_t max_capacity() const noexcept;
+  simdjson_pure simdjson_inline size_t max_capacity() const noexcept;
   simdjson_inline void set_max_capacity(size_t max_capacity) noexcept;
   /**
    * The maximum depth of this parser (the most deeply nested objects and arrays it can process).
@@ -252,7 +255,7 @@ public:
    * The document's instance current_depth() method should be used to monitor the parsing
    * depth and limit it if desired.
    */
-  simdjson_inline size_t max_depth() const noexcept;
+  simdjson_pure simdjson_inline size_t max_depth() const noexcept;
 
   /**
    * Ensure this parser has enough memory to process JSON documents up to `capacity` bytes in length

include/simdjson/generic/ondemand/value-inl.h

@@ -239,6 +239,26 @@ simdjson_inline int32_t value::current_depth() const noexcept{
   return iter.json_iter().depth();
 }
 
+inline bool is_pointer_well_formed(std::string_view json_pointer) noexcept {
+  if (simdjson_unlikely(json_pointer.empty())) { // can't be
+    return false;
+  }
+  if (simdjson_unlikely(json_pointer[0] != '/')) {
+    return false;
+  }
+  size_t escape = json_pointer.find('~');
+  if (escape == std::string_view::npos) {
+    return true;
+  }
+  if (escape == json_pointer.size() - 1) {
+    return false;
+  }
+  if (json_pointer[escape + 1] != '0' && json_pointer[escape + 1] != '1') {
+    return false;
+  }
+  return true;
+}
+
 simdjson_inline simdjson_result<value> value::at_pointer(std::string_view json_pointer) noexcept {
   json_type t;
   SIMDJSON_TRY(type().get(t));
@@ -249,6 +269,10 @@ simdjson_inline simdjson_result<value> value::at_pointer(std::string_view json_p
     case json_type::object:
       return (*this).get_object().at_pointer(json_pointer);
     default:
+      // a non-empty string can be invalid, or accessing a primitive (issue 2154)
+      if (is_pointer_well_formed(json_pointer)) {
+        return NO_SUCH_FIELD;
+      }
       return INVALID_JSON_POINTER;
   }
 }

include/simdjson/internal/dom_parser_implementation.h

@@ -177,14 +177,14 @@ public:
    *
    * @return Current capacity, in bytes.
    */
-  simdjson_inline size_t capacity() const noexcept;
+  simdjson_pure simdjson_inline size_t capacity() const noexcept;
 
   /**
    * The maximum level of nested object and arrays supported by this parser.
    *
    * @return Maximum depth, in bytes.
    */
-  simdjson_inline size_t max_depth() const noexcept;
+  simdjson_pure simdjson_inline size_t max_depth() const noexcept;
 
   /**
    * Ensure this parser has enough memory to process JSON documents up to `capacity` bytes in length
@@ -225,11 +225,11 @@ simdjson_inline dom_parser_implementation::dom_parser_implementation() noexcept
 simdjson_inline dom_parser_implementation::dom_parser_implementation(dom_parser_implementation &&other) noexcept = default;
 simdjson_inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parser_implementation &&other) noexcept = default;
 
-simdjson_inline size_t dom_parser_implementation::capacity() const noexcept {
+simdjson_pure simdjson_inline size_t dom_parser_implementation::capacity() const noexcept {
   return _capacity;
 }
 
-simdjson_inline size_t dom_parser_implementation::max_depth() const noexcept {
+simdjson_pure simdjson_inline size_t dom_parser_implementation::max_depth() const noexcept {
   return _max_depth;
 }
 

include/simdjson/nonstd/string_view.hpp

@@ -6,13 +6,13 @@
 // Distributed under the Boost Software License, Version 1.0.
 // (See accompanying file LICENSE.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
 
-// #pragma once // We remove #pragma once here as it generates a warning in some cases. We rely on the include guard.
+#pragma once
 
 #ifndef NONSTD_SV_LITE_H_INCLUDED
 #define NONSTD_SV_LITE_H_INCLUDED
 
 #define string_view_lite_MAJOR  1
-#define string_view_lite_MINOR  7
+#define string_view_lite_MINOR  8
 #define string_view_lite_PATCH  0
 
 #define string_view_lite_VERSION  nssv_STRINGIFY(string_view_lite_MAJOR) "." nssv_STRINGIFY(string_view_lite_MINOR) "." nssv_STRINGIFY(string_view_lite_PATCH)
@@ -134,6 +134,8 @@
 
 #if nssv_CONFIG_CONVERSION_STD_STRING_FREE_FUNCTIONS
 
+#include <string>
+
 namespace nonstd {
 
 template< class CharT, class Traits, class Allocator = std::allocator<CharT> >

include/simdjson/padded_string-inl.h

@@ -84,6 +84,7 @@ inline padded_string::padded_string(std::string_view sv_) noexcept
 inline padded_string::padded_string(padded_string &&o) noexcept
     : viable_size(o.viable_size), data_ptr(o.data_ptr) {
   o.data_ptr = nullptr; // we take ownership
+  o.viable_size = 0;
 }
 
 inline padded_string &padded_string::operator=(padded_string &&o) noexcept {

include/simdjson/portability.h

@@ -11,6 +11,9 @@
 #include <strings.h>
 #endif
 
+// We are using size_t without namespace std:: throughout the project
+using std::size_t;
+
 #ifdef _MSC_VER
 #define SIMDJSON_VISUAL_STUDIO 1
 /**
@@ -148,6 +151,11 @@
 #define SIMDJSON_NO_SANITIZE_UNDEFINED
 #endif
 
+#if defined(__clang__) || defined(__GNUC__)
+#define simdjson_pure [[gnu::pure]]
+#else
+#define simdjson_pure
+#endif
 
 #if defined(__clang__) || defined(__GNUC__)
 #if defined(__has_feature)

include/simdjson/simdjson_version.h

@@ -4,7 +4,7 @@
 #define SIMDJSON_SIMDJSON_VERSION_H
 
 /** The version of simdjson being used (major.minor.revision) */
-#define SIMDJSON_VERSION "3.9.2"
+#define SIMDJSON_VERSION "3.9.4"
 
 namespace simdjson {
 enum {
@@ -19,7 +19,7 @@ enum {
   /**
    * The revision (major.minor.REVISION) of simdjson being used.
    */
-  SIMDJSON_VERSION_REVISION = 2
+  SIMDJSON_VERSION_REVISION = 4
 };
 } // namespace simdjson
 

singleheader/simdjson.cpp

@@ -1,4 +1,4 @@
-/* auto-generated on 2024-05-07 18:04:59 -0400. Do not edit! */
+/* auto-generated on 2024-06-11 14:08:20 -0400. Do not edit! */
 /* including simdjson.cpp:  */
 /* begin file simdjson.cpp */
 #define SIMDJSON_SRC_SIMDJSON_CPP
@@ -84,6 +84,9 @@
 #include <strings.h>
 #endif
 
+// We are using size_t without namespace std:: throughout the project
+using std::size_t;
+
 #ifdef _MSC_VER
 #define SIMDJSON_VISUAL_STUDIO 1
 /**
@@ -527,13 +530,13 @@ SIMDJSON_PUSH_DISABLE_ALL_WARNINGS
 // Distributed under the Boost Software License, Version 1.0.
 // (See accompanying file LICENSE.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
 
-// #pragma once // We remove #pragma once here as it generates a warning in some cases. We rely on the include guard.
+#pragma once
 
 #ifndef NONSTD_SV_LITE_H_INCLUDED
 #define NONSTD_SV_LITE_H_INCLUDED
 
 #define string_view_lite_MAJOR  1
-#define string_view_lite_MINOR  7
+#define string_view_lite_MINOR  8
 #define string_view_lite_PATCH  0
 
 #define string_view_lite_VERSION  nssv_STRINGIFY(string_view_lite_MAJOR) "." nssv_STRINGIFY(string_view_lite_MINOR) "." nssv_STRINGIFY(string_view_lite_PATCH)
@@ -655,6 +658,8 @@ SIMDJSON_PUSH_DISABLE_ALL_WARNINGS
 
 #if nssv_CONFIG_CONVERSION_STD_STRING_FREE_FUNCTIONS
 
+#include <string>
+
 namespace nonstd {
 
 template< class CharT, class Traits, class Allocator = std::allocator<CharT> >
@@ -2359,7 +2364,7 @@ enum error_code {
   INDEX_OUT_OF_BOUNDS,        ///< JSON array index too large
   NO_SUCH_FIELD,              ///< JSON field not found in object
   IO_ERROR,                   ///< Error reading a file
-  INVALID_JSON_POINTER,       ///< Invalid JSON pointer reference
+  INVALID_JSON_POINTER,       ///< Invalid JSON pointer syntax
   INVALID_URI_FRAGMENT,       ///< Invalid URI fragment
   UNEXPECTED_ERROR,           ///< indicative of a bug in simdjson
   PARSER_IN_USE,              ///< parser is already in use.
@@ -6896,6 +6901,7 @@ static inline uint32_t detect_supported_architectures() {
 /* end file internal/isadetection.h */
 
 #include <initializer_list>
+#include <type_traits>
 
 namespace simdjson {
 

src/implementation.cpp

@@ -7,6 +7,7 @@
 #include <internal/isadetection.h>
 
 #include <initializer_list>
+#include <type_traits>
 
 namespace simdjson {
 

tests/dom/document_stream_tests.cpp

@@ -249,6 +249,27 @@ namespace document_stream_tests {
       TEST_SUCCEED();
   }
 
+  bool issue2181() {
+      TEST_START();
+      auto json = R"(1 2 34)"_padded;
+      simdjson::dom::parser parser;
+      simdjson::dom::document_stream stream;
+      ASSERT_SUCCESS(parser.parse_many(json).get(stream));
+      auto i = stream.begin();
+      size_t count{0};
+      std::vector<size_t> indexes = { 0, 2, 4 };
+      std::vector<std::string_view> expected = { "1", "2", "34" };
+
+      for(; i != stream.end(); ++i) {
+          auto doc = *i;
+          ASSERT_SUCCESS(doc);
+          ASSERT_TRUE(count < 3);
+          ASSERT_EQUAL(i.current_index(), indexes[count]);
+          ASSERT_EQUAL(i.source(), expected[count]);
+          count++;
+      }
+      TEST_SUCCEED();
+  }
   bool issue1310() {
     std::cout << "Running " << __func__ << std::endl;
     // hex  : 20 20 5B 20 33 2C 31 5D 20 22 22 22 22 22 22 22 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20
@@ -963,7 +984,8 @@ namespace document_stream_tests {
   }
 
   bool run() {
-    return issue2170() &&
+    return issue2181() &&
+           issue2170() &&
            skipbom() &&
            fuzzaccess() &&
            baby_fuzzer() &&

tests/dom/pointercheck.cpp

@@ -191,10 +191,40 @@ bool issue1142() {
   return true;
 }
 
+bool issue2154() { // mistakenly taking value as path should not raise INVALID_JSON_POINTER
+#if SIMDJSON_EXCEPTIONS
+  std::cout << "issue 2154" << std::endl;
+  auto example_json = R"__(
+      {
+        "obj": {
+          "s": "42",
+          "n": 42,
+          "f": 4.2
+        }
+      }
+    )__"_padded;
+  dom::parser parser;
+  dom::element example = parser.parse(example_json);
+  std::string_view sfield = example.at_pointer("/obj/s");
+  ASSERT_EQUAL(sfield, "42");
+  int64_t nfield = example.at_pointer("/obj/n");
+  ASSERT_EQUAL(nfield, 42);
+  ASSERT_ERROR(example.at_pointer("/obj/X/42").error(), NO_SUCH_FIELD);
+  ASSERT_ERROR(example.at_pointer("/obj/s/42").error(), NO_SUCH_FIELD);
+  ASSERT_ERROR(example.at_pointer("/obj/n/42").error(), NO_SUCH_FIELD);
+  ASSERT_ERROR(example.at_pointer("/obj/f/4.2").error(), NO_SUCH_FIELD);
+  ASSERT_ERROR(example.at_pointer("/obj/f/4~").error(), INVALID_JSON_POINTER);
+  ASSERT_ERROR(example.at_pointer("/obj/f/~").error(), INVALID_JSON_POINTER);
+  ASSERT_ERROR(example.at_pointer("/obj/f/~1").error(), NO_SUCH_FIELD);
+#endif
+  return true;
+}
+
 int main() {
   if (true
     && demo()
     && issue1142()
+    && issue2154()
 #ifdef SIMDJSON_ENABLE_DEPRECATED_API
     && legacy_support()
 #endif

tests/ondemand/CMakeLists.txt

@@ -28,6 +28,9 @@ add_cpp_test(ondemand_to_string              LABELS ondemand acceptance per_impl
 add_cpp_test(ondemand_twitter_tests          LABELS ondemand acceptance per_implementation)
 add_cpp_test(ondemand_wrong_type_error_tests LABELS ondemand acceptance per_implementation)
 add_cpp_test(ondemand_iterate_many_csv       LABELS ondemand acceptance per_implementation)
+if(NOT SIMDJSON_SANITIZE)
+  add_cpp_test(ondemand_cacheline              LABELS ondemand acceptance per_implementation)
+endif()
 
 if(HAVE_POSIX_FORK AND HAVE_POSIX_WAIT) # assert tests use fork and wait, which aren't on MSVC
   add_cpp_test(ondemand_assert_out_of_order_values LABELS assert per_implementation explicitonly ondemand)

tests/ondemand/ondemand_cacheline.cpp

@@ -0,0 +1,68 @@
+#ifdef _WIN32
+#include <windows.h>
+#include <sysinfoapi.h>
+#else
+#include <unistd.h>
+#endif
+#include "simdjson.h"
+#include <cstdio>
+// Returns the default size of the page in bytes on this system.
+long page_size() {
+#ifdef _WIN32
+  SYSTEM_INFO sysInfo;
+  GetSystemInfo(&sysInfo);
+  long pagesize = sysInfo.dwPageSize;
+#else
+  long pagesize = sysconf(_SC_PAGESIZE);
+#endif
+  return pagesize;
+}
+
+// Returns true if the buffer + len + simdjson::SIMDJSON_PADDING crosses the
+// page boundary.
+bool need_allocation(const char *buf, size_t len) {
+  return ((reinterpret_cast<uintptr_t>(buf + len - 1) % page_size()) <
+          simdjson::SIMDJSON_PADDING);
+}
+
+simdjson::padded_string_view
+get_padded_string_view(const char *buf, size_t len,
+                       simdjson::padded_string &jsonbuffer) {
+  if (need_allocation(buf, len)) { // unlikely case
+    jsonbuffer = simdjson::padded_string(buf, len);
+    return jsonbuffer;
+  } else { // no reallcation needed (very likely)
+    return simdjson::padded_string_view(buf, len,
+                                            len + simdjson::SIMDJSON_PADDING);
+  }
+}
+
+int main() {
+  printf("page_size: %ld\n", page_size());
+  const char *jsonpoiner = R"(
+        {
+            "key": "value"
+        }
+    )";
+  size_t len = strlen(jsonpoiner);
+  simdjson::padded_string jsonbuffer; // only allocate if needed
+  simdjson::ondemand::parser parser;
+  simdjson::ondemand::document doc;
+  simdjson::error_code error =
+      parser.iterate(get_padded_string_view(jsonpoiner, len, jsonbuffer))
+          .get(doc);
+  if (error) {
+    printf("error: %s\n", simdjson::error_message(error));
+    return EXIT_FAILURE;
+  }
+  std::string_view value;
+  error = doc["key"].get_string().get(value);
+  if (error) {
+    return EXIT_FAILURE;
+  }
+  printf("Value: \"%.*s\"\n", (int)value.size(), value.data());
+  if (value != "value") {
+    return EXIT_FAILURE;
+  }
+  return EXIT_SUCCESS;
+}

tests/ondemand/ondemand_document_stream_tests.cpp

@@ -217,6 +217,27 @@ namespace document_stream_tests {
         TEST_SUCCEED();
     }
 
+    bool issue2181() {
+        TEST_START();
+        auto json = R"(1 2 34)"_padded;
+        ondemand::parser parser;
+        ondemand::document_stream stream;
+        ASSERT_SUCCESS(parser.iterate_many(json).get(stream));
+        auto i = stream.begin();
+        size_t count{0};
+        std::vector<size_t> indexes = { 0, 2, 4 };
+        std::vector<std::string_view> expected = { "1", "2", "34" };
+
+        for(; i != stream.end(); ++i) {
+            ASSERT_SUCCESS(i.error());
+            ASSERT_TRUE(count < 3);
+            ASSERT_EQUAL(i.current_index(), indexes[count]);
+            ASSERT_EQUAL(i.source(), expected[count]);
+            count++;
+        }
+        TEST_SUCCEED();
+    }
+
     bool issue1977() {
         TEST_START();
         std::string json = R"( 1111 })";
@@ -902,6 +923,7 @@ namespace document_stream_tests {
 
     bool run() {
         return
+            issue2181() &&
             issue2170() &&
             issue2137() &&
             skipbom() &&

tests/ondemand/ondemand_json_pointer_tests.cpp

@@ -385,8 +385,38 @@ namespace json_pointer_tests {
         TEST_SUCCEED();
     }
 #endif
+
+    bool issue2154() { // mistakenly taking value as path should not raise INVALID_JSON_POINTER
+#if SIMDJSON_EXCEPTIONS
+      std::cout << "issue 2154" << std::endl;
+      auto example_json = R"__({
+            "obj": {
+                "s": "42",
+                "n": 42,
+                "f": 4.2
+            }
+        })__"_padded;
+        ondemand::parser parser;
+        ondemand::document doc;
+        ASSERT_SUCCESS(parser.iterate(example_json).get(doc));
+        std::string_view sfield = doc.at_pointer("/obj/s");
+        ASSERT_EQUAL(sfield, "42");
+        int64_t nfield = doc.at_pointer("/obj/n");
+        ASSERT_EQUAL(nfield, 42);
+        ASSERT_ERROR(doc.at_pointer("/obj/X/42").error(), NO_SUCH_FIELD);
+        ASSERT_ERROR(doc.at_pointer("/obj/s/42").error(), NO_SUCH_FIELD);
+        ASSERT_ERROR(doc.at_pointer("/obj/n/42").error(), NO_SUCH_FIELD);
+        ASSERT_ERROR(doc.at_pointer("/obj/f/4.2").error(), NO_SUCH_FIELD);
+        ASSERT_ERROR(doc.at_pointer("/obj/f/4~").error(), INVALID_JSON_POINTER);
+        ASSERT_ERROR(doc.at_pointer("/obj/f/~").error(), INVALID_JSON_POINTER);
+        ASSERT_ERROR(doc.at_pointer("/obj/f/~1").error(), NO_SUCH_FIELD);
+#endif
+      return true;
+    }
+
     bool run() {
         return
+                issue2154() &&
 #if SIMDJSON_EXCEPTIONS
                 json_pointer_invalidation_exceptions() &&
 #endif

tests/ondemand/ondemand_misc_tests.cpp

@@ -5,6 +5,22 @@ using namespace simdjson;
 
 namespace misc_tests {
   using namespace std;
+#if SIMDJSON_EXCEPTIONS
+  // user reported an asan error:
+  bool issue2199() {
+    TEST_START();
+    static constexpr std::string_view kJsonString = R"( { "name": "name", "version": 100, } )";
+    try {
+      simdjson::padded_string buffer{kJsonString};
+      simdjson::ondemand::parser parser;
+      simdjson::ondemand::document document = parser.iterate(buffer);
+      (void)document;
+    } catch (simdjson::simdjson_error& /*error*/) {
+      std::cerr << "Caught simdjson_error" << std::endl;
+    }
+    TEST_SUCCEED();
+  }
+#endif
   bool issue1981_success() {
     auto error_phrase = R"(false)"_padded;
     TEST_START();
@@ -532,7 +548,7 @@ namespace misc_tests {
     string_view token;
     ASSERT_SUCCESS(o["value"].raw_json_token().get(token));
     ASSERT_EQUAL(token, "12321323213213213213213213213211223");
-    return true;
+    TEST_SUCCEED();
   }
   simdjson_warn_unused bool big_integer_in_string() {
     TEST_START();
@@ -545,7 +561,7 @@ namespace misc_tests {
     string_view token;
     ASSERT_SUCCESS(o["value"].raw_json_token().get(token));
     ASSERT_EQUAL(token, "\"12321323213213213213213213213211223\"");
-    return true;
+    TEST_SUCCEED();
   }
   simdjson_warn_unused bool test_raw_json_token(string_view json, string_view expected_token, int expected_start_index = 0) {
     string title("'");
@@ -558,7 +574,7 @@ namespace misc_tests {
       ASSERT_EQUAL( token, expected_token );
       // Validate the text is inside the original buffer
       ASSERT_EQUAL( reinterpret_cast<const void*>(token.data()), reinterpret_cast<const void*>(&json_padded.data()[expected_start_index]));
-      return true;
+      TEST_SUCCEED();
     }));
 
     // Test values
@@ -576,10 +592,9 @@ namespace misc_tests {
       // Validate the text is inside the original buffer
       // Adjust for the {"a":
       ASSERT_EQUAL( reinterpret_cast<const void*>(token.data()), reinterpret_cast<const void*>(&json_padded.data()[5+expected_start_index]));
-      return true;
+      TEST_SUCCEED();
     }));
-
-    return true;
+    TEST_SUCCEED();
   }
 
   bool raw_json_token() {
@@ -605,6 +620,9 @@ namespace misc_tests {
 
   bool run() {
     return
+#if SIMDJSON_EXCEPTIONS
+           issue2199() &&
+#endif
            skipbom() &&
            issue1981_success() &&
            issue1981_failure() &&

tests/ondemand/ondemand_readme_examples.cpp

@@ -153,6 +153,25 @@ simdjson_inline simdjson_result<Car> simdjson::ondemand::document::get() & noexc
 
 #if SIMDJSON_EXCEPTIONS
 
+void main_capture() {
+  padded_string json_padded = "{\"a\":[1,2,3], \"b\": 2, \"c\": \"hello\"}"_padded;
+  std::vector<std::string_view> fields;
+
+  ondemand::parser parser;
+  auto doc = parser.iterate(json_padded);
+  auto object = doc.get_object();
+  for (auto field : object) {
+    fields.push_back(field.value().raw_json());
+  }
+  // Output the fields
+  // Expected output:
+  // [1,2,3]
+  // 2
+  // "hello"
+  for (std::string_view field_ref : fields) {
+    std::cout << field_ref << std::endl;
+  }
+}
 
 int custom_type_on_document() {
   padded_string json = R"( { "make": "Toyota", "model": "Camry",  "year": 2018,