version bump

* [utf8-validator] eliminate unnecessary comparison from must_be_2_3_continuation

* Fix comment in

.github/workflows/alpine.yml

@@ -9,7 +9,7 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/cache@v3
         with:
           path: dependencies/.cache

.github/workflows/debian.yaml

@@ -16,7 +16,7 @@ jobs:
       image: debian:testing
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Install dependencies
       run: |

.github/workflows/documentation.yml

@@ -19,7 +19,7 @@ jobs:
       url: ${{ steps.deployment.outputs.page_url }}
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Install Doxygen
         run: sudo apt-get install doxygen graphviz -y
       - run: mkdir docs

.github/workflows/fix-trailing-whitespace.yml

@@ -6,7 +6,7 @@ jobs:
   whitespace:
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Remove whitespace and check the diff
         run: |
           set -eu

.github/workflows/fuzzers.yml

@@ -38,7 +38,7 @@ jobs:
         chmod +x llvm.sh
         sudo ./llvm.sh $CLANGVERSION
 
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - uses: actions/cache@v3
       with:

.github/workflows/macos.yml

@@ -9,7 +9,7 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: macos-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/cache@v3
         with:
           path: dependencies/.cache

.github/workflows/msys2-clang.yml

@@ -24,7 +24,7 @@ jobs:
       CMAKE_GENERATOR: Ninja
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/cache@v3
         with:
           path: dependencies/.cache

.github/workflows/msys2.yml

@@ -26,7 +26,7 @@ jobs:
       CMAKE_GENERATOR: Ninja
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/cache@v3
         with:
           path: dependencies/.cache

.github/workflows/power-fuzz.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-20.04
     name: Build on ubuntu-20.04 ppc64le
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: uraimo/run-on-arch-action@v2.0.5
         name: Run commands
         id: runcmd

.github/workflows/ppc64.yml

@@ -12,7 +12,7 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: uraimo/run-on-arch-action@v2
         name: Test
         id: runcmd

.github/workflows/riscv64.yml

@@ -0,0 +1,29 @@
+name: Ubuntu riscv64 (GCC 11)
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: uraimo/run-on-arch-action@v2
+        name: Test
+        id: runcmd
+        with:
+          arch: riscv64
+          distro: ubuntu_latest
+          githubToken: ${{ github.token }}
+          install: |
+            apt-get update -q -y
+            apt-get install -y cmake make g++
+          run: |
+            cmake -DCMAKE_BUILD_TYPE=Release -B build
+            cmake --build build -j=2
+            ctest --output-on-failure --test-dir build

.github/workflows/s390x.yml

@@ -12,7 +12,7 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: uraimo/run-on-arch-action@v2
         name: Test
         id: runcmd

.github/workflows/ubuntu20-checkperf.yml

@@ -15,7 +15,7 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/cache@v3
         with:
           path: dependencies/.cache

.github/workflows/ubuntu20-gcc8.yml

@@ -12,7 +12,7 @@ jobs:
       CXX: g++-8
       CC: gcc-8
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/cache@v3
         with:
           path: dependencies/.cache

.github/workflows/ubuntu20-noexcept.yml

@@ -9,7 +9,7 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/cache@v3
         with:
           path: dependencies/.cache

.github/workflows/ubuntu20-nothread.yml

@@ -9,7 +9,7 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/cache@v3
         with:
           path: dependencies/.cache

.github/workflows/ubuntu20-sani.yml

@@ -9,7 +9,7 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/cache@v3
         with:
           path: dependencies/.cache
@@ -27,12 +27,11 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/cache@v3
         with:
           path: dependencies/.cache
           key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
-          ctest --output-on-failure -LE explicitonly -j
       - name: Use cmake with undefined sanitizer
         run: |
           mkdir builddebugundefsani &&

.github/workflows/ubuntu20.yml

@@ -9,7 +9,7 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/cache@v3
         with:
           path: dependencies/.cache

.github/workflows/ubuntu22-clang13.yml

@@ -9,7 +9,7 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/cache@v3
         with:
           path: dependencies/.cache

.github/workflows/ubuntu22-clang14.yml

@@ -9,7 +9,7 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/cache@v3
         with:
           path: dependencies/.cache

.github/workflows/ubuntu22-cxx20.yml

@@ -9,7 +9,7 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/cache@v3
         with:
           path: dependencies/.cache

.github/workflows/ubuntu22-gcc12.yml

@@ -9,13 +9,11 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/cache@v3
         with:
           path: dependencies/.cache
           key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
-      - name: Install gcc12
-        run: sudo apt-get install -y g++-12
       - name: Use cmake
         run: |
           mkdir build &&

.github/workflows/ubuntu22-gcc13.yml

@@ -0,0 +1,23 @@
+name: Ubuntu 22.04 CI (GCC 13)
+
+on: [push, pull_request]
+
+jobs:
+  ubuntu-build:
+    if: >-
+      ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
+      ! contains(toJSON(github.event.commits.*.message), '[skip github]')
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/cache@v3
+        with:
+          path: dependencies/.cache
+          key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
+      - name: Use cmake
+        run: |
+          mkdir build &&
+          cd build &&
+          CXX=g++-13 cmake -DSIMDJSON_DEVELOPER_MODE=ON ..  &&
+          cmake --build .   &&
+          ctest --output-on-failure -LE explicitonly -j

.github/workflows/ubuntu22-glibcxxassertions.yml

@@ -8,7 +8,7 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/cache@v3
         with:
           path: dependencies/.cache

.github/workflows/ubuntu22-threadsani.yml

@@ -9,7 +9,7 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/cache@v3
         with:
           path: dependencies/.cache

.github/workflows/ubuntu22.yml

@@ -9,7 +9,7 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/cache@v3
         with:
           path: dependencies/.cache

.github/workflows/vs17-arm-ci.yml

@@ -14,7 +14,7 @@ jobs:
            - {arch: ARM64}
      steps:
        - name: checkout
-         uses: actions/checkout@v3
+         uses: actions/checkout@v4
        - name: Use cmake
          run: |
            cmake -A ${{ matrix.arch }} -DCMAKE_CROSSCOMPILING=1 -DSIMDJSON_DEVELOPER_MODE=ON -D SIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_EXCEPTIONS=OFF -B build  &&

.github/workflows/vs17-ci-cxx20.yml

@@ -19,7 +19,7 @@ jobs:
           - {gen: Visual Studio 17 2022, arch: x64, shared: OFF}
     steps:
     - name: checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
     - name: Configure
       run: |
         cmake -DSIMDJSON_CXX_STANDARD=20 -G "${{matrix.gen}}" -A ${{matrix.arch}} -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -DBUILD_SHARED_LIBS=${{matrix.shared}} -B build

.github/workflows/vs17-ci.yml

@@ -19,7 +19,7 @@ jobs:
           - {gen: Visual Studio 17 2022, arch: x64, shared: OFF}
     steps:
     - name: checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
     - name: Configure
       run: |
         cmake -G "${{matrix.gen}}" -A ${{matrix.arch}} -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -DBUILD_SHARED_LIBS=${{matrix.shared}} -B build

.github/workflows/vs17-clang-ci.yml

@@ -16,7 +16,7 @@ jobs:
           - {gen: Visual Studio 17 2022, arch: x64}
     steps:
     - name: checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
     - name: Configure
       run: |
         cmake -G "${{matrix.gen}}" -A ${{matrix.arch}}  -T ClangCL -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -B build

.github/workflows/vs17-noexcept-ci.yml

@@ -7,7 +7,7 @@ jobs:
     name: windows-vs17
     runs-on: windows-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - uses: actions/cache@v3
       with:
         path: dependencies/.cache

CMakeLists.txt

@@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.14)
 project(
     simdjson
     # The version number is modified by tools/release.py
-    VERSION 3.4.0
+    VERSION 3.6.4
     DESCRIPTION "Parsing gigabytes of JSON per second"
     HOMEPAGE_URL "https://simdjson.org/"
     LANGUAGES CXX C
@@ -20,8 +20,8 @@ string(
 # ---- Options, variables ----
 
 # These version numbers are modified by tools/release.py
-set(SIMDJSON_LIB_VERSION "17.0.0" CACHE STRING "simdjson library version")
-set(SIMDJSON_LIB_SOVERSION "17" CACHE STRING "simdjson library soversion")
+set(SIMDJSON_LIB_VERSION "19.0.0" CACHE STRING "simdjson library version")
+set(SIMDJSON_LIB_SOVERSION "19" CACHE STRING "simdjson library soversion")
 
 option(SIMDJSON_BUILD_STATIC_LIB "Build simdjson_static library along with simdjson" OFF)
 

Doxyfile

@@ -38,7 +38,7 @@ PROJECT_NAME           = simdjson
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = "3.4.0"
+PROJECT_NUMBER         = "3.6.4"
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a

README.md

@@ -29,6 +29,7 @@ Table of Contents
 * [Real-world usage](#real-world-usage)
 * [Quick Start](#quick-start)
 * [Documentation](#documentation)
+* [Godbolt](#godbolt)
 * [Performance results](#performance-results)
 * [Bindings and Ports of simdjson](#bindings-and-ports-of-simdjson)
 * [About simdjson](#about-simdjson)
@@ -40,8 +41,10 @@ Table of Contents
 Real-world usage
 ----------------
 
+- [Node.js](https://nodejs.org/)
 - [ClickHouse](https://github.com/ClickHouse/ClickHouse)
-- [Facebook/Meta Velox](https://velox-lib.io)
+- [Meta Velox](https://velox-lib.io)
+- [Google Pax](https://github.com/google/paxml)
 - [milvus](https://github.com/milvus-io/milvus)
 - [Clang Build Analyzer](https://github.com/aras-p/ClangBuildAnalyzer)
 - [Shopify HeapProfiler](https://github.com/Shopify/heap-profiler)
@@ -66,9 +69,9 @@ The simdjson library is easily consumable with a single .h and .cpp file.
 
 0. Prerequisites: `g++` (version 7 or better) or `clang++` (version 6 or better), and a 64-bit
    system with a command-line shell (e.g., Linux, macOS, freeBSD). We also support programming
-   environments like Visual Studio and Xcode, but different steps are needed.
+   environments like Visual Studio and Xcode, but different steps are needed. Users of clang++ may need to specify the C++ version (e.g., `c++ -std=c++17`) since clang++ tends to default on C++98.
 1. Pull [simdjson.h](singleheader/simdjson.h) and [simdjson.cpp](singleheader/simdjson.cpp) into a
-   directory, along with the sample file [twitter.json](jsonexamples/twitter.json).
+   directory, along with the sample file [twitter.json](jsonexamples/twitter.json). You can download them with the `wget` utility:
 
    ```
    wget https://raw.githubusercontent.com/simdjson/simdjson/master/singleheader/simdjson.h https://raw.githubusercontent.com/simdjson/simdjson/master/singleheader/simdjson.cpp https://raw.githubusercontent.com/simdjson/simdjson/master/jsonexamples/twitter.json
@@ -93,6 +96,7 @@ int main(void) {
    100 results.
   ```
 
+
 Documentation
 -------------
 
@@ -104,11 +108,12 @@ Usage documentation is available:
   how you can work with it.
 * [API](https://simdjson.github.io/simdjson/) contains the automatically generated API documentation.
 
+Godbolt
+-------------
 
 Some users may want to browse code along with the compiled assembly. You want to check out the following lists of examples:
-
-* [simdjson examples with errors handled through exceptions](https://godbolt.org/z/98Kx9Kqjn)
-* [simdjson examples with errors without exceptions](https://godbolt.org/z/PKG7GdbPo)
+* [simdjson examples with errors handled through exceptions](https://godbolt.org/z/7G5qE4sr9)
+* [simdjson examples with errors without exceptions](https://godbolt.org/z/e9dWb9E4v)
 
 Performance results
 -------------------
@@ -163,6 +168,7 @@ We distinguish between "bindings" (which just wrap the C++ code) and a port to a
 - [simdjzon](https://github.com/travisstaloch/simdjzon): zig port.
 - [JSON-Simd](https://github.com/rawleyfowler/JSON-simd): Raku bindings.
 - [JSON::SIMD](https://metacpan.org/pod/JSON::SIMD): Perl bindings; fully-featured JSON module that uses simdjson for decoding.
+- [gemmaJSON](https://github.com/sainttttt/gemmaJSON): Nim json parser based on simdjson bindings.
 
 About simdjson
 --------------
@@ -171,7 +177,11 @@ The simdjson library takes advantage of modern microarchitectures, parallelizing
 instructions, reducing branch misprediction, and reducing data dependency to take advantage of each
 CPU's multiple execution cores.
 
-Some people [enjoy reading our paper](https://arxiv.org/abs/1902.08318): A description of the design
+Our default front-end is called On Demand, and we wrote a paper about it:
+
+- John Keiser, Daniel Lemire, [On-Demand JSON: A Better Way to Parse Documents?](http://arxiv.org/abs/2312.17149), Software: Practice and Experience (to appear)
+
+Some people [enjoy reading the first (2019) simdjson paper](https://arxiv.org/abs/1902.08318): A description of the design
 and implementation of simdjson is in our research article:
 - Geoff Langdale, Daniel Lemire, [Parsing Gigabytes of JSON per Second](https://arxiv.org/abs/1902.08318), VLDB Journal 28 (6), 2019.
 

benchmark/CMakeLists.txt

@@ -26,6 +26,9 @@ if (TARGET benchmark::benchmark)
     if(TARGET nlohmann_json)
       target_link_libraries(bench_ondemand PRIVATE nlohmann_json)
     endif()
+    if(TARGET boostjson)
+      target_link_libraries(bench_ondemand PRIVATE boostjson)
+    endif()
   endif()
 endif()
 

benchmark/bench_ondemand.cpp

@@ -21,6 +21,10 @@ SIMDJSON_PUSH_DISABLE_ALL_WARNINGS
 #include <nlohmann/json.hpp>
 #endif
 
+#ifdef SIMDJSON_COMPETITION_BOOSTJSON
+#include <boost/json.hpp>
+#endif
+
 // This has to be last, for reasons I don't yet understand
 #include <benchmark/benchmark.h>
 
@@ -33,6 +37,7 @@ SIMDJSON_POP_DISABLE_WARNINGS
 #include "json2msgpack/sajson.h"
 #endif // SIMDJSON_COMPETITION_ONDEMAND_SAJSON
 #include "json2msgpack/nlohmann_json.h"
+#include "json2msgpack/boostjson.h"
 
 #include "partial_tweets/simdjson_ondemand.h"
 #include "partial_tweets/simdjson_dom.h"
@@ -48,6 +53,7 @@ SIMDJSON_POP_DISABLE_WARNINGS
 #if SIMDJSON_COMPETITION_SAX
 #include "partial_tweets/nlohmann_json_sax.h"
 #endif // SIMDJSON_COMPETITION_SAX
+#include "partial_tweets/boostjson.h"
 
 
 #include "distinct_user_id/simdjson_ondemand.h"
@@ -66,6 +72,7 @@ SIMDJSON_POP_DISABLE_WARNINGS
 #if SIMDJSON_COMPETITION_SAX
 #include "distinct_user_id/nlohmann_json_sax.h"
 #endif // SIMDJSON_COMPETITION_SAX
+#include "distinct_user_id/boostjson.h"
 
 #include "find_tweet/simdjson_ondemand.h"
 #include "find_tweet/simdjson_dom.h"
@@ -81,6 +88,7 @@ SIMDJSON_POP_DISABLE_WARNINGS
 #if SIMDJSON_COMPETITION_SAX
 #include "find_tweet/nlohmann_json_sax.h"
 #endif // SIMDJSON_COMPETITION_SAX
+#include "find_tweet/boostjson.h"
 
 #include "top_tweet/simdjson_ondemand.h"
 #include "top_tweet/simdjson_dom.h"
@@ -96,6 +104,7 @@ SIMDJSON_POP_DISABLE_WARNINGS
 #if SIMDJSON_COMPETITION_SAX
 #include "top_tweet/nlohmann_json_sax.h"
 #endif // SIMDJSON_COMPETITION_SAX
+#include "top_tweet/boostjson.h"
 
 
 #include "kostya/simdjson_ondemand.h"
@@ -112,6 +121,7 @@ SIMDJSON_POP_DISABLE_WARNINGS
 #if SIMDJSON_COMPETITION_SAX
 #include "kostya/nlohmann_json_sax.h"
 #endif // SIMDJSON_COMPETITION_SAX
+#include "kostya/boostjson.h"
 
 #include "large_random/simdjson_ondemand.h"
 #if SIMDJSON_COMPETITION_ONDEMAND_UNORDERED
@@ -130,6 +140,7 @@ SIMDJSON_POP_DISABLE_WARNINGS
 #if SIMDJSON_COMPETITION_SAX
 #include "large_random/nlohmann_json_sax.h"
 #endif // SIMDJSON_COMPETITION_SAX
+#include "large_random/boostjson.h"
 
 #include "amazon_cellphones/simdjson_dom.h"
 #include "amazon_cellphones/simdjson_ondemand.h"

benchmark/distinct_user_id/boostjson.h

@@ -0,0 +1,29 @@
+#pragma once
+
+#ifdef SIMDJSON_COMPETITION_BOOSTJSON
+
+#include "distinct_user_id.h"
+
+namespace distinct_user_id {
+
+struct boostjson {
+  bool run(simdjson::padded_string &json, std::vector<uint64_t> &result) {
+
+    auto root = boost::json::parse(json);
+    for (const auto &tweet : root.at("statuses").as_array()) {
+      result.push_back(tweet.at("user").at("id").to_number<uint64_t>());
+
+      if (tweet.as_object().if_contains("retweeted_status")) {
+        result.push_back(tweet.at("retweeted_status").at("user").at("id").to_number<uint64_t>());
+      }
+    }
+
+    return true;
+  }
+};
+
+BENCHMARK_TEMPLATE(distinct_user_id, boostjson)->UseManualTime();
+
+} // namespace distinct_user_id
+
+#endif // SIMDJSON_COMPETITION_BOOSTJSON

benchmark/find_tweet/boostjson.h

@@ -0,0 +1,30 @@
+#pragma once
+
+#ifdef SIMDJSON_COMPETITION_BOOSTJSON
+
+#include "find_tweet.h"
+
+namespace find_tweet {
+
+struct boostjson {
+  using StringType=std::string;
+
+  bool run(simdjson::padded_string &json, uint64_t find_id, std::string &result) {
+
+    auto root = boost::json::parse(json);
+    for (const auto &tweet : root.at("statuses").as_array()) {
+      if (tweet.at("id") == find_id) {
+        result = tweet.at("text").as_string();
+        return true;
+      }
+    }
+
+    return false;
+  }
+};
+
+BENCHMARK_TEMPLATE(find_tweet, boostjson)->UseManualTime();
+
+} // namespace find_tweet
+
+#endif // SIMDJSON_COMPETITION_BOOSTJSON

benchmark/json2msgpack/boostjson.h

@@ -0,0 +1,104 @@
+#pragma once
+
+#ifdef SIMDJSON_COMPETITION_BOOSTJSON
+
+#include "json2msgpack.h"
+
+namespace json2msgpack {
+
+struct boostjson2msgpack {
+  inline std::string_view to_msgpack(const boost::json::value &root, uint8_t *buf) {
+    buff = buf;
+    recursive_processor(root);
+    return std::string_view(reinterpret_cast<char *>(buf), size_t(buff - buf));
+  }
+
+private:
+  uint8_t *buff{};
+
+  inline void write_double(const double d) noexcept {
+    *buff++ = 0xcb;
+    ::memcpy(buff, &d, sizeof(d));
+    buff += sizeof(d);
+  }
+
+  inline void write_byte(const uint8_t b) noexcept {
+    *buff = b;
+    buff++;
+  }
+
+  inline void write_uint32(const uint32_t w) noexcept {
+    ::memcpy(buff, &w, sizeof(w));
+    buff += sizeof(w);
+  }
+
+  inline void write_string(const std::string & str) {
+    write_byte(0xdb);
+    write_uint32(uint32_t(str.size()));
+    ::memcpy(buff, str.data(), str.size());
+    buff += str.size();
+  }
+
+  inline void recursive_processor(const boost::json::value &element) {
+    switch(element.kind()) {
+    case boost::json::kind::array: {
+      write_byte(0xdd);
+      const auto &array = element.as_array();
+      write_uint32(static_cast<uint32_t>(array.size()));
+      for (const auto &child : array) {
+        recursive_processor(child);
+      }
+    } break;
+
+    case boost::json::kind::object: {
+      write_byte(0xdf);
+      const auto &object = element.as_object();
+      write_uint32(static_cast<uint32_t>(object.size()));
+      for (const auto &child : object) {
+        write_string(child.key_c_str());
+        recursive_processor(child.value());
+      }
+    } break;
+
+    case boost::json::kind::int64:
+    case boost::json::kind::uint64:
+    case boost::json::kind::double_:
+      write_double(element.to_number<double>());
+      break;
+
+    case boost::json::kind::string:
+      write_string(element.as_string().c_str());
+      break;
+
+    case boost::json::kind::bool_:
+      write_byte(0xc2 + element.as_bool());
+      break;
+
+    case boost::json::kind::null:
+      write_byte(0xc0);
+      break;
+
+    default:
+      printf("unexpected\n");
+      break;
+    }
+  }
+};
+
+struct boostjson {
+  using StringType=std::string;
+
+  boostjson2msgpack parser{};
+
+  bool run(simdjson::padded_string &json, char *buffer, std::string_view &result) {
+    auto root = boost::json::parse(json);
+    result = parser.to_msgpack(root, reinterpret_cast<uint8_t *>(buffer));
+    return true;
+  }
+};
+
+BENCHMARK_TEMPLATE(json2msgpack, boostjson)->UseManualTime();
+
+} // namespace json2msgpack
+
+#endif // SIMDJSON_COMPETITION_BOOSTJSON

benchmark/kostya/boostjson.h

@@ -0,0 +1,29 @@
+#pragma once
+
+#ifdef SIMDJSON_COMPETITION_BOOSTJSON
+
+#include "kostya.h"
+
+namespace kostya {
+
+struct boostjson {
+  static constexpr diff_flags DiffFlags = diff_flags::IMPRECISE_FLOATS;
+
+  bool run(simdjson::padded_string &json, std::vector<point> &result) {
+    auto root = boost::json::parse(json);
+    for (const auto &point : root.at("coordinates").as_array()) {
+      result.emplace_back(json_benchmark::point{
+        point.at("x").to_number<double>(),
+        point.at("y").to_number<double>(),
+        point.at("z").to_number<double>()
+      });
+    }
+    return true;
+  }
+};
+
+BENCHMARK_TEMPLATE(kostya, boostjson)->UseManualTime();
+
+} // namespace kostya
+
+#endif // SIMDJSON_COMPETITION_BOOSTJSON

benchmark/large_random/boostjson.h

@@ -0,0 +1,29 @@
+#pragma once
+
+#ifdef SIMDJSON_COMPETITION_BOOSTJSON
+
+#include "large_random.h"
+
+namespace large_random {
+
+struct boostjson {
+  static constexpr diff_flags DiffFlags = diff_flags::IMPRECISE_FLOATS;
+
+  bool run(simdjson::padded_string &json, std::vector<point> &result) {
+    auto root = boost::json::parse(json);
+    for (const auto &point : root.as_array()) {
+      result.emplace_back(json_benchmark::point{
+        point.at("x").to_number<double>(),
+        point.at("y").to_number<double>(),
+        point.at("z").to_number<double>()
+      });
+    }
+    return true;
+  }
+};
+
+BENCHMARK_TEMPLATE(large_random, boostjson)->UseManualTime();
+
+} // namespace large_random
+
+#endif // SIMDJSON_COMPETITION_BOOSTJSON

benchmark/partial_tweets/boostjson.h

@@ -0,0 +1,43 @@
+#pragma once
+
+#ifdef SIMDJSON_COMPETITION_BOOSTJSON
+
+#include "partial_tweets.h"
+
+namespace partial_tweets {
+
+struct boostjson {
+  using StringType=std::string;
+
+  bool run(simdjson::padded_string &json, std::vector<tweet<StringType>> &result) {
+
+    auto root = boost::json::parse(json);
+    for (const auto &tweet : root.at("statuses").as_array()) {
+      const auto &user = tweet.at("user");
+
+      auto in_reply_to_status_id = tweet.as_object().if_contains("in_reply_to_status_id")
+        ? tweet.at("in_reply_to_status_id") : boost::json::value();
+
+      result.emplace_back(partial_tweets::tweet<StringType>{
+        tweet.at("created_at").as_string().c_str(),
+        tweet.at("id").to_number<uint64_t>(),
+        tweet.at("text").as_string().c_str(),
+        in_reply_to_status_id.is_null() ? 0 : in_reply_to_status_id.to_number<uint64_t>(),
+        {
+          user.at("id").to_number<uint64_t>(),
+          user.at("screen_name").as_string().c_str()
+        },
+        tweet.at("retweet_count").to_number<uint64_t>(),
+        tweet.at("favorite_count").to_number<uint64_t>()
+      });
+    }
+
+    return true;
+  }
+};
+
+BENCHMARK_TEMPLATE(partial_tweets, boostjson)->UseManualTime();
+
+} // namespace partial_tweets
+
+#endif // SIMDJSON_COMPETITION_BOOSTJSON

benchmark/top_tweet/boostjson.h

@@ -0,0 +1,37 @@
+#pragma once
+
+#if SIMDJSON_COMPETITION_BOOSTJSON
+
+#include "top_tweet.h"
+
+namespace top_tweet {
+
+using namespace simdjson;
+
+struct boostjson {
+  using StringType=std::string;
+
+  bool run(simdjson::padded_string &json, int64_t max_retweet_count, top_tweet_result<StringType> &result) {
+    result.retweet_count = -1;
+    boost::json::value top_tweet{};
+
+    auto root = boost::json::parse(json);
+    for (const auto &tweet : root.at("statuses").as_array()) {
+      int64_t retweet_count = tweet.at("retweet_count").as_int64();
+      if (retweet_count <= max_retweet_count && retweet_count >= result.retweet_count) {
+        result.retweet_count = retweet_count;
+        top_tweet = tweet;
+      }
+    }
+
+    result.text = top_tweet.at("text").as_string();
+    result.screen_name = top_tweet.at("user").at("screen_name").as_string();
+    return result.retweet_count != -1;
+  }
+};
+
+BENCHMARK_TEMPLATE(top_tweet, boostjson)->UseManualTime();
+
+} // namespace top_tweet
+
+#endif // SIMDJSON_COMPETITION_BOOSTJSON

doc/basics.md

@@ -30,11 +30,13 @@ An overview of what you need to know to use simdjson, with examples.
   - [Dynamic Number Types](#dynamic-number-types)
   - [Raw Strings](#raw-strings)
   - [General Direct Access to the Raw JSON String](#general-direct-access-to-the-raw-json-string)
+  - [Storing Directly into an Existing String Instance](#storing-directly-into-an-existing-string-instance)
   - [Thread Safety](#thread-safety)
   - [Standard Compliance](#standard-compliance)
   - [Backwards Compatibility](#backwards-compatibility)
   - [Examples](#examples)
   - [Performance Tips](#performance-tips)
+  - [Further Reading](#further-reading)
 
 
 Requirements
@@ -85,7 +87,7 @@ include(FetchContent)
 FetchContent_Declare(
   simdjson
   GIT_REPOSITORY https://github.com/simdjson/simdjson.git
-  GIT_TAG  tags/v0.9.6
+  GIT_TAG  tags/v3.6.0
   GIT_SHALLOW TRUE)
 
 FetchContent_MakeAvailable(simdjson)
@@ -218,7 +220,8 @@ This means that while you iterate an array, or search for a field in an object,
 walking through the original JSON text, merrily reading commas and colons and brackets to make sure
 you get where you are going. This is the key to On Demand's performance: since it's just an iterator,
 it lets you parse values as you use them. And particularly, it lets you *skip* values you do not want
-to use.
+to use. On Demand is also ideally suited when you want to capture part of the document without parsing it
+immediately (e.g., see [Raw Strings](#raw-strings)).
 
 We refer to "On Demand" as a front-end component since it is an interface between the
 low-level parsing functions and the user. It hides much of the complexity of parsing JSON
@@ -253,7 +256,8 @@ copy the data into their own favorite class instances (e.g., alternatives to `st
 
 A `std::string_view` instance is effectively just a pointer to a region in memory representing
 a string. In simdjson, we return `std::string_view` instances that either point within the
-input string you parsed, or to a temporary string buffer inside our parser class instances.
+input string you parsed (when using [raw Strings](#raw-strings)), or to a temporary string buffer inside
+our parser class instances that is valid until the parser object is destroyed or you use it to parse another document.
 When using `std::string_view` instances, it is your responsibility to ensure that
 `std::string_view` instance does not outlive the pointed-to memory (e.g., either the input
 buffer or the parser instance). Furthermore, some operations reset the string buffer
@@ -261,6 +265,7 @@ inside our parser instances: e.g., when we parse a new document. Thus a `std::st
 is often best viewed as a temporary string value that is tied to the document you are parsing.
 At the cost of some memory allocation, you may convert your `std::string_view` instances for long-term storage into `std::string` instances:
 `std::string mycopy(view)` (C++17) or  `std::string mycopy(view.begin(), view.end())` (prior to C++17).
+For convenience, we also allow [storing an escaped string directly into an existing string instance](#storing-directly-into-an-existing-string-instance).
 
 The `std::string_view` class has become standard as part of C++17 but it is not always available
 on compilers which only supports C++11. When we detect that `string_view` is natively
@@ -355,7 +360,7 @@ support for users who avoid exceptions. See [the simdjson error handling documen
 * **Field Access:** To get the value of the "foo" field in an object, use `object["foo"]`. This will
   scan through the object looking for the field with the matching string, doing a character-by-character
   comparison. It may generate the error `simdjson::NO_SUCH_FIELD` if there is no such key in the object, it may throw an exception (see [Error Handling](#error-handling)). For efficiency reason, you should avoid looking up the same field repeatedly: e.g., do
-  not do `object["foo"]` followed by `object["foo"]` with the same `object` instance. Keep in mind that On Demand does not buffer or save the result of the parsing: if you repeatedly access `object["foo"]`, then it must repeatedly seek the key and parse the content. The library does not provide a distinct function to check if a key is present, instead we recommend you attempt to access the key: e.g., by doing `ondemand::value val{}; if (!object["foo"].get(val)) {...}`, you have that `val` contains the requested value inside the if clause.  It is your responsibility as a user to temporarily keep a reference to the value (`auto v = object["foo"]`), or to consume the content and store it in your own data structures. If you consume an
+  not do `object["foo"]` followed by `object["foo"]` with the same `object` instance. For best performance, you should try to query the keys in the same order they appear in the document. If you need several keys and you cannot predict the order they will appear in, it is recommended to iterate through all keys `for(auto field : object) {...}`.  Keep in mind that On Demand does not buffer or save the result of the parsing: if you repeatedly access `object["foo"]`, then it must repeatedly seek the key and parse the content. The library does not provide a distinct function to check if a key is present, instead we recommend you attempt to access the key: e.g., by doing `ondemand::value val{}; if (!object["foo"].get(val)) {...}`, you have that `val` contains the requested value inside the if clause.  It is your responsibility as a user to temporarily keep a reference to the value (`auto v = object["foo"]`), or to consume the content and store it in your own data structures. If you consume an
   object twice: `std::string_view(object["foo"]` followed by `std::string_view(object["foo"]` then your code
   is in error. Furthermore, you can only consume one field at a time, on the same object. The
   value instance you get from  `content["bids"]` becomes invalid when you call `content["asks"]`.
@@ -541,9 +546,7 @@ support for users who avoid exceptions. See [the simdjson error handling documen
 * **Tree Walking and JSON Element Types:** Sometimes you don't necessarily have a document
   with a known type, and are trying to generically inspect or walk over JSON elements.
   You can also represent arbitrary JSON values with
-  `ondemand::value` instances: it can represent anything except a scalar document (lone number, string, null or Boolean). You can check for scalar documents with the method `scalar()`.
-  You can query the type of a document or a value with the `type()` method.
-  The `type()` method does not consume or validate documents and values, but it tells you whether they are
+  `ondemand::value` instances: it can represent anything except a scalar document (lone number, string, null or Boolean). You can check for scalar documents with the method `scalar()`. You can cast a document that is either an array or an object to an `ondemand::value` instance immediately after you create the document instance: you cannot create a `ondemand::value` instance from a document that has already been accessed as it would mean that you would have two instances of the object or array simultaneously (see [rewinding](#rewinding)). You can query the type of a document or a value with the `type()` method. The `type()` method does not consume or validate documents and values, but it tells you whether they are
   - arrays (`json_type::array`),
   - objects (`json_type::object`)
   - numbers (`json_type::number`),
@@ -1654,8 +1657,8 @@ JSON string to a user-provided buffer:
 General Direct Access to the Raw JSON String
 --------------------------------
 If your value is a string, the `raw_json_string` you with `get_raw_json_string()` gives you direct access to the unprocessed
-string. The simdjson library allows you to have access to the raw underlying JSON
-more generally.
+string. But the simdjson library allows you to have access to the raw underlying JSON
+more generally, not just for strings.
 
 The simdjson library makes explicit assumptions about types. For examples, numbers
 must be integers (up to 64-bit integers) or binary64 floating-point numbers. Some users
@@ -1688,9 +1691,9 @@ string_view token = obj["value"].raw_json_token();
 
 The `raw_json_token()` should be fast and free of allocation.
 
-If you value is an array or an object, `raw_json_token()` returns effectively a single
+If your value is an array or an object, `raw_json_token()` returns effectively a single
 character (`[`) or (`}`) which is not very useful. For arrays and objects, we have another
-method called `raw_json()` which consumes (traverse) the array or the object.
+method called `raw_json()` which consumes (traverses) the array or the object.
 
 ```C++
 simdjson::ondemand::parser parser;
@@ -1722,13 +1725,42 @@ obj.reset(); // revise the object
 uint64_t x = obj["value"]; // gives me 123
 ```
 
-Storing Directly into an Existing std::string Instance
+You can use `raw_json()` with the values inside an array and object. When
+calling `raw_json()` on an untyped value, it acts as `raw_json()` when the
+value is an array or an object. Otherwise, it acts as `raw_json_token()`.
+It is useful if you do not care for the type of the value and just wants a
+string representation.
+
+```C++
+  auto json = R"( [1,2,"fds", {"a":1}, [1,344]] )"_padded;
+  ondemand::parser parser;
+  ondemand::document doc = parser.iterate(json);
+  size_t counter = 0;
+  for(auto array: doc) {
+    std::string_view raw = array.raw_json();
+    // will capture "1", "2", "\"fds\"", "{\"a\":1}", "[1,344]"
+  }
+```
+
+```C++
+  auto json = R"( {"key1":1,"key2":2,"key3":"fds", "key4":{"a":1}, "key5":[1,344]} )"_padded;
+  ondemand::parser parser;
+  ondemand::document doc = parser.iterate(json);
+  size_t counter = 0;
+  for(auto key_value: doc.get_object()) {
+    std::string_view raw = key_value.value().raw_json();
+    // will capture "1", "2", "\"fds\"", "{\"a\":1}", "[1,344]"
+  }
+```
+
+
+Storing Directly into an Existing String Instance
 -----------------------------------------------------
 
 The simdjson library favours  the use of `std::string_view` instances because
 it tends to lead to better performance due to causing fewer memory allocations.
 However, they are cases where you need to store a string result in an `std::string``
-instance. You can do so with a version of the `to_string()` method which takes as
+instance. You can do so with a templated version of the `to_string()` method which takes as
 a parameter a reference to an `std::string`.
 
 ```C++
@@ -1750,10 +1782,22 @@ The same routine can be written without exceptions handling:
   if(err) { /* handle error */ }
 ```
 
-The `std::string` instance, once created, is independent. Unlike our `std::string_view` instances, it does not point at data that is
-within our `parser` instance. The same caveat applies: you should
+The `std::string` instance, once created, is independent. Unlike our `std::string_view` instances,
+it does not point at data that is within our `parser` instance. The same caveat applies: you should
 only consume a JSON string once.
 
+Because `get_string()` is a template that requires a type that can be assigned an `std::string`, you
+can use it with features such as `std::optional`:
+
+```C++
+  auto json = R"({ "foo1": "3.1416" } )"_padded;
+  ondemand::parser parser;
+  ondemand::document doc = parser.iterate(json);
+  std::optional<std::string> value;
+  if(doc["foo1"].get_string(value)) { /* error */ }
+  // value was populated with "3.1416"
+```
+
 You should be mindful of the trade-off: allocating multiple
 `std::string` instances can become expensive.
 
@@ -2084,3 +2128,9 @@ Performance Tips
 	std::string_view rating = data["rating"];
   ```
 - To better understand the operation of your On Demand parser, and whether it is performing as well as you think it should be, there is a  logger feature built in to simdjson! To use it, define the pre-processor directive `SIMDJSON_VERBOSE_LOGGING` prior to including the `simdjson.h` header, which enables logging in simdjson. Run your code. It may generate a lot of logging output; adding printouts from your application that show each section may be helpful. The log's output will show step-by-step information on state, buffer pointer position, depth, and key retrieval status. Importantly, unless `SIMDJSON_VERBOSE_LOGGING` is defined, logging is entirely disabled and thus carries no overhead.
+
+Further reading
+--------
+
+
+- John Keiser, Daniel Lemire, [On-Demand JSON: A Better Way to Parse Documents?](http://arxiv.org/abs/2312.17149), Software: Practice and Experience (to appear)

include/simdjson/arm64/simd.h

@@ -17,89 +17,53 @@ namespace {
 // Start of private section with Visual Studio workaround
 
 
-/**
- * make_uint8x16_t initializes a SIMD register (uint8x16_t).
- * This is needed because, incredibly, the syntax uint8x16_t x = {1,2,3...}
- * is not recognized under Visual Studio! This is a workaround.
- * Using a std::initializer_list<uint8_t>  as a parameter resulted in
- * inefficient code. With the current approach, if the parameters are
- * compile-time constants,
- * GNU GCC compiles it to ldr, the same as uint8x16_t x = {1,2,3...}.
- * You should not use this function except for compile-time constants:
- * it is not efficient.
- */
-simdjson_inline uint8x16_t make_uint8x16_t(uint8_t x1,  uint8_t x2,  uint8_t x3,  uint8_t x4,
-                                         uint8_t x5,  uint8_t x6,  uint8_t x7,  uint8_t x8,
-                                         uint8_t x9,  uint8_t x10, uint8_t x11, uint8_t x12,
-                                         uint8_t x13, uint8_t x14, uint8_t x15, uint8_t x16) {
-  // Doing a load like so end ups generating worse code.
-  // uint8_t array[16] = {x1, x2, x3, x4, x5, x6, x7, x8,
-  //                     x9, x10,x11,x12,x13,x14,x15,x16};
-  // return vld1q_u8(array);
-  uint8x16_t x{};
-  // incredibly, Visual Studio does not allow x[0] = x1
-  x = vsetq_lane_u8(x1, x, 0);
-  x = vsetq_lane_u8(x2, x, 1);
-  x = vsetq_lane_u8(x3, x, 2);
-  x = vsetq_lane_u8(x4, x, 3);
-  x = vsetq_lane_u8(x5, x, 4);
-  x = vsetq_lane_u8(x6, x, 5);
-  x = vsetq_lane_u8(x7, x, 6);
-  x = vsetq_lane_u8(x8, x, 7);
-  x = vsetq_lane_u8(x9, x, 8);
-  x = vsetq_lane_u8(x10, x, 9);
-  x = vsetq_lane_u8(x11, x, 10);
-  x = vsetq_lane_u8(x12, x, 11);
-  x = vsetq_lane_u8(x13, x, 12);
-  x = vsetq_lane_u8(x14, x, 13);
-  x = vsetq_lane_u8(x15, x, 14);
-  x = vsetq_lane_u8(x16, x, 15);
-  return x;
-}
+#ifndef simdjson_make_uint8x16_t
+#define simdjson_make_uint8x16_t(x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, \
+                             x13, x14, x15, x16)                                   \
+   ([=]() {                                                                        \
+     uint8_t array[16] = {x1, x2,  x3,  x4,  x5,  x6,  x7,  x8,                    \
+                                 x9, x10, x11, x12, x13, x14, x15, x16};           \
+     return vld1q_u8(array);                                                       \
+   }())
+#endif
+#ifndef simdjson_make_int8x16_t
+#define simdjson_make_int8x16_t(x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, \
+                             x13, x14, x15, x16)                                  \
+   ([=]() {                                                                       \
+     int8_t array[16] = {x1, x2,  x3,  x4,  x5,  x6,  x7,  x8,                    \
+                                 x9, x10, x11, x12, x13, x14, x15, x16};          \
+     return vld1q_s8(array);                                                      \
+   }())
+#endif
 
-simdjson_inline uint8x8_t make_uint8x8_t(uint8_t x1,  uint8_t x2,  uint8_t x3,  uint8_t x4,
-                                         uint8_t x5,  uint8_t x6,  uint8_t x7,  uint8_t x8) {
-  uint8x8_t x{};
-  x = vset_lane_u8(x1, x, 0);
-  x = vset_lane_u8(x2, x, 1);
-  x = vset_lane_u8(x3, x, 2);
-  x = vset_lane_u8(x4, x, 3);
-  x = vset_lane_u8(x5, x, 4);
-  x = vset_lane_u8(x6, x, 5);
-  x = vset_lane_u8(x7, x, 6);
-  x = vset_lane_u8(x8, x, 7);
-  return x;
-}
-
-// We have to do the same work for make_int8x16_t
-simdjson_inline int8x16_t make_int8x16_t(int8_t x1,  int8_t x2,  int8_t x3,  int8_t x4,
-                                       int8_t x5,  int8_t x6,  int8_t x7,  int8_t x8,
-                                       int8_t x9,  int8_t x10, int8_t x11, int8_t x12,
-                                       int8_t x13, int8_t x14, int8_t x15, int8_t x16) {
-  // Doing a load like so end ups generating worse code.
-  // int8_t array[16] = {x1, x2, x3, x4, x5, x6, x7, x8,
-  //                     x9, x10,x11,x12,x13,x14,x15,x16};
-  // return vld1q_s8(array);
-  int8x16_t x{};
-  // incredibly, Visual Studio does not allow x[0] = x1
-  x = vsetq_lane_s8(x1, x, 0);
-  x = vsetq_lane_s8(x2, x, 1);
-  x = vsetq_lane_s8(x3, x, 2);
-  x = vsetq_lane_s8(x4, x, 3);
-  x = vsetq_lane_s8(x5, x, 4);
-  x = vsetq_lane_s8(x6, x, 5);
-  x = vsetq_lane_s8(x7, x, 6);
-  x = vsetq_lane_s8(x8, x, 7);
-  x = vsetq_lane_s8(x9, x, 8);
-  x = vsetq_lane_s8(x10, x, 9);
-  x = vsetq_lane_s8(x11, x, 10);
-  x = vsetq_lane_s8(x12, x, 11);
-  x = vsetq_lane_s8(x13, x, 12);
-  x = vsetq_lane_s8(x14, x, 13);
-  x = vsetq_lane_s8(x15, x, 14);
-  x = vsetq_lane_s8(x16, x, 15);
-  return x;
-}
+#ifndef simdjson_make_uint8x8_t
+#define simdjson_make_uint8x8_t(x1, x2, x3, x4, x5, x6, x7, x8)                \
+   ([=]() {                                                                    \
+     uint8_t array[8] = {x1, x2,  x3,  x4,  x5,  x6,  x7,  x8};                \
+     return vld1_u8(array);                                                    \
+   }())
+#endif
+#ifndef simdjson_make_int8x8_t
+#define simdjson_make_int8x8_t(x1, x2, x3, x4, x5, x6, x7, x8)                 \
+   ([=]() {                                                                    \
+     int8_t array[8] = {x1, x2,  x3,  x4,  x5,  x6,  x7,  x8};                 \
+     return vld1_s8(array);                                                    \
+   }())
+#endif
+#ifndef simdjson_make_uint16x8_t
+#define simdjson_make_uint16x8_t(x1, x2, x3, x4, x5, x6, x7, x8)               \
+   ([=]() {                                                                    \
+     uint16_t array[8] = {x1, x2,  x3,  x4,  x5,  x6,  x7,  x8};               \
+     return vld1q_u16(array);                                                  \
+   }())
+#endif
+#ifndef simdjson_make_int16x8_t
+#define simdjson_make_int16x8_t(x1, x2, x3, x4, x5, x6, x7, x8)                \
+   ([=]() {                                                                    \
+     int16_t array[8] = {x1, x2,  x3,  x4,  x5,  x6,  x7,  x8};                \
+     return vld1q_s16(array);                                                  \
+   }())
+#endif
 
 // End of private section with Visual Studio workaround
 } // namespace
@@ -158,7 +122,7 @@ simdjson_inline int8x16_t make_int8x16_t(int8_t x1,  int8_t x2,  int8_t x3,  int
     // purposes (cutting it down to uint16_t costs performance in some compilers).
     simdjson_inline uint32_t to_bitmask() const {
 #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
-      const uint8x16_t bit_mask =  make_uint8x16_t(0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80,
+      const uint8x16_t bit_mask =  simdjson_make_uint8x16_t(0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80,
                                                    0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80);
 #else
       const uint8x16_t bit_mask =  {0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80,
@@ -192,7 +156,7 @@ simdjson_inline int8x16_t make_int8x16_t(int8_t x1,  int8_t x2,  int8_t x3,  int
     simdjson_inline simd8(
       uint8_t v0,  uint8_t v1,  uint8_t v2,  uint8_t v3,  uint8_t v4,  uint8_t v5,  uint8_t v6,  uint8_t v7,
       uint8_t v8,  uint8_t v9,  uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15
-    ) : simd8(make_uint8x16_t(
+    ) : simd8(simdjson_make_uint8x16_t(
       v0, v1, v2, v3, v4, v5, v6, v7,
       v8, v9, v10,v11,v12,v13,v14,v15
     )) {}
@@ -283,7 +247,7 @@ simdjson_inline int8x16_t make_int8x16_t(int8_t x1,  int8_t x2,  int8_t x3,  int
       uint8x16_t shufmask = vreinterpretq_u8_u64(shufmask64);
       // we increment by 0x08 the second half of the mask
 #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
-      uint8x16_t inc = make_uint8x16_t(0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08);
+      uint8x16_t inc = simdjson_make_uint8x16_t(0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08);
 #else
       uint8x16_t inc = {0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08};
 #endif
@@ -313,7 +277,7 @@ simdjson_inline int8x16_t make_int8x16_t(int8_t x1,  int8_t x2,  int8_t x3,  int
       uint8x8_t compactmask2 = vcreate_u8(thintable_epi8[mask2]);
       // we increment by 0x08 the second half of the mask
 #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
-      uint8x8_t inc = make_uint8x8_t(0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08);
+      uint8x8_t inc = simdjson_make_uint8x8_t(0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08);
 #else
       uint8x8_t inc = {0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08};
 #endif
@@ -368,7 +332,7 @@ simdjson_inline int8x16_t make_int8x16_t(int8_t x1,  int8_t x2,  int8_t x3,  int
     simdjson_inline simd8(
       int8_t v0,  int8_t v1,  int8_t v2,  int8_t v3, int8_t v4,  int8_t v5,  int8_t v6,  int8_t v7,
       int8_t v8,  int8_t v9,  int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15
-    ) : simd8(make_int8x16_t(
+    ) : simd8(simdjson_make_int8x16_t(
       v0, v1, v2, v3, v4, v5, v6, v7,
       v8, v9, v10,v11,v12,v13,v14,v15
     )) {}
@@ -486,7 +450,7 @@ simdjson_inline int8x16_t make_int8x16_t(int8_t x1,  int8_t x2,  int8_t x3,  int
 
     simdjson_inline uint64_t to_bitmask() const {
 #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
-      const uint8x16_t bit_mask = make_uint8x16_t(
+      const uint8x16_t bit_mask = simdjson_make_uint8x16_t(
         0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80,
         0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80
       );

include/simdjson/dom/document_stream.h

@@ -224,8 +224,7 @@ private:
    * Parse the next document found in the buffer previously given to document_stream.
    *
    * The content should be a valid JSON document encoded as UTF-8. If there is a
-   * UTF-8 BOM, the caller is responsible for omitting it, UTF-8 BOM are
-   * discouraged.
+   * UTF-8 BOM, the parser skips it.
    *
    * You do NOT need to pre-allocate a parser.  This function takes care of
    * pre-allocating a capacity defined by the batch_size defined when creating the

include/simdjson/dom/element.h

@@ -214,7 +214,7 @@ public:
     static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library. "
       "The supported types are Boolean (bool), numbers (double, uint64_t, int64_t), "
       "strings (std::string_view, const char *), arrays (dom::array) and objects (dom::object). "
-      "We recommand you use get_double(), get_bool(), get_uint64(), get_int64(), "
+      "We recommend you use get_double(), get_bool(), get_uint64(), get_int64(), "
       "get_object(), get_array() or get_string() instead of the get template.");
   }
 

include/simdjson/dom/parser-inl.h

@@ -12,6 +12,7 @@
 #include "simdjson/dom/element-inl.h"
 
 #include <climits>
+#include <cstring> /* memcmp */
 
 namespace simdjson {
 namespace dom {
@@ -91,10 +92,14 @@ inline simdjson_result<size_t> parser::read_file(const std::string &path) noexce
 }
 
 inline simdjson_result<element> parser::load(const std::string &path) & noexcept {
+  return load_into_document(doc, path);
+}
+
+inline simdjson_result<element> parser::load_into_document(document& provided_doc, const std::string &path) & noexcept {
   size_t len;
   auto _error = read_file(path).get(len);
   if (_error) { return _error; }
-  return parse(loaded_bytes.get(), len, false);
+  return parse_into_document(provided_doc, loaded_bytes.get(), len, false);
 }
 
 inline simdjson_result<document_stream> parser::load_many(const std::string &path, size_t batch_size) noexcept {
@@ -120,8 +125,14 @@ inline simdjson_result<element> parser::parse_into_document(document& provided_d
       _loaded_bytes_capacity = len;
     }
     std::memcpy(static_cast<void *>(loaded_bytes.get()), buf, len);
+    buf = reinterpret_cast<const uint8_t*>(loaded_bytes.get());
   }
-  _error = implementation->parse(realloc_if_needed ? reinterpret_cast<const uint8_t*>(loaded_bytes.get()): buf, len, provided_doc);
+
+  if((len >= 3) && (std::memcmp(buf, "\xEF\xBB\xBF", 3) == 0)) {
+    buf += 3;
+    len -= 3;
+  }
+  _error = implementation->parse(buf, len, provided_doc);
 
   if (_error) { return _error; }
 
@@ -158,6 +169,10 @@ simdjson_inline simdjson_result<element> parser::parse(const padded_string_view
 
 inline simdjson_result<document_stream> parser::parse_many(const uint8_t *buf, size_t len, size_t batch_size) noexcept {
   if(batch_size < MINIMAL_BATCH_SIZE) { batch_size = MINIMAL_BATCH_SIZE; }
+  if((len >= 3) && (std::memcmp(buf, "\xEF\xBB\xBF", 3) == 0)) {
+    buf += 3;
+    len -= 3;
+  }
   return document_stream(*this, buf, len, batch_size);
 }
 inline simdjson_result<document_stream> parser::parse_many(const char *buf, size_t len, size_t batch_size) noexcept {

include/simdjson/dom/parser.h

@@ -95,6 +95,53 @@ public:
    */
   inline simdjson_result<element> load(const std::string &path) & noexcept;
   inline simdjson_result<element> load(const std::string &path) &&  = delete ;
+
+  /**
+   * Load a JSON document from a file into a provide document instance and return a temporary reference to it.
+   * It is similar to the function `load` except that instead of parsing into the internal
+   * `document` instance associated with the parser, it allows the user to provide a document
+   * instance.
+   *
+   *   dom::parser parser;
+   *   dom::document doc;
+   *   element doc_root = parser.load_into_document(doc, "jsonexamples/twitter.json");
+   *
+   * The function is eager: the file's content is loaded in memory inside the parser instance
+   * and immediately parsed. The file can be deleted after the `parser.load_into_document` call.
+   *
+   * ### IMPORTANT: Document Lifetime
+   *
+   * After the call to load_into_document, the parser is no longer needed.
+   *
+   * The JSON document lives in the document instance: you must keep the document
+   * instance alive while you navigate through it (i.e., used the returned value from
+   * load_into_document). You are encourage to reuse the document instance
+   * many times with new data to avoid reallocations:
+   *
+   *   dom::document doc;
+   *   element doc_root1 = parser.load_into_document(doc, "jsonexamples/twitter.json");
+   *   //... doc_root1 is a pointer inside doc
+   *   element doc_root2 = parser.load_into_document(doc, "jsonexamples/twitter.json");
+   *   //... doc_root2 is a pointer inside doc
+   *   // at this point doc_root1 is no longer safe
+   *
+   * Moving the document instance is safe, but it invalidates the element instances. After
+   * moving a document, you can recover safe access to the document root with its `root()` method.
+   *
+   * @param doc The document instance where the parsed data will be stored (on success).
+   * @param path The path to load.
+   * @return The document, or an error:
+   *         - IO_ERROR if there was an error opening or reading the file.
+   *           Be mindful that on some 32-bit systems,
+   *           the file size might be limited to 2 GB.
+   *         - MEMALLOC if the parser does not have enough capacity and memory allocation fails.
+   *         - CAPACITY if the parser does not have enough capacity and len > max_capacity.
+   *         - other json errors if parsing fails. You should not rely on these errors to always the same for the
+   *           same document: they may vary under runtime dispatch (so they may vary depending on your system and hardware).
+   */
+  inline simdjson_result<element> load_into_document(document& doc, const std::string &path) & noexcept;
+  inline simdjson_result<element> load_into_document(document& doc, const std::string &path) && =delete;
+
   /**
    * Parse a JSON document and return a temporary reference to it.
    *
@@ -254,6 +301,8 @@ public:
    * And, possibly, no document many have been parsed when the `parser.load_many(path)` function
    * returned.
    *
+   * If there is a UTF-8 BOM, the parser skips it.
+   *
    * ### Format
    *
    * The file must contain a series of one or more JSON documents, concatenated into a single
@@ -346,6 +395,8 @@ public:
    *     cout << std::string(doc["title"]) << endl;
    *   }
    *
+   * If there is a UTF-8 BOM, the parser skips it.
+   *
    * ### Format
    *
    * The buffer must contain a series of one or more JSON documents, concatenated into a single

include/simdjson/error.h

@@ -42,7 +42,7 @@ enum error_code {
   INVALID_URI_FRAGMENT,       ///< Invalid URI fragment
   UNEXPECTED_ERROR,           ///< indicative of a bug in simdjson
   PARSER_IN_USE,              ///< parser is already in use.
-  OUT_OF_ORDER_ITERATION,     ///< tried to iterate an array or object out of order
+  OUT_OF_ORDER_ITERATION,     ///< tried to iterate an array or object out of order (checked when SIMDJSON_DEVELOPMENT_CHECKS=1)
   INSUFFICIENT_PADDING,       ///< The JSON doesn't have enough padding for simdjson to safely parse it.
   INCOMPLETE_ARRAY_OR_OBJECT, ///< The document ends early.
   SCALAR_DOCUMENT_AS_VALUE,   ///< A scalar document is treated as a value.
@@ -51,6 +51,13 @@ enum error_code {
   NUM_ERROR_CODES
 };
 
+/**
+ * It is the convention throughout the code that  the macro SIMDJSON_DEVELOPMENT_CHECKS determines whether
+ * we check for OUT_OF_ORDER_ITERATION. The logic behind it is that these errors only occurs when the code
+ * that was written while breaking some simdjson::ondemand requirement. They should not occur in released
+ * code after these issues were fixed.
+ */
+
 /**
  * Get the error message for the given error code.
  *
@@ -210,11 +217,11 @@ struct simdjson_result : public internal::simdjson_result_base<T> {
    */
   simdjson_inline simdjson_result() noexcept;
   /**
-   * @private Create a new error result.
+   * @private Create a new successful result.
    */
   simdjson_inline simdjson_result(T &&value) noexcept;
   /**
-   * @private Create a new successful result.
+   * @private Create a new error result.
    */
   simdjson_inline simdjson_result(error_code error_code) noexcept;
   /**

include/simdjson/generic/numberparsing.h

@@ -446,11 +446,8 @@ simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t d
 } // unnamed namespace
 
 /** @private */
-template<typename W>
-error_code slow_float_parsing(simdjson_unused const uint8_t * src, W writer) {
-  double d;
-  if (parse_float_fallback(src, &d)) {
-    writer.append_double(d);
+static error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) {
+  if (parse_float_fallback(src, answer)) {
     return SUCCESS;
   }
   return INVALID_NUMBER(src);
@@ -474,13 +471,13 @@ simdjson_inline error_code write_float(const uint8_t *const src, bool negative,
     // 10000000000000000000000000000000000000000000e+308
     // 3.1415926535897932384626433832795028841971693993751
     //
-    // NOTE: This makes a *copy* of the writer and passes it to slow_float_parsing. This happens
-    // because slow_float_parsing is a non-inlined function. If we passed our writer reference to
-    // it, it would force it to be stored in memory, preventing the compiler from picking it apart
-    // and putting into registers. i.e. if we pass it as reference, it gets slow.
-    // This is what forces the skip_double, as well.
-    error_code error = slow_float_parsing(src, writer);
-    writer.skip_double();
+    // NOTE: We do not pass a reference to the to slow_float_parsing. If we passed our writer
+    // reference to it, it would force it to be stored in memory, preventing the compiler from
+    // picking it apart and putting into registers. i.e. if we pass it as reference,
+    // it gets slow.
+    double d;
+    error_code error = slow_float_parsing(src, &d);
+    writer.append_double(d);
     return error;
   }
   // NOTE: it's weird that the simdjson_unlikely() only wraps half the if, but it seems to get slower any other
@@ -1004,7 +1001,7 @@ simdjson_unused simdjson_inline simdjson_result<double> parse_double(const uint8
       // Skip leading 0.00000 and see if it still overflows
       const uint8_t *start_digits = src + 2;
       while (*start_digits == '0') { start_digits++; }
-      overflow = start_digits-src > 19;
+      overflow = p-start_digits > 19;
     }
   } else {
     overflow = p-src > 19;
@@ -1203,7 +1200,7 @@ simdjson_unused simdjson_inline simdjson_result<double> parse_double_in_string(c
       // Skip leading 0.00000 and see if it still overflows
       const uint8_t *start_digits = src + 2;
       while (*start_digits == '0') { start_digits++; }
-      overflow = start_digits-src > 19;
+      overflow = p-start_digits > 19;
     }
   } else {
     overflow = p-src > 19;

include/simdjson/generic/ondemand/document-inl.h

@@ -68,7 +68,17 @@ simdjson_inline simdjson_result<object> document::start_or_resume_object() noexc
 simdjson_inline simdjson_result<value> document::get_value() noexcept {
   // Make sure we start any arrays or objects before returning, so that start_root_<object/array>()
   // gets called.
-  iter.assert_at_document_depth();
+
+  // It is the convention throughout the code that  the macro `SIMDJSON_DEVELOPMENT_CHECKS` determines whether
+  // we check for OUT_OF_ORDER_ITERATION. Proper on::demand code should never trigger this error.
+#if SIMDJSON_DEVELOPMENT_CHECKS
+  if (!iter.at_root()) { return OUT_OF_ORDER_ITERATION; }
+#endif
+  // assert_at_root() serves two purposes: in Debug mode, whether or not
+  // SIMDJSON_DEVELOPMENT_CHECKS is set or not, it checks that we are at the root of
+  // the document (this will typically be redundant). In release mode, it generates
+  // SIMDJSON_ASSUME statements to allow the compiler to make assumptions.
+  iter.assert_at_root();
   switch (*iter.peek()) {
     case '[': {
       // The following lines check that the document ends with ].
@@ -128,7 +138,8 @@ simdjson_inline simdjson_result<double> document::get_double_in_string() noexcep
 simdjson_inline simdjson_result<std::string_view> document::get_string(bool allow_replacement) noexcept {
   return get_root_value_iterator().get_root_string(true, allow_replacement);
 }
-simdjson_inline error_code document::get_string(std::string& receiver, bool allow_replacement) noexcept {
+template <typename string_type>
+simdjson_inline error_code document::get_string(string_type& receiver, bool allow_replacement) noexcept {
   return get_root_value_iterator().get_root_string(receiver, true, allow_replacement);
 }
 simdjson_inline simdjson_result<std::string_view> document::get_wobbly_string() noexcept {
@@ -400,7 +411,8 @@ simdjson_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLE
   if (error()) { return error(); }
   return first.get_string(allow_replacement);
 }
-simdjson_inline error_code simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::get_string(std::string& receiver, bool allow_replacement) noexcept {
+template <typename string_type>
+simdjson_inline error_code simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::get_string(string_type& receiver, bool allow_replacement) noexcept {
   if (error()) { return error(); }
   return first.get_string(receiver, allow_replacement);
 }
@@ -590,7 +602,8 @@ simdjson_inline simdjson_result<int64_t> document_reference::get_int64_in_string
 simdjson_inline simdjson_result<double> document_reference::get_double() noexcept { return doc->get_root_value_iterator().get_root_double(false); }
 simdjson_inline simdjson_result<double> document_reference::get_double_in_string() noexcept { return doc->get_root_value_iterator().get_root_double(false); }
 simdjson_inline simdjson_result<std::string_view> document_reference::get_string(bool allow_replacement) noexcept { return doc->get_root_value_iterator().get_root_string(false, allow_replacement); }
-simdjson_inline error_code document_reference::get_string(std::string& receiver, bool allow_replacement) noexcept { return doc->get_root_value_iterator().get_root_string(receiver, false, allow_replacement); }
+template <typename string_type>
+simdjson_inline error_code document_reference::get_string(string_type& receiver, bool allow_replacement) noexcept { return doc->get_root_value_iterator().get_root_string(receiver, false, allow_replacement); }
 simdjson_inline simdjson_result<std::string_view> document_reference::get_wobbly_string() noexcept { return doc->get_root_value_iterator().get_root_wobbly_string(false); }
 simdjson_inline simdjson_result<raw_json_string> document_reference::get_raw_json_string() noexcept { return doc->get_root_value_iterator().get_root_raw_json_string(false); }
 simdjson_inline simdjson_result<bool> document_reference::get_bool() noexcept { return doc->get_root_value_iterator().get_root_bool(false); }
@@ -727,7 +740,8 @@ simdjson_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLE
   if (error()) { return error(); }
   return first.get_string(allow_replacement);
 }
-simdjson_inline error_code simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::get_string(std::string& receiver, bool allow_replacement) noexcept {
+template <typename string_type>
+simdjson_inline error_code simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::get_string(string_type& receiver, bool allow_replacement) noexcept {
   if (error()) { return error(); }
   return first.get_string(receiver, allow_replacement);
 }

include/simdjson/generic/ondemand/document.h

@@ -113,7 +113,8 @@ public:
    *
    * @returns INCORRECT_TYPE if the JSON value is not a string. Otherwise, we return SUCCESS.
    */
-  simdjson_inline error_code get_string(std::string& receiver, bool allow_replacement = false) noexcept;
+  template <typename string_type>
+  simdjson_inline error_code get_string(string_type& receiver, bool allow_replacement = false) noexcept;
   /**
    * Cast this JSON value to a string.
    *
@@ -145,6 +146,12 @@ public:
   /**
    * Cast this JSON value to a value when the document is an object or an array.
    *
+   * You must not have begun iterating through the object or array. When
+   * SIMDJSON_DEVELOPMENT_CHECKS is set to 1 (which is the case when building in Debug mode
+   * by default), and you have already begun iterating,
+   * you will get an OUT_OF_ORDER_ITERATION error. If you have begun iterating, you can use
+   * rewind() to reset the document to its initial state before calling this method.
+   *
    * @returns A value if a JSON array or object cannot be found.
    * @returns SCALAR_DOCUMENT_AS_VALUE error is the document is a scalar (see is_scalar() function).
    */
@@ -267,10 +274,15 @@ public:
    */
   simdjson_inline operator bool() noexcept(false);
   /**
-   * Cast this JSON value to a value.
+   * Cast this JSON value to a value when the document is an object or an array.
    *
-   * @returns A value value.
-   * @exception if a JSON value cannot be found
+   * You must not have begun iterating through the object or array. When
+   * SIMDJSON_DEVELOPMENT_CHECKS is defined, and you have already begun iterating,
+   * you will get an OUT_OF_ORDER_ITERATION error. If you have begun iterating, you can use
+   * rewind() to reset the document to its initial state before calling this method.
+   *
+   * @returns A value value if a JSON array or object cannot be found.
+   * @exception SCALAR_DOCUMENT_AS_VALUE error is the document is a scalar (see is_scalar() function).
    */
   simdjson_inline operator value() noexcept(false);
 #endif
@@ -638,7 +650,8 @@ public:
   simdjson_inline simdjson_result<double> get_double() noexcept;
   simdjson_inline simdjson_result<double> get_double_in_string() noexcept;
   simdjson_inline simdjson_result<std::string_view> get_string(bool allow_replacement = false) noexcept;
-  simdjson_inline error_code get_string(std::string& receiver, bool allow_replacement = false) noexcept;
+  template <typename string_type>
+  simdjson_inline error_code get_string(string_type& receiver, bool allow_replacement = false) noexcept;
   simdjson_inline simdjson_result<std::string_view> get_wobbly_string() noexcept;
   simdjson_inline simdjson_result<raw_json_string> get_raw_json_string() noexcept;
   simdjson_inline simdjson_result<bool> get_bool() noexcept;
@@ -708,7 +721,8 @@ public:
   simdjson_inline simdjson_result<double> get_double() noexcept;
   simdjson_inline simdjson_result<double> get_double_in_string() noexcept;
   simdjson_inline simdjson_result<std::string_view> get_string(bool allow_replacement = false) noexcept;
-  simdjson_inline error_code get_string(std::string& receiver, bool allow_replacement = false) noexcept;
+  template <typename string_type>
+  simdjson_inline error_code get_string(string_type& receiver, bool allow_replacement = false) noexcept;
   simdjson_inline simdjson_result<std::string_view> get_wobbly_string() noexcept;
   simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::raw_json_string> get_raw_json_string() noexcept;
   simdjson_inline simdjson_result<bool> get_bool() noexcept;
@@ -781,7 +795,8 @@ public:
   simdjson_inline simdjson_result<double> get_double() noexcept;
   simdjson_inline simdjson_result<double> get_double_in_string() noexcept;
   simdjson_inline simdjson_result<std::string_view> get_string(bool allow_replacement = false) noexcept;
-  simdjson_inline error_code get_string(std::string& receiver, bool allow_replacement = false) noexcept;
+  template <typename string_type>
+  simdjson_inline error_code get_string(string_type& receiver, bool allow_replacement = false) noexcept;
   simdjson_inline simdjson_result<std::string_view> get_wobbly_string() noexcept;
   simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::raw_json_string> get_raw_json_string() noexcept;
   simdjson_inline simdjson_result<bool> get_bool() noexcept;

include/simdjson/generic/ondemand/document_stream.h

@@ -241,8 +241,7 @@ private:
    * Parse the next document found in the buffer previously given to document_stream.
    *
    * The content should be a valid JSON document encoded as UTF-8. If there is a
-   * UTF-8 BOM, the caller is responsible for omitting it, UTF-8 BOM are
-   * discouraged.
+   * UTF-8 BOM, the parser skips it.
    *
    * You do NOT need to pre-allocate a parser.  This function takes care of
    * pre-allocating a capacity defined by the batch_size defined when creating the

include/simdjson/generic/ondemand/json_type.h

@@ -92,8 +92,6 @@ protected:
    */
   friend class value_iterator;
   template<typename W>
-  friend error_code numberparsing::slow_float_parsing(simdjson_unused const uint8_t * src, W writer);
-  template<typename W>
   friend error_code numberparsing::write_float(const uint8_t *const src, bool negative, uint64_t i, const uint8_t * start_digits, size_t digit_count, int64_t exponent, W &writer);
   template<typename W>
   friend error_code numberparsing::parse_number(const uint8_t *const src, W &writer);

include/simdjson/generic/ondemand/parser-inl.h

@@ -46,6 +46,8 @@ simdjson_warn_unused simdjson_inline error_code parser::allocate(size_t new_capa
 simdjson_warn_unused simdjson_inline simdjson_result<document> parser::iterate(padded_string_view json) & noexcept {
   if (json.padding() < SIMDJSON_PADDING) { return INSUFFICIENT_PADDING; }
 
+  json.remove_utf8_bom();
+
   // Allocate if needed
   if (capacity() < json.length() || !string_buf) {
     SIMDJSON_TRY( allocate(json.length(), max_depth()) );
@@ -96,6 +98,8 @@ simdjson_warn_unused simdjson_inline simdjson_result<document> parser::iterate(c
 simdjson_warn_unused simdjson_inline simdjson_result<json_iterator> parser::iterate_raw(padded_string_view json) & noexcept {
   if (json.padding() < SIMDJSON_PADDING) { return INSUFFICIENT_PADDING; }
 
+  json.remove_utf8_bom();
+
   // Allocate if needed
   if (capacity() < json.length()) {
     SIMDJSON_TRY( allocate(json.length(), max_depth()) );
@@ -108,6 +112,10 @@ simdjson_warn_unused simdjson_inline simdjson_result<json_iterator> parser::iter
 
 inline simdjson_result<document_stream> parser::iterate_many(const uint8_t *buf, size_t len, size_t batch_size, bool allow_comma_separated) noexcept {
   if(batch_size < MINIMAL_BATCH_SIZE) { batch_size = MINIMAL_BATCH_SIZE; }
+  if((len >= 3) && (std::memcmp(buf, "\xEF\xBB\xBF", 3) == 0)) {
+    buf += 3;
+    len -= 3;
+  }
   if(allow_comma_separated && batch_size < len) { batch_size = len; }
   return document_stream(*this, buf, len, batch_size, allow_comma_separated);
 }

include/simdjson/generic/ondemand/parser.h

@@ -59,7 +59,7 @@ public:
    * It is expected that the content is a valid UTF-8 file, containing a valid JSON document.
    * Otherwise the iterate method may return an error. In particular, the whole input should be
    * valid: we do not attempt to tolerate incorrect content either before or after a JSON
-   * document.
+   * document. If there is a UTF-8 BOM, the parser skips it.
    *
    * ### IMPORTANT: Validate what you use
    *
@@ -188,6 +188,7 @@ public:
    * arrays or objects) MUST be separated with ASCII whitespace.
    *
    * The characters inside a JSON document, and between JSON documents, must be valid Unicode (UTF-8).
+   * If there is a UTF-8 BOM, the parser skips it.
    *
    * The documents must not exceed batch_size bytes (by default 1MB) or they will fail to parse.
    * Setting batch_size to excessively large or excessively small values may impact negatively the

include/simdjson/generic/ondemand/value-inl.h

@@ -47,7 +47,8 @@ simdjson_inline simdjson_result<raw_json_string> value::get_raw_json_string() no
 simdjson_inline simdjson_result<std::string_view> value::get_string(bool allow_replacement) noexcept {
   return iter.get_string(allow_replacement);
 }
-simdjson_inline error_code value::get_string(std::string& receiver, bool allow_replacement) noexcept {
+template <typename string_type>
+simdjson_inline error_code value::get_string(string_type& receiver, bool allow_replacement) noexcept {
   return iter.get_string(receiver, allow_replacement);
 }
 simdjson_inline simdjson_result<std::string_view> value::get_wobbly_string() noexcept {
@@ -196,6 +197,26 @@ simdjson_inline std::string_view value::raw_json_token() noexcept {
   return std::string_view(reinterpret_cast<const char*>(iter.peek_start()), iter.peek_start_length());
 }
 
+simdjson_inline simdjson_result<std::string_view> value::raw_json() noexcept {
+  json_type t;
+  SIMDJSON_TRY(type().get(t));
+  switch (t)
+  {
+    case json_type::array: {
+      ondemand::array array;
+      SIMDJSON_TRY(get_array().get(array));
+      return array.raw_json();
+    }
+    case json_type::object: {
+      ondemand::object object;
+      SIMDJSON_TRY(get_object().get(object));
+      return object.raw_json();
+    }
+    default:
+      return raw_json_token();
+  }
+}
+
 simdjson_inline simdjson_result<const char *> value::current_location() noexcept {
   return iter.json_iter().current_location();
 }
@@ -322,7 +343,8 @@ simdjson_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLE
   if (error()) { return error(); }
   return first.get_string(allow_replacement);
 }
-simdjson_inline error_code simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::get_string(std::string& receiver, bool allow_replacement) noexcept {
+template <typename string_type>
+simdjson_inline error_code simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::get_string(string_type& receiver, bool allow_replacement) noexcept {
   if (error()) { return error(); }
   return first.get_string(receiver, allow_replacement);
 }
@@ -426,6 +448,11 @@ simdjson_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLE
   return first.raw_json_token();
 }
 
+simdjson_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::raw_json() noexcept {
+  if (error()) { return error(); }
+  return first.raw_json();
+}
+
 simdjson_inline simdjson_result<const char *> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::current_location() noexcept {
   if (error()) { return error(); }
   return first.current_location();

include/simdjson/generic/ondemand/value.h

@@ -148,7 +148,8 @@ public:
    *
    * @returns INCORRECT_TYPE if the JSON value is not a string. Otherwise, we return SUCCESS.
    */
-  simdjson_inline error_code get_string(std::string& receiver, bool allow_replacement = false) noexcept;
+  template <typename string_type>
+  simdjson_inline error_code get_string(string_type& receiver, bool allow_replacement = false) noexcept;
 
   /**
    * Cast this JSON value to a "wobbly" string.
@@ -492,9 +493,21 @@ public:
    * - true
    * - false
    * - null
+   *
+   * See also value::raw_json().
    */
   simdjson_inline std::string_view raw_json_token() noexcept;
 
+  /**
+   * Get a string_view pointing at this value in the JSON document.
+   * If this element is an array or an object, it consumes the array or the object
+   * and returns a string_view instance corresponding to the
+   * array as represented in JSON. It points inside the original document.
+   * If this element is a scalar (string, number, Boolean, null), it returns what
+   * raw_json_token() would return.
+   */
+  simdjson_inline simdjson_result<std::string_view> raw_json() noexcept;
+
   /**
    * Returns the current location in the document if in bounds.
    */
@@ -619,7 +632,8 @@ public:
   simdjson_inline simdjson_result<double> get_double() noexcept;
   simdjson_inline simdjson_result<double> get_double_in_string() noexcept;
   simdjson_inline simdjson_result<std::string_view> get_string(bool allow_replacement = false) noexcept;
-  simdjson_inline error_code get_string(std::string& receiver, bool allow_replacement = false) noexcept;
+  template <typename string_type>
+  simdjson_inline error_code get_string(string_type& receiver, bool allow_replacement = false) noexcept;
   simdjson_inline simdjson_result<std::string_view> get_wobbly_string() noexcept;
   simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::raw_json_string> get_raw_json_string() noexcept;
   simdjson_inline simdjson_result<bool> get_bool() noexcept;
@@ -712,6 +726,7 @@ public:
 
   /** @copydoc simdjson_inline std::string_view value::raw_json_token() const noexcept */
   simdjson_inline simdjson_result<std::string_view> raw_json_token() noexcept;
+  simdjson_inline simdjson_result<std::string_view> raw_json() noexcept;
 
   /** @copydoc simdjson_inline simdjson_result<const char *> current_location() noexcept */
   simdjson_inline simdjson_result<const char *> current_location() noexcept;

include/simdjson/generic/ondemand/value_iterator-inl.h

@@ -513,11 +513,12 @@ simdjson_warn_unused simdjson_inline simdjson_result<bool> value_iterator::parse
 simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> value_iterator::get_string(bool allow_replacement) noexcept {
   return get_raw_json_string().unescape(json_iter(), allow_replacement);
 }
-simdjson_warn_unused simdjson_inline error_code value_iterator::get_string(std::string& receiver, bool allow_replacement) noexcept {
+template <typename string_type>
+simdjson_warn_unused simdjson_inline error_code value_iterator::get_string(string_type& receiver, bool allow_replacement) noexcept {
   std::string_view content;
   auto err = get_string(allow_replacement).get(content);
   if (err) { return err; }
-  receiver = std::string(content);
+  receiver = content;
   return SUCCESS;
 }
 simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> value_iterator::get_wobbly_string() noexcept {
@@ -643,11 +644,12 @@ simdjson_inline simdjson_result<number> value_iterator::get_root_number(bool che
 simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> value_iterator::get_root_string(bool check_trailing, bool allow_replacement) noexcept {
   return get_root_raw_json_string(check_trailing).unescape(json_iter(), allow_replacement);
 }
-simdjson_warn_unused simdjson_inline error_code value_iterator::get_root_string(std::string& receiver, bool check_trailing, bool allow_replacement) noexcept {
+template <typename string_type>
+simdjson_warn_unused simdjson_inline error_code value_iterator::get_root_string(string_type& receiver, bool check_trailing, bool allow_replacement) noexcept {
   std::string_view content;
   auto err = get_root_string(check_trailing, allow_replacement).get(content);
   if (err) { return err; }
-  receiver = std::string(content);
+  receiver = content;
   return SUCCESS;
 }
 simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> value_iterator::get_root_wobbly_string(bool check_trailing) noexcept {

include/simdjson/generic/ondemand/value_iterator.h

@@ -296,7 +296,8 @@ public:
    */
 
   simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> get_string(bool allow_replacement) noexcept;
-  simdjson_warn_unused simdjson_inline error_code get_string(std::string& receiver, bool allow_replacement) noexcept;
+  template <typename string_type>
+  simdjson_warn_unused simdjson_inline error_code get_string(string_type& receiver, bool allow_replacement) noexcept;
   simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> get_wobbly_string() noexcept;
   simdjson_warn_unused simdjson_inline simdjson_result<raw_json_string> get_raw_json_string() noexcept;
   simdjson_warn_unused simdjson_inline simdjson_result<uint64_t> get_uint64() noexcept;
@@ -313,7 +314,8 @@ public:
   simdjson_warn_unused simdjson_inline simdjson_result<number> get_number() noexcept;
 
   simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> get_root_string(bool check_trailing, bool allow_replacement) noexcept;
-  simdjson_warn_unused simdjson_inline error_code get_root_string(std::string& receiver, bool check_trailing, bool allow_replacement) noexcept;
+  template <typename string_type>
+  simdjson_warn_unused simdjson_inline error_code get_root_string(string_type& receiver, bool check_trailing, bool allow_replacement) noexcept;
   simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> get_root_wobbly_string(bool check_trailing) noexcept;
   simdjson_warn_unused simdjson_inline simdjson_result<raw_json_string> get_root_raw_json_string(bool check_trailing) noexcept;
   simdjson_warn_unused simdjson_inline simdjson_result<uint64_t> get_root_uint64(bool check_trailing) noexcept;

include/simdjson/nonstd/string_view.hpp

@@ -6,7 +6,7 @@
 // Distributed under the Boost Software License, Version 1.0.
 // (See accompanying file LICENSE.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
 
-#pragma once
+// #pragma once // We remove #pragma once here as it generates a warning in some cases. We rely on the include guard.
 
 #ifndef NONSTD_SV_LITE_H_INCLUDED
 #define NONSTD_SV_LITE_H_INCLUDED
@@ -69,6 +69,10 @@
 # define nssv_CONFIG_NO_STREAM_INSERTION  0
 #endif
 
+#ifndef  nssv_CONFIG_CONSTEXPR11_STD_SEARCH
+# define nssv_CONFIG_CONSTEXPR11_STD_SEARCH  1
+#endif
+
 // Control presence of exception handling (try and auto discover):
 
 #ifndef nssv_CONFIG_NO_EXCEPTIONS
@@ -267,7 +271,7 @@ using std::operator<<;
 # define nssv_HAS_CPP0X  0
 #endif
 
-// Unless defined otherwise below, consider VC14 as C++11 for variant-lite:
+// Unless defined otherwise below, consider VC14 as C++11 for string-view-lite:
 
 #if nssv_COMPILER_MSVC_VER >= 1900
 # undef  nssv_CPP11_OR_GREATER
@@ -434,9 +438,9 @@ using std::operator<<;
 # pragma clang diagnostic ignored "-Wreserved-user-defined-literal"
 # pragma clang diagnostic push
 # pragma clang diagnostic ignored "-Wuser-defined-literals"
-#elif defined(__GNUC__)
-# pragma  GCC  diagnostic push
-# pragma  GCC  diagnostic ignored "-Wliteral-suffix"
+#elif nssv_COMPILER_GNUC_VERSION >= 480
+#  pragma  GCC  diagnostic push
+#  pragma  GCC  diagnostic ignored "-Wliteral-suffix"
 #endif // __clang__
 
 #if nssv_COMPILER_MSVC_VERSION >= 140
@@ -451,8 +455,8 @@ using std::operator<<;
 
 #if defined(__clang__)
 # define nssv_RESTORE_WARNINGS()  _Pragma("clang diagnostic pop")
-#elif defined(__GNUC__)
-# define nssv_RESTORE_WARNINGS()  _Pragma("GCC diagnostic pop")
+#elif nssv_COMPILER_GNUC_VERSION >= 480
+#  define nssv_RESTORE_WARNINGS()  _Pragma("GCC diagnostic pop")
 #elif nssv_COMPILER_MSVC_VERSION >= 140
 # define nssv_RESTORE_WARNINGS()  __pragma(warning(pop ))
 #else
@@ -567,12 +571,31 @@ constexpr const CharT* search( basic_string_view<CharT, Traits> haystack, basic_
 
 // non-recursive:
 
+#if nssv_CONFIG_CONSTEXPR11_STD_SEARCH
+
 template< class CharT, class Traits = std::char_traits<CharT> >
 constexpr const CharT* search( basic_string_view<CharT, Traits> haystack, basic_string_view<CharT, Traits> needle )
 {
     return std::search( haystack.begin(), haystack.end(), needle.begin(), needle.end() );
 }
 
+#else // nssv_CONFIG_CONSTEXPR11_STD_SEARCH
+
+template< class CharT, class Traits = std::char_traits<CharT> >
+nssv_constexpr14 const CharT* search( basic_string_view<CharT, Traits> haystack, basic_string_view<CharT, Traits> needle )
+{
+    while ( needle.size() <= haystack.size() )
+    {
+        if  ( haystack.starts_with(needle) )
+        {
+            return haystack.cbegin();
+        }
+        haystack = basic_string_view<CharT, Traits>{ haystack.begin() + 1, haystack.size() - 1U };
+    }
+    return haystack.cend();
+}
+#endif // nssv_CONFIG_CONSTEXPR11_STD_SEARCH
+
 #endif // OPTIMIZE
 #endif // nssv_CPP11_OR_GREATER && ! nssv_CPP17_OR_GREATER
 
@@ -845,7 +868,7 @@ public:
 
     // find(), 4x:
 
-    nssv_constexpr size_type find( basic_string_view v, size_type pos = 0 ) const nssv_noexcept  // (1)
+    nssv_constexpr14 size_type find( basic_string_view v, size_type pos = 0 ) const nssv_noexcept  // (1)
     {
         return assert( v.size() == 0 || v.data() != nssv_nullptr )
             , pos >= size()

include/simdjson/padded_string_view-inl.h

@@ -2,9 +2,10 @@
 #define SIMDJSON_PADDED_STRING_VIEW_INL_H
 
 #include "simdjson/padded_string_view.h"
-
 #include "simdjson/error-inl.h"
 
+#include <cstring> /* memcmp */
+
 namespace simdjson {
 
 inline padded_string_view::padded_string_view(const char* s, size_t len, size_t capacity) noexcept
@@ -31,6 +32,16 @@ inline size_t padded_string_view::capacity() const noexcept { return _capacity;
 
 inline size_t padded_string_view::padding() const noexcept { return capacity() - length(); }
 
+inline bool padded_string_view::remove_utf8_bom() noexcept {
+  if(length() < 3) { return false; }
+  if (std::memcmp(data(), "\xEF\xBB\xBF", 3) == 0) {
+    remove_prefix(3);
+    _capacity -= 3;
+    return true;
+  }
+  return false;
+}
+
 #if SIMDJSON_EXCEPTIONS
 inline std::ostream& operator<<(std::ostream& out, simdjson_result<padded_string_view> &s) noexcept(false) { return out << s.value(); }
 #endif

include/simdjson/padded_string_view.h

@@ -54,6 +54,13 @@ public:
   /** The number of allocated bytes. */
   inline size_t capacity() const noexcept;
 
+  /**
+   * Remove the UTF-8 Byte Order Mark (BOM) if it exists.
+   *
+   * @return whether a BOM was found and removed
+   */
+  inline bool remove_utf8_bom() noexcept;
+
   /** The amount of padding on the string (capacity() - length()) */
   inline size_t padding() const noexcept;
 

include/simdjson/portability.h

@@ -36,6 +36,8 @@
 #define SIMDJSON_IS_X86_64 1
 #elif defined(__aarch64__) || defined(_M_ARM64)
 #define SIMDJSON_IS_ARM64 1
+#elif defined(__riscv) && __riscv_xlen == 64
+#define SIMDJSON_IS_RISCV64 1
 #elif defined(__PPC64__) || defined(_M_PPC64)
 #if defined(__ALTIVEC__)
 #define SIMDJSON_IS_PPC64_VMX 1

include/simdjson/simdjson_version.h

@@ -4,7 +4,7 @@
 #define SIMDJSON_SIMDJSON_VERSION_H
 
 /** The version of simdjson being used (major.minor.revision) */
-#define SIMDJSON_VERSION "3.4.0"
+#define SIMDJSON_VERSION "3.6.4"
 
 namespace simdjson {
 enum {
@@ -15,11 +15,11 @@ enum {
   /**
    * The minor version (major.MINOR.revision) of simdjson being used.
    */
-  SIMDJSON_VERSION_MINOR = 4,
+  SIMDJSON_VERSION_MINOR = 6,
   /**
    * The revision (major.minor.REVISION) of simdjson being used.
    */
-  SIMDJSON_VERSION_REVISION = 0
+  SIMDJSON_VERSION_REVISION = 4
 };
 } // namespace simdjson
 

src/arm64.cpp

@@ -108,10 +108,10 @@ simdjson_unused simdjson_inline simd8<bool> must_be_continuation(const simd8<uin
     return is_second_byte ^ is_third_byte ^ is_fourth_byte;
 }
 
-simdjson_inline simd8<bool> must_be_2_3_continuation(const simd8<uint8_t> prev2, const simd8<uint8_t> prev3) {
-    simd8<bool> is_third_byte  = prev2 >= uint8_t(0xe0u);
-    simd8<bool> is_fourth_byte = prev3 >= uint8_t(0xf0u);
-    return is_third_byte ^ is_fourth_byte;
+simdjson_inline simd8<uint8_t> must_be_2_3_continuation(const simd8<uint8_t> prev2, const simd8<uint8_t> prev3) {
+    simd8<uint8_t> is_third_byte  = prev2.saturating_sub(0xe0u-0x80); // Only 111_____ will be >= 0x80
+    simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0xf0u-0x80); // Only 1111____ will be >= 0x80
+    return is_third_byte | is_fourth_byte;
 }
 
 } // unnamed namespace

src/generic/dom_parser_implementation.h

@@ -11,7 +11,7 @@ namespace simdjson {
 namespace SIMDJSON_IMPLEMENTATION {
 namespace {
 
-simdjson_inline simd8<bool> must_be_2_3_continuation(const simd8<uint8_t> prev2, const simd8<uint8_t> prev3);
+simdjson_inline simd8<uint8_t> must_be_2_3_continuation(const simd8<uint8_t> prev2, const simd8<uint8_t> prev3);
 simdjson_inline bool is_ascii(const simd8x64<uint8_t>& input);
 
 } // unnamed namespace

src/generic/stage1/utf8_lookup4_algorithm.h

@@ -107,7 +107,7 @@ using namespace simd;
       const simd8<uint8_t> prev_input, const simd8<uint8_t> sc) {
     simd8<uint8_t> prev2 = input.prev<2>(prev_input);
     simd8<uint8_t> prev3 = input.prev<3>(prev_input);
-    simd8<uint8_t> must23 = simd8<uint8_t>(must_be_2_3_continuation(prev2, prev3));
+    simd8<uint8_t> must23 = must_be_2_3_continuation(prev2, prev3);
     simd8<uint8_t> must23_80 = must23 & uint8_t(0x80);
     return must23_80 ^ sc;
   }

src/generic/stage2/stringparsing.h

@@ -189,8 +189,6 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, u
       dst += backslash_and_quote::BYTES_PROCESSED;
     }
   }
-  /* can't be reached */
-  return nullptr;
 }
 
 simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) {
@@ -236,8 +234,6 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t
       dst += backslash_and_quote::BYTES_PROCESSED;
     }
   }
-  /* can't be reached */
-  return nullptr;
 }
 
 } // namespace stringparsing

src/haswell.cpp

@@ -105,11 +105,10 @@ simdjson_unused simdjson_inline simd8<bool> must_be_continuation(const simd8<uin
   return simd8<int8_t>(is_second_byte | is_third_byte | is_fourth_byte) > int8_t(0);
 }
 
-simdjson_inline simd8<bool> must_be_2_3_continuation(const simd8<uint8_t> prev2, const simd8<uint8_t> prev3) {
-  simd8<uint8_t> is_third_byte  = prev2.saturating_sub(0xe0u-1); // Only 111_____ will be > 0
-  simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0xf0u-1); // Only 1111____ will be > 0
-  // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine.
-  return simd8<int8_t>(is_third_byte | is_fourth_byte) > int8_t(0);
+simdjson_inline simd8<uint8_t> must_be_2_3_continuation(const simd8<uint8_t> prev2, const simd8<uint8_t> prev3) {
+  simd8<uint8_t> is_third_byte  = prev2.saturating_sub(0xe0u-0x80); // Only 111_____ will be >= 0x80
+  simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0xf0u-0x80); // Only 1111____ will be >= 0x80
+  return is_third_byte | is_fourth_byte;
 }
 
 } // unnamed namespace

src/icelake.cpp

@@ -107,11 +107,10 @@ simdjson_unused simdjson_inline simd8<bool> must_be_continuation(const simd8<uin
   return simd8<int8_t>(is_second_byte | is_third_byte | is_fourth_byte) > int8_t(0);
 }
 
-simdjson_inline simd8<bool> must_be_2_3_continuation(const simd8<uint8_t> prev2, const simd8<uint8_t> prev3) {
-  simd8<uint8_t> is_third_byte  = prev2.saturating_sub(0xe0u-1); // Only 111_____ will be > 0
-  simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0xf0u-1); // Only 1111____ will be > 0
-  // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine.
-  return simd8<int8_t>(is_third_byte | is_fourth_byte) > int8_t(0);
+simdjson_inline simd8<uint8_t> must_be_2_3_continuation(const simd8<uint8_t> prev2, const simd8<uint8_t> prev3) {
+  simd8<uint8_t> is_third_byte  = prev2.saturating_sub(0xe0u-0x80); // Only 111_____ will be >= 0x80
+  simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0xf0u-0x80); // Only 1111____ will be >= 0x80
+  return is_third_byte | is_fourth_byte;
 }
 
 } // unnamed namespace

src/ppc64.cpp

@@ -78,11 +78,10 @@ simdjson_unused simdjson_inline simd8<bool> must_be_continuation(const simd8<uin
   return simd8<int8_t>(is_second_byte | is_third_byte | is_fourth_byte) > int8_t(0);
 }
 
-simdjson_inline simd8<bool> must_be_2_3_continuation(const simd8<uint8_t> prev2, const simd8<uint8_t> prev3) {
-  simd8<uint8_t> is_third_byte  = prev2.saturating_sub(0xe0u-1); // Only 111_____ will be > 0
-  simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0xf0u-1); // Only 1111____ will be > 0
-  // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine.
-  return simd8<int8_t>(is_third_byte | is_fourth_byte) > int8_t(0);
+simdjson_inline simd8<uint8_t> must_be_2_3_continuation(const simd8<uint8_t> prev2, const simd8<uint8_t> prev3) {
+  simd8<uint8_t> is_third_byte  = prev2.saturating_sub(0xe0u-0x80); // Only 111_____ will be >= 0x80
+  simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0xf0u-0x80); // Only 1111____ will be >= 0x80
+  return is_third_byte | is_fourth_byte;
 }
 
 } // unnamed namespace

src/westmere.cpp

@@ -109,11 +109,10 @@ simdjson_unused simdjson_inline simd8<bool> must_be_continuation(const simd8<uin
   return simd8<int8_t>(is_second_byte | is_third_byte | is_fourth_byte) > int8_t(0);
 }
 
-simdjson_inline simd8<bool> must_be_2_3_continuation(const simd8<uint8_t> prev2, const simd8<uint8_t> prev3) {
-  simd8<uint8_t> is_third_byte  = prev2.saturating_sub(0xe0u-1); // Only 111_____ will be > 0
-  simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0xf0u-1); // Only 1111____ will be > 0
-  // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine.
-  return simd8<int8_t>(is_third_byte | is_fourth_byte) > int8_t(0);
+simdjson_inline simd8<uint8_t> must_be_2_3_continuation(const simd8<uint8_t> prev2, const simd8<uint8_t> prev3) {
+  simd8<uint8_t> is_third_byte  = prev2.saturating_sub(0xe0u-0x80); // Only 111_____ will be >= 0x80
+  simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0xf0u-0x80); // Only 1111____ will be >= 0x80
+  return is_third_byte | is_fourth_byte;
 }
 
 } // unnamed namespace

tests/dom/basictests.cpp

@@ -66,6 +66,16 @@ namespace number_tests {
     return true;
   }
 
+  bool bomskip() {
+    TEST_START();
+    simdjson::dom::parser parser;
+    simdjson::padded_string docdata = "\xEF\xBB\xBF{\"score\":0.8825149536132812}"_padded;
+    double score;
+    ASSERT_SUCCESS(parser.parse(docdata)["score"].get_double().get(score));
+    ASSERT_EQUAL(score, 0.8825149536132812);
+    TEST_SUCCEED();
+  }
+
   bool issue2017() {
     TEST_START();
     simdjson::dom::parser parser;
@@ -386,7 +396,8 @@ namespace number_tests {
   }
 
   bool run() {
-    return issue2017() &&
+    return bomskip() &&
+           issue2017() &&
            truncated_borderline() &&
            specific_tests() &&
            ground_truth() &&
@@ -441,14 +452,18 @@ namespace parse_api_tests {
     element doc_root1 = parser.parse_into_document(doc, input);
     if(simdjson::to_string(doc_root1) != "[1,2,3]") { return false; }
     //... doc_root1 is a pointer inside doc
-    element doc_root2 = parser.parse_into_document(doc, input);
+    element doc_root2 = parser.load_into_document(doc, TWITTER_JSON);
     //... doc_root2 is a pointer inside doc
-    if(simdjson::to_string(doc_root2) != "[1,2,3]") { return false; }
+    if(uint64_t(doc_root2["search_metadata"]["count"]) != 100) { return false; }
+    if(uint64_t(doc.root()["search_metadata"]["count"]) != 100) { return false; }
+    element doc_root3 = parser.parse_into_document(doc, input);
+    //... doc_root3 is a pointer inside doc
+    if(simdjson::to_string(doc_root3) != "[1,2,3]") { return false; }
 
     // Here let us take moving the document:
     dom::document docm = std::move(doc);
-    element doc_root3 = docm.root();
-    if(simdjson::to_string(doc_root3) != "[1,2,3]") { return false; }
+    element doc_root4 = docm.root();
+    if(simdjson::to_string(doc_root4) != "[1,2,3]") { return false; }
     return true;
   }
 
@@ -1658,6 +1673,7 @@ namespace validate_tests {
     }
     return true;
   }
+
   bool test_validate() {
     std::cout << "Running " << __func__ << std::endl;
     const std::string test = R"({ "foo" : 1, "bar" : [ 1, 2, 3 ], "baz": { "a": 1, "b": 2, "c": 3 } })";
@@ -1666,6 +1682,7 @@ namespace validate_tests {
     }
     return true;
   }
+
   bool test_range() {
     std::cout << "Running " << __func__ << std::endl;
     for(size_t len = 0; len <= 128; len++) {
@@ -1683,6 +1700,7 @@ namespace validate_tests {
     }
     return true;
   }
+
   bool test_issue1169() {
     std::cout << "Running " << __func__ << std::endl;
     std::vector<uint8_t> source(64,' ');
@@ -1693,6 +1711,7 @@ namespace validate_tests {
     }
     return true;
   }
+
   bool test_issue1169_long() {
     std::cout << "Running " << __func__ << std::endl;
     for(size_t len = 1; len <= 128; len++) {
@@ -1702,6 +1721,7 @@ namespace validate_tests {
     }
     return true;
   }
+
   bool test_random() {
     std::cout << "Running " << __func__ << std::endl;
     std::vector<uint8_t> source(64,' ');
@@ -1763,6 +1783,7 @@ namespace minify_tests {
     }
     return true;
   }
+
   // this is meant to test buffer overflows.
   bool test_various_lengths2() {
     std::cout << "Running " << __func__ << std::endl;
@@ -1781,6 +1802,7 @@ namespace minify_tests {
     }
     return true;
   }
+
   bool test_single_quote() {
     std::cout << "Running " << __func__ << std::endl;
     const std::string test = "\"";
@@ -1801,12 +1823,14 @@ namespace minify_tests {
     const std::string minified(R"({"foo":1,"bar":[1,2,0.11111111111111113],"baz":{"a":3.1415926535897936,"b":2,"c":3.141592653589794}})");
     return check_minification(test.c_str(), test.size(), minified.c_str(), minified.size());
   }
+
   bool test_minify_array() {
     std::cout << "Running " << __func__ << std::endl;
     std::string test("[ 1,    2,    3]");
     std::string minified("[1,2,3]");
     return check_minification(test.c_str(), test.size(), minified.c_str(), minified.size());
   }
+
   bool test_minify_object() {
     std::cout << "Running " << __func__ << std::endl;
     std::string test(R"({ "foo   " : 1, "b  ar" : [ 1, 2, 3 ], "baz": { "a": 1, "b": 2, "c": 3 } })");
@@ -1849,6 +1873,7 @@ namespace format_tests {
     s << doc;
     return assert_minified(s);
   }
+
   bool print_minify_parser_parse() {
     std::cout << "Running " << __func__ << std::endl;
     dom::parser parser;
@@ -1868,6 +1893,7 @@ namespace format_tests {
     s << value;
     return assert_minified(s, "1");
   }
+
   bool print_minify_element() {
     std::cout << "Running " << __func__ << std::endl;
     dom::parser parser;
@@ -1887,6 +1913,7 @@ namespace format_tests {
     s << array;
     return assert_minified(s, "[1,2,0.11111111111111113]");
   }
+
   bool print_minify_array() {
     std::cout << "Running " << __func__ << std::endl;
     dom::parser parser;
@@ -1906,6 +1933,7 @@ namespace format_tests {
     s << object;
     return assert_minified(s, R"({"a":3.1415926535897936,"b":2,"c":3.141592653589794})");
   }
+
   bool print_minify_object() {
     std::cout << "Running " << __func__ << std::endl;
     dom::parser parser;
@@ -1925,6 +1953,7 @@ namespace format_tests {
     s << parser.parse(DOCUMENT);
     return assert_minified(s);
   }
+
   bool print_minify_parser_parse_exception() {
     std::cout << "Running " << __func__ << std::endl;
     dom::parser parser;
@@ -1940,6 +1969,7 @@ namespace format_tests {
     s << parser.parse(DOCUMENT)["foo"];
     return assert_minified(s, "1");
   }
+
   bool print_minify_element_result_exception() {
     std::cout << "Running " << __func__ << std::endl;
     dom::parser parser;
@@ -1956,6 +1986,7 @@ namespace format_tests {
     s << value;
     return assert_minified(s, "1");
   }
+
   bool print_minify_element_exception() {
     std::cout << "Running " << __func__ << std::endl;
     dom::parser parser;
@@ -1972,6 +2003,7 @@ namespace format_tests {
     s << parser.parse(DOCUMENT)["bar"].get_array();
     return assert_minified(s, "[1,2,0.11111111111111113]");
   }
+
   bool print_minify_array_result_exception() {
     std::cout << "Running " << __func__ << std::endl;
     dom::parser parser;

tests/dom/document_stream_tests.cpp

@@ -397,6 +397,33 @@ namespace document_stream_tests {
     std::cout << "number of documents " << count << std::endl;
     return count == 1;
   }
+
+  bool skipbom() {
+    std::cout << "Running " << __func__ << std::endl;
+    simdjson::dom::parser parser;
+    auto json = "\xEF\xBB\xBF{\"hello\": \"world\"}"_padded;
+    simdjson::dom::document_stream stream;
+    ASSERT_SUCCESS(parser.parse_many(json).get(stream));
+    size_t count = 0;
+    for (auto doc : stream) {
+        if(doc.error()) {
+          std::cerr << "Unexpected error: " << doc.error() << std::endl;
+          return false;
+        }
+        std::string expected = R"({"hello":"world"})";
+        simdjson::dom::element this_document;
+        ASSERT_SUCCESS(doc.get(this_document));
+
+        std::string answer = simdjson::minify(this_document);
+        if(answer != expected) {
+          std::cout << this_document << std::endl;
+          return false;
+        }
+        count += 1;
+    }
+    std::cout << "number of documents " << count << std::endl;
+    return count == 1;
+  }
 #if SIMDJSON_EXCEPTIONS
   bool single_document_exceptions() {
     std::cout << "Running " << __func__ << std::endl;
@@ -913,7 +940,8 @@ namespace document_stream_tests {
   }
 
   bool run() {
-    return fuzzaccess() &&
+    return skipbom() &&
+           fuzzaccess() &&
            baby_fuzzer() &&
            issue1649() &&
            adversarial_single_document_array() &&

tests/ondemand/ondemand_array_tests.cpp

@@ -6,6 +6,17 @@ using namespace simdjson;
 namespace array_tests {
   using namespace std;
   using simdjson::ondemand::json_type;
+
+  bool document_is_array_treated_as_object() {
+    TEST_START();
+    auto json = R"(   [  {"key" : "value"}  ]  )"_padded;
+    ondemand::parser parser;
+    ondemand::document doc;
+    ASSERT_SUCCESS(parser.iterate(json).get(doc));
+    std::string_view v;
+    ASSERT_ERROR(doc["key"].get_string().get(v), INCORRECT_TYPE);
+    TEST_SUCCEED();
+  }
   bool issue1977() {
     TEST_START();
     auto json = R"([1, 2] foo ])"_padded;
@@ -844,6 +855,7 @@ namespace array_tests {
 
   bool run() {
     return
+           document_is_array_treated_as_object() &&
            issue1977() &&
            issue1876() &&
            issue1742() &&

tests/ondemand/ondemand_document_stream_tests.cpp

@@ -322,6 +322,43 @@ namespace document_stream_tests {
         TEST_SUCCEED();
     }
 
+    bool skipbom() {
+        TEST_START();
+        auto json = "\xEF\xBB\xBF[1,[1,2]] {\"a\":1,\"b\":2} {\"o\":{\"1\":1,\"2\":2}} [1,2,3]"_padded;
+        ondemand::parser parser;
+        ondemand::document_stream stream;
+        ASSERT_SUCCESS(parser.iterate_many(json).get(stream));
+        std::string_view expected[4] = {"[1,[1,2]]", "{\"a\":1,\"b\":2}", "{\"o\":{\"1\":1,\"2\":2}}", "[1,2,3]"};
+        size_t counter{0};
+        auto i = stream.begin();
+        int64_t x;
+
+        ASSERT_EQUAL(i.source(),expected[counter++]);
+        ASSERT_SUCCESS( (*i).at_pointer("/1/1").get(x) );
+        ASSERT_EQUAL(x,2);
+        ++i;
+
+        ASSERT_EQUAL(i.source(),expected[counter++]);
+        simdjson_result<ondemand::document_reference> xxx = *i;
+        ASSERT_SUCCESS( xxx.find_field("a").get(x) );
+        ASSERT_EQUAL(x,1);
+        ++i;
+
+        ASSERT_EQUAL(i.source(),expected[counter++]);
+        ASSERT_SUCCESS( (*i).at_pointer("/o/2").get(x) );
+        ASSERT_EQUAL(x,2);
+        ++i;
+
+        ASSERT_EQUAL(i.source(),expected[counter++]);
+        ASSERT_SUCCESS( (*i).at_pointer("/2").get(x) );
+        ASSERT_EQUAL(x,3);
+        ++i;
+
+        if (i != stream.end()) { return false; }
+
+        TEST_SUCCEED();
+    }
+
     bool atoms_json() {
         TEST_START();
         auto json = R"(5 true 20.3 "string" )"_padded;
@@ -803,6 +840,7 @@ namespace document_stream_tests {
 
     bool run() {
         return
+            skipbom() &&
             issue1977() &&
             string_with_trailing() &&
             uint64_with_trailing() &&

tests/ondemand/ondemand_misc_tests.cpp

@@ -12,10 +12,22 @@ namespace misc_tests {
     ondemand::document doc;
     ASSERT_SUCCESS(parser.iterate(error_phrase).get(doc));
     bool b;
-    ASSERT_SUCCESS( doc.get_bool().get(b));
+    ASSERT_SUCCESS(doc.get_bool().get(b));
     ASSERT_FALSE(b);
     TEST_SUCCEED();
   }
+  bool skipbom() {
+    auto error_phrase = "\xEF\xBB\xBF false"_padded;
+    TEST_START();
+    ondemand::parser parser;
+    ondemand::document doc;
+    ASSERT_SUCCESS(parser.iterate(error_phrase).get(doc));
+    bool b;
+    ASSERT_SUCCESS(doc.get_bool().get(b));
+    ASSERT_FALSE(b);
+    TEST_SUCCEED();
+  }
+
 
   bool issue1981_failure() {
     auto error_phrase = R"(falseA)"_padded;
@@ -593,6 +605,7 @@ namespace misc_tests {
 
   bool run() {
     return
+           skipbom() &&
            issue1981_success() &&
            issue1981_failure() &&
            replacement_char() &&

tests/ondemand/ondemand_number_tests.cpp

@@ -278,6 +278,19 @@ namespace number_tests {
     TEST_SUCCEED();
   }
 
+  bool issue2099() {
+    TEST_START();
+    ondemand::parser parser;
+    auto json = "1000000000.000000001"_padded;
+    ondemand::document doc;
+    ASSERT_SUCCESS(parser.iterate(json).get(doc));
+    ondemand::number number;
+    ASSERT_SUCCESS(doc.get_number().get(number));
+    ASSERT_EQUAL(number.get_number_type(), ondemand::number_type::floating_point_number);
+    ASSERT_EQUAL(number.get_double(), 1e9);
+    TEST_SUCCEED();
+}
+
   bool issue1878() {
     TEST_START();
     ondemand::parser parser;
@@ -415,8 +428,23 @@ namespace number_tests {
     TEST_SUCCEED();
   }
 
+
+  bool issue2093() {
+    TEST_START();
+    ondemand::parser parser;
+    ondemand::document doc;
+    padded_string docdata = R"(0.95000000000000000000)"_padded;
+    ASSERT_SUCCESS(parser.iterate(docdata).get(doc));
+    double d;
+    ASSERT_SUCCESS(doc.get_double().get(d));
+    ASSERT_EQUAL(d, 0.95);
+    TEST_SUCCEED();
+  }
+
   bool run() {
-    return issue2045() &&
+    return issue2099() &&
+           issue2093() &&
+           issue2045() &&
            issue2017() &&
            issue_1898() &&
            issue1878() &&

tests/ondemand/ondemand_object_error_tests.cpp

@@ -137,6 +137,27 @@ namespace object_error_tests {
   }
 
 #ifdef SIMDJSON_DEVELOPMENT_CHECKS
+  bool issue2084() {
+    TEST_START();
+    auto json = R"( {"foo": "bar"} )"_padded;
+    SUBTEST("document->value after access", test_ondemand_doc(json, [&](auto doc) {
+      std::string_view foo;
+      ASSERT_SUCCESS(doc["foo"].get(foo));
+      ondemand::value value;
+      ASSERT_ERROR(doc.get_value().get(value), OUT_OF_ORDER_ITERATION);
+      return true;
+    }));
+    SUBTEST("document->value after access and rewind", test_ondemand_doc(json, [&](auto doc) {
+      std::string_view foo;
+      ASSERT_SUCCESS(doc["foo"].get(foo));
+      doc.rewind();
+      ondemand::value value;
+      ASSERT_SUCCESS(doc.get_value().get(value));
+      return true;
+    }));
+    TEST_SUCCEED();
+  }
+
   bool out_of_order_object_iteration_error() {
     TEST_START();
     auto json = R"([ { "x": 1, "y": 2 } ])"_padded;
@@ -555,6 +576,7 @@ namespace object_error_tests {
            object_lookup_miss_wrong_key_type_error() &&
            object_lookup_miss_next_error() &&
 #ifdef SIMDJSON_DEVELOPMENT_CHECKS
+           issue2084() &&
            out_of_order_object_iteration_error() &&
            out_of_order_top_level_object_iteration_error() &&
            out_of_order_object_index_child_error() &&

tests/ondemand/ondemand_readme_examples.cpp

@@ -1,6 +1,8 @@
 #include "simdjson.h"
 #include "test_ondemand.h"
-
+#if __cpp_lib_optional >= 201606L
+#include <optional>
+#endif
 using namespace std;
 using namespace simdjson;
 using error_code=simdjson::error_code;
@@ -1479,10 +1481,60 @@ bool example1958() {
   }
   return true;
 }
+
+
+bool to_optional() {
+  TEST_START();
+  auto json = R"({ "foo1": "3.1416" } )"_padded;
+  ondemand::parser parser;
+  ondemand::document doc = parser.iterate(json);
+#if __cpp_lib_optional >= 201606L
+  std::optional<std::string> value;
+  ASSERT_SUCCESS(doc["foo1"].get_string(value));
+  std::cout << value.value() << std::endl;
+#else
+  std::string value;
+  ASSERT_SUCCESS(doc["foo1"].get_string(value));
+  std::cout << value << std::endl;
+#endif
+  TEST_SUCCEED();
+}
+
+bool value_raw_json_array() {
+  TEST_START();
+  auto json = R"( [1,2,"fds", {"a":1}, [1,344]] )"_padded;
+  ondemand::parser parser;
+  ondemand::document doc = parser.iterate(json);
+  std::string_view expected[] = {"1", "2", "\"fds\"", "{\"a\":1}", "[1,344]"};
+  size_t counter = 0;
+  for(auto array: doc) {
+    std::string_view raw = array.raw_json();
+    ASSERT_EQUAL(raw, expected[counter++]);
+  }
+  TEST_SUCCEED();
+}
+
+
+bool value_raw_json_object() {
+  TEST_START();
+  auto json = R"( {"key1":1,"key2":2,"key3":"fds", "key4":{"a":1}, "key5":[1,344]} )"_padded;
+  ondemand::parser parser;
+  ondemand::document doc = parser.iterate(json);
+  std::string_view expected[] = {"1", "2", "\"fds\"", "{\"a\":1}", "[1,344]"};
+  size_t counter = 0;
+  for(auto key_value: doc.get_object()) {
+    std::string_view raw = key_value.value().raw_json();
+    ASSERT_EQUAL(raw, expected[counter++]);
+  }
+  TEST_SUCCEED();
+}
+
 #endif
 bool run() {
   return true
 #if SIMDJSON_EXCEPTIONS
+    && to_optional()
+    && value_raw_json_array() && value_raw_json_object()
     && gen_raw1() && gen_raw2() && gen_raw3()
     && at_end()
     && example1956() && example1958()