Disabling again for GCC 7

Patch for GCC 8.

.appveyor.yml

@@ -15,22 +15,22 @@ environment:
     - job_name: VS2019
       CMAKE_ARGS: -A %Platform%
     - job_name: VS2019ARM
-      CMAKE_ARGS: -A ARM64 -DCMAKE_CROSSCOMPILING=1 -D SIMDJSON_GOOGLE_BENCHMARKS=OFF # Does Google Benchmark builds under VS ARM?
+      CMAKE_ARGS: -A ARM64   -DSIMDJSON_DEVELOPER_MODE=ON -DCMAKE_CROSSCOMPILING=1 -D SIMDJSON_GOOGLE_BENCHMARKS=OFF # Does Google Benchmark builds under VS ARM?
     - job_name: VS2017 (Static, No Threads)
       image: Visual Studio 2017
-      CMAKE_ARGS: -A %Platform% -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_ENABLE_THREADS=OFF
+      CMAKE_ARGS: -A %Platform%   -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_ENABLE_THREADS=OFF
       CTEST_ARGS: -LE explicitonly
     - job_name: VS2019 (Win32)
       platform: Win32
-      CMAKE_ARGS: -A %Platform% -DBUILD_SHARED_LIBS=ON -DSIMDJSON_ENABLE_THREADS=ON # This should be the default. Testing anyway.
+      CMAKE_ARGS: -A %Platform%   -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=ON -DSIMDJSON_ENABLE_THREADS=ON # This should be the default. Testing anyway.
       CTEST_ARGS: -LE explicitonly
     - job_name: VS2019 (Win32, No Exceptions)
       platform: Win32
-      CMAKE_ARGS: -A %Platform% -DBUILD_SHARED_LIBS=ON -DSIMDJSON_ENABLE_THREADS=ON -DSIMDJSON_EXCEPTIONS=OFF
+      CMAKE_ARGS: -A %Platform%   -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=ON -DSIMDJSON_ENABLE_THREADS=ON -DSIMDJSON_EXCEPTIONS=OFF
       CTEST_ARGS: -LE explicitonly
     - job_name: VS2015
       image: Visual Studio 2015
-      CMAKE_ARGS: -A %Platform% -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_ENABLE_THREADS=OFF
+      CMAKE_ARGS: -A %Platform%   -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_ENABLE_THREADS=OFF
       CTEST_ARGS: -LE explicitonly
 
 build_script:

.circleci/config.yml

@@ -88,13 +88,13 @@ commands:
     steps:
       - cmake_prep
       - dependency_restore
-      - run: cmake $CMAKE_FLAGS -DCMAKE_INSTALL_PREFIX:PATH=destination -B build . 
+      - run: cmake  -DSIMDJSON_DEVELOPER_MODE=ON $CMAKE_FLAGS -DCMAKE_INSTALL_PREFIX:PATH=destination -B build .
       - dependency_cache # dependencies are produced in the configure step
 
   cmake_build:
     steps:
       - cmake_build_cache
-      - run: cmake --build build
+      - run: cmake  --build build
 
   cmake_test:
     steps:
@@ -191,12 +191,12 @@ jobs:
   sanitize-gcc10:
     description: Build and run tests on GCC 10 and AVX 2 with a cmake sanitize build
     executor: gcc10
-    environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly }
+    environment: { CMAKE_FLAGS: -DCMAKE_BUILD_TYPE=Debug -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly }
     steps: [ cmake_test ]
   sanitize-clang10:
     description: Build and run tests on clang 10 and AVX 2 with a cmake sanitize build
     executor: clang10
-    environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly }
+    environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_NO_FORCE_INLINING=ON -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly }
     steps: [ cmake_test ]
   threadsanitize-gcc10:
     description: Build and run tests on GCC 10 and AVX 2 with a cmake sanitize build
@@ -206,15 +206,7 @@ jobs:
   threadsanitize-clang10:
     description: Build and run tests on clang 10 and AVX 2 with a cmake sanitize build
     executor: clang10
-    environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE_THREADS=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly }
+    environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON  -DSIMDJSON_NO_FORCE_INLINING=ON -DSIMDJSON_SANITIZE_THREADS=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly }
-    steps: [ cmake_test ]
-  nocheckeof-clang10:
-    description: Validate that when __SIMDJSON_CHECK_EOF=0, everything still succeeds
-    environment:
-      CXXFLAGS: -D__SIMDJSON_CHECK_EOF=0
-      CMAKE_BUILD_FLAGS: --target ondemand_tests
-      CTEST_FLAGS: --output-on-failure -R ondemand_
-    executor: clang10
     steps: [ cmake_test ]
   # dynamic
   dynamic-gcc10:
@@ -270,12 +262,12 @@ jobs:
   sanitize-haswell-gcc10:
     description: Build and run tests on GCC 10 and AVX 2 with a cmake sanitize build
     executor: gcc10
-    environment: {  CXXFLAGS: -march=haswell, CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly }
+    environment: {  CXXFLAGS: -march=haswell, CMAKE_FLAGS: -DCMAKE_BUILD_TYPE=Debug -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly }
     steps: [ cmake_test ]
   sanitize-haswell-clang10:
     description: Build and run tests on clang 10 and AVX 2 with a cmake sanitize build
     executor: clang10
-    environment: { CXXFLAGS: -march=haswell, CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly }
+    environment: { CXXFLAGS: -march=haswell, CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON  -DSIMDJSON_NO_FORCE_INLINING=ON -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly }
     steps: [ cmake_test ]
 
 workflows:

.drone.yml

@@ -1,46 +1,4 @@
 kind: pipeline
-name: i386-gcc # we do not support 32-bit systems, but we run tests
-platform: { os: linux, arch: amd64 }
-steps:
-- name: Build and Test
-  image: i386/ubuntu
-  environment:
-    CC: gcc
-    CXX: g++
-    BUILD_FLAGS: -- -j
-    CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF
-    CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly 
-  commands:
-    - scripts/addcmakeppa.sh "$(env -i sh -c '. /etc/os-release; echo $VERSION_CODENAME')"
-    - apt-get install -y g++ cmake gcc git
-    - mkdir build
-    - cd build
-    - cmake $CMAKE_FLAGS ..
-    - cmake --build . $BUILD_FLAGS
-    - ctest $CTEST_FLAGS 
----
-kind: pipeline
-name: i386-clang # we do not support 32-bit systems, but we run tests
-platform: { os: linux, arch: amd64 }
-steps:
-- name: Build and Test
-  image: i386/ubuntu
-  environment:
-    CC: clang-6.0
-    CXX: clang++-6.0
-    BUILD_FLAGS: -- -j
-    CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF
-    CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly 
-  commands:
-    - scripts/addcmakeppa.sh "$(env -i sh -c '. /etc/os-release; echo $VERSION_CODENAME')"
-    - apt-get install -y clang++-6.0 cmake git
-    - mkdir build
-    - cd build
-    - cmake $CMAKE_FLAGS ..
-    - cmake --build . $BUILD_FLAGS
-    - ctest $CTEST_FLAGS 
----
-kind: pipeline
 name: gcc9
 platform: { os: linux, arch: amd64 }
 steps:
@@ -50,7 +8,7 @@ steps:
     CC: gcc
     CXX: g++
     BUILD_FLAGS: -- -j
-    CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback
+    CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=icelake;haswell;westmere;fallback
     CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
   commands:
     - echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list
@@ -61,6 +19,7 @@ steps:
     - cmake $CMAKE_FLAGS ..
     - cmake --build . $BUILD_FLAGS
     - ctest $CTEST_FLAGS -L acceptance -LE per_implementation
+    - SIMDJSON_FORCE_IMPLEMENTATION=icelake ctest $CTEST_FLAGS -L per_implementation
     - SIMDJSON_FORCE_IMPLEMENTATION=haswell ctest $CTEST_FLAGS -L per_implementation
     - SIMDJSON_FORCE_IMPLEMENTATION=westmere ctest $CTEST_FLAGS -L per_implementation
     - SIMDJSON_FORCE_IMPLEMENTATION=fallback ctest $CTEST_FLAGS -L per_implementation
@@ -77,7 +36,7 @@ steps:
     CC: clang-6.0
     CXX: clang++-6.0
     BUILD_FLAGS: -- -j
-    CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback
+    CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=icelake;haswell;westmere;fallback
     CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
   commands:
     - mkdir build
@@ -85,6 +44,7 @@ steps:
     - cmake $CMAKE_FLAGS ..
     - cmake --build . $BUILD_FLAGS
     - ctest $CTEST_FLAGS -L acceptance -LE per_implementation
+    - SIMDJSON_FORCE_IMPLEMENTATION=icelake ctest $CTEST_FLAGS -L per_implementation
     - SIMDJSON_FORCE_IMPLEMENTATION=haswell ctest $CTEST_FLAGS -L per_implementation
     - SIMDJSON_FORCE_IMPLEMENTATION=westmere ctest $CTEST_FLAGS -L per_implementation
     - SIMDJSON_FORCE_IMPLEMENTATION=fallback ctest $CTEST_FLAGS -L per_implementation
@@ -142,7 +102,7 @@ steps:
     CC: gcc
     CXX: g++
     BUILD_FLAGS: -- -j
-    CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback
+    CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=icelake;haswell;westmere;fallback
     CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
   commands:
     - echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list
@@ -153,6 +113,7 @@ steps:
     - cmake $CMAKE_FLAGS ..
     - cmake --build . $BUILD_FLAGS
     - ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L acceptance -LE per_implementation
+    - SIMDJSON_FORCE_IMPLEMENTATION=icelake ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation
     - SIMDJSON_FORCE_IMPLEMENTATION=haswell ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation
     - SIMDJSON_FORCE_IMPLEMENTATION=westmere ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation
     - SIMDJSON_FORCE_IMPLEMENTATION=fallback ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation
@@ -168,7 +129,7 @@ steps:
   environment:
     CC: clang-9
     CXX: clang++-9
-    CMAKE_FLAGS: -DSIMDJSON_SANITIZE=ON -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback
+    CMAKE_FLAGS: -DSIMDJSON_SANITIZE=ON -DSIMDJSON_IMPLEMENTATION=icelake;haswell;westmere;fallback
     BUILD_FLAGS: -- -j
     CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
   commands:
@@ -177,6 +138,7 @@ steps:
     - cmake $CMAKE_FLAGS ..
     - cmake --build . $BUILD_FLAGS
     - ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L acceptance -LE per_implementation
+    - SIMDJSON_FORCE_IMPLEMENTATION=icelake ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation
     - SIMDJSON_FORCE_IMPLEMENTATION=haswell ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation
     - SIMDJSON_FORCE_IMPLEMENTATION=westmere ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation
     - SIMDJSON_FORCE_IMPLEMENTATION=fallback ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation

.github/workflows/alpine.yml

@@ -1,12 +1,6 @@
 name: Alpine Linux
 
-on:
+on: [push, pull_request]
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
 
 jobs:
   ubuntu-build:
@@ -15,8 +9,8 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
-      - uses: actions/cache@v2
+      - uses: actions/cache@v3
         with:
           path: dependencies/.cache
           key: ${{ hashFiles('dependencies/CMakeLists.txt') }}

.github/workflows/cifuzz.yml

@@ -0,0 +1,24 @@
+name: CIFuzz
+on: [pull_request]
+jobs:
+  Fuzzing:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Build Fuzzers
+      id: build
+      uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master
+      with:
+        oss-fuzz-project-name: 'simdjson'
+        dry-run: false
+    - name: Run Fuzzers
+      uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master
+      with:
+        oss-fuzz-project-name: 'simdjson'
+        fuzz-seconds: 600
+        dry-run: false
+    - name: Upload Crash
+      uses: actions/upload-artifact@v1
+      if: failure() && steps.build.outcome == 'success'
+      with:
+        name: artifacts
+        path: ./out/artifacts

.github/workflows/fix-trailing-whitespace.yml

@@ -1,15 +1,12 @@
 name: Detect trailing whitespace
 
-on:
+on: [push, pull_request]
-  pull_request:
-    branches:
-      - master
 
 jobs:
   whitespace:
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Remove whitespace and check the diff
         run: |
           set -eu

.github/workflows/fuzzers.yml

@@ -37,14 +37,14 @@ jobs:
         chmod +x llvm.sh
         sudo ./llvm.sh $CLANGVERSION
 
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
 
-    - uses: actions/cache@v2
+    - uses: actions/cache@v3
       with:
         path: dependencies/.cache
         key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
 
-    - uses: actions/cache@v2
+    - uses: actions/cache@v3
       id: cache-corpus
       with:
         path: out/

.github/workflows/macos-11.yml

@@ -0,0 +1,34 @@
+name: Macos (Xcode 11)
+
+on: [push, pull_request]
+
+jobs:
+  macos-build:
+    if: >-
+      ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
+      ! contains(toJSON(github.event.commits.*.message), '[skip github]')
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
+        with:
+          path: dependencies/.cache
+          key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
+      - name: Use cmake
+        run: |
+          xcversion select 11.7
+          mkdir builddebug &&
+          cd builddebug &&
+          cmake -DCMAKE_BUILD_TYPE=Debug -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF ..  &&
+          cmake --build .   &&
+          ctest -j --output-on-failure -LE explicitonly   &&
+          cd .. &&
+          mkdir build &&
+          cd build &&
+          cmake  -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination ..  &&
+          cmake --build .   &&
+          ctest -j --output-on-failure -LE explicitonly   &&
+          cmake --install . &&
+          echo -e '#include <simdjson.h>\nint main(int argc,char**argv) {simdjson::dom::parser parser;simdjson::dom::element tweets = parser.load(argv[1]); }' > tmp.cpp && c++ -Idestination/include -Ldestination/lib -std=c++17 -Wl,-rpath,destination/lib -o linkandrun tmp.cpp -lsimdjson && ./linkandrun jsonexamples/twitter.json &&
+          cd ../tests/installation_tests/find &&
+          mkdir build && cd build && cmake -DCMAKE_INSTALL_PREFIX:PATH=../../../build/destination .. &&  cmake --build .

.github/workflows/macos.yml

@@ -0,0 +1,33 @@
+name: Macos
+
+on: [push, pull_request]
+
+jobs:
+  macos-build:
+    if: >-
+      ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
+      ! contains(toJSON(github.event.commits.*.message), '[skip github]')
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
+        with:
+          path: dependencies/.cache
+          key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
+      - name: Use cmake
+        run: |
+          mkdir builddebug &&
+          cd builddebug &&
+          cmake -DCMAKE_BUILD_TYPE=Debug -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF ..  &&
+          cmake --build .   &&
+          ctest -j --output-on-failure -LE explicitonly   &&
+          cd .. &&
+          mkdir build &&
+          cd build &&
+          cmake  -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination ..  &&
+          cmake --build .   &&
+          ctest -j --output-on-failure -LE explicitonly   &&
+          cmake --install . &&
+          echo -e '#include <simdjson.h>\nint main(int argc,char**argv) {simdjson::dom::parser parser;simdjson::dom::element tweets = parser.load(argv[1]); }' > tmp.cpp && c++ -Idestination/include -Ldestination/lib -std=c++17 -Wl,-rpath,destination/lib -o linkandrun tmp.cpp -lsimdjson && ./linkandrun jsonexamples/twitter.json &&
+          cd ../tests/installation_tests/find &&
+          mkdir build && cd build && cmake -DCMAKE_INSTALL_PREFIX:PATH=../../../build/destination .. &&  cmake --build .

.github/workflows/mingw-ci.yml

@@ -1,68 +0,0 @@
-name: MinGW32-CI
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
-
-# Important: scoop will either install 32-bit GCC or 64-bit GCC, not both.
-
-# It is important to build static libraries because cmake is not smart enough under Windows/mingw to take care of the path. So
-# with a dynamic library, you could get failures due to the fact that the EXE can't find its DLL.
-
-jobs:
-  ci:
-    if: >-
-      ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
-      ! contains(toJSON(github.event.commits.*.message), '[skip github]')
-    name: windows-gcc
-    runs-on: windows-2016
-
-    env:
-      CMAKE_GENERATOR: Ninja # This is critical, try ' cmake  -GNinja-DBUILD_SHARED_LIBS=OFF .. ' if using the command line
-      CC: gcc
-      CXX: g++
-
-    steps: # To reproduce what is below, start a powershell with administrative rights, using scoop *is* a good idea
-      - uses: actions/checkout@v2
-
-      - uses: actions/cache@v2 # we cache the scoop setup with 32-bit GCC
-        id: cache
-        with:
-          path: |
-            C:\ProgramData\scoop
-          key: scoop32 # static key: should be good forever
-
-      - uses: actions/cache@v2
-        with:
-          path: dependencies/.cache
-          key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
-
-      - name: Setup Windows # This should almost never run if the cache works.
-        if: steps.cache.outputs.cache-hit != 'true'
-        shell: powershell
-        run: |
-          Invoke-Expression (New-Object System.Net.WebClient).DownloadString('https://get.scoop.sh')
-          scoop install sudo --global
-          sudo scoop install git --global 
-          sudo scoop install ninja --global
-          sudo scoop install cmake --global
-          sudo scoop install gcc --arch 32bit --global
-          $env:path
-          Write-Host 'Everything has been installed, you are good!'
-      - name: Build and Test 32-bit x86
-        shell: powershell
-        run: |
-          $ENV:PATH="C:\ProgramData\scoop\shims;C:\ProgramData\scoop\apps\gcc\current\bin;C:\ProgramData\scoop\apps\ninja\current;$ENV:PATH"
-          g++ --version
-          cmake --version
-          ninja --version
-          git --version
-          mkdir build32
-          cd build32
-          cmake -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_ENABLE_THREADS=OFF ..
-          cmake --build . --target acceptance_tests --verbose
-          ctest -L acceptance --output-on-failure

.github/workflows/mingw64-ci.yml

@@ -1,74 +0,0 @@
-name: MinGW64-CI
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
-
-# Important: scoop will either install 32-bit GCC or 64-bit GCC, not both.
-
-# It is important to build static libraries because cmake is not smart enough under Windows/mingw to take care of the path. So
-# with a dynamic library, you could get failures due to the fact that the EXE can't find its DLL.
-
-jobs:
-  ci:
-    if: >-
-      ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
-      ! contains(toJSON(github.event.commits.*.message), '[skip github]')
-    name: windows-gcc
-    runs-on: windows-2016
-
-    env:
-      CMAKE_GENERATOR: Ninja # This is critical, try ' cmake  -GNinja-DBUILD_SHARED_LIBS=OFF .. ' if using the command line
-      CC: gcc
-      CXX: g++
-
-    steps: # To reproduce what is below, start a powershell with administrative rights, using scoop *is* a good idea
-      - uses: actions/checkout@v2
-
-      - uses: actions/cache@v2 # we cache the scoop setup with 64-bit GCC
-        id: cache
-        with:
-          path: |
-            C:\ProgramData\scoop
-          key: scoop64 # static key: should be good forever
-
-      - uses: actions/cache@v2
-        with:
-          path: dependencies/.cache
-          key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
-
-      - name: Setup Windows # This should almost never run if the cache works.
-        if: steps.cache.outputs.cache-hit != 'true'
-        shell: powershell
-        run: |
-          Invoke-Expression (New-Object System.Net.WebClient).DownloadString('https://get.scoop.sh')
-          scoop install sudo --global
-          sudo scoop install git --global
-          sudo scoop install ninja --global
-          sudo scoop install cmake --global
-          sudo scoop install gcc --arch 64bit --global
-          $env:path
-          Write-Host 'Everything has been installed, you are good!'
-      - name: Build and Test 64-bit x64
-        shell: powershell
-        run: |
-          $ENV:PATH="C:\ProgramData\scoop\shims;C:\ProgramData\scoop\apps\gcc\current\bin;C:\ProgramData\scoop\apps\ninja\current;$ENV:PATH"
-          g++ --version
-          cmake --version
-          ninja --version
-          git --version
-          mkdir build64
-          cd build64
-          cmake -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_ENABLE_THREADS=OFF ..
-          cmake --build . --target acceptance_tests --verbose
-          ctest -L acceptance --output-on-failure
-          cd ..
-          mkdir build64debug
-          cd build64debug
-          cmake -DSIMDJSON_DEVELOPER_MODE=ON -DCMAKE_BUILD_TYPE=Debug -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_ENABLE_THREADS=OFF ..
-          cmake --build . --target acceptance_tests --verbose
-          ctest -L acceptance --output-on-failure

.github/workflows/msys2-clang.yml

@@ -1,12 +1,7 @@
 name: MSYS2-CLANG-CI
 
-on:
+on: [push, pull_request]
-  push:
+
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
 
 jobs:
   windows-mingw:
@@ -22,21 +17,15 @@ jobs:
           - msystem: "MINGW64"
             install: mingw-w64-x86_64-libxml2 mingw-w64-x86_64-cmake mingw-w64-x86_64-ninja mingw-w64-x86_64-clang
             type: Release
-          - msystem: "MINGW32"
-            install: mingw-w64-i686-libxml2 mingw-w64-i686-cmake mingw-w64-i686-ninja mingw-w64-i686-clang
-            type: Release
           - msystem: "MINGW64"
             install: mingw-w64-x86_64-libxml2 mingw-w64-x86_64-cmake mingw-w64-x86_64-ninja mingw-w64-x86_64-clang
             type: Debug
-          - msystem: "MINGW32"
-            install: mingw-w64-i686-libxml2 mingw-w64-i686-cmake mingw-w64-i686-ninja mingw-w64-i686-clang
-            type: Debug
     env:
       CMAKE_GENERATOR: Ninja
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
-      - uses: actions/cache@v2
+      - uses: actions/cache@v3
         with:
           path: dependencies/.cache
           key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
@@ -49,6 +38,6 @@ jobs:
         run: |
           mkdir build
           cd build
-          cmake -DSIMDJSON_DEVELOPER_MODE=ON -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_BUILD_TYPE=${{ matrix.type }} -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_DO_NOT_USE_THREADS_NO_MATTER_WHAT=ON ..
+          cmake -DSIMDJSON_DEVELOPER_MODE=ON -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_BUILD_TYPE=${{ matrix.type }} -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_DO_NOT_USE_THREADS_NO_MATTER_WHAT=ON ..
           cmake --build . --verbose
           ctest -j4 --output-on-failure -LE explicitonly

.github/workflows/msys2.yml

@@ -1,12 +1,6 @@
 name: MSYS2-CI
 
-on:
+on: [push, pull_request]
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
 
 jobs:
   windows-mingw:
@@ -25,21 +19,15 @@ jobs:
           - msystem: "MINGW64"
             install: mingw-w64-x86_64-cmake mingw-w64-x86_64-ninja mingw-w64-x86_64-gcc
             type: Release
-          - msystem: "MINGW32"
-            install: mingw-w64-i686-cmake mingw-w64-i686-ninja mingw-w64-i686-gcc
-            type: Release
           - msystem: "MINGW64"
             install: mingw-w64-x86_64-cmake mingw-w64-x86_64-ninja mingw-w64-x86_64-gcc
             type: Debug
-          - msystem: "MINGW32"
-            install: mingw-w64-i686-cmake mingw-w64-i686-ninja mingw-w64-i686-gcc
-            type: Debug
     env:
       CMAKE_GENERATOR: Ninja
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
-      - uses: actions/cache@v2
+      - uses: actions/cache@v3
         with:
           path: dependencies/.cache
           key: ${{ hashFiles('dependencies/CMakeLists.txt') }}

.github/workflows/ubuntu18-checkperf.yml

@@ -15,8 +15,8 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: ubuntu-18.04
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
-      - uses: actions/cache@v2
+      - uses: actions/cache@v3
         with:
           path: dependencies/.cache
           key: ${{ hashFiles('dependencies/CMakeLists.txt') }}

.github/workflows/ubuntu18-oldclang.yml

@@ -0,0 +1,31 @@
+name: Ubuntu 18.04 CI (LLVM 7)
+
+on: [push, pull_request]
+
+jobs:
+  ubuntu-build:
+    if: >-
+      ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
+      ! contains(toJSON(github.event.commits.*.message), '[skip github]')
+    runs-on: ubuntu-18.04
+
+    env:
+      CC: clang-7
+      CXX: clang++-7
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
+        with:
+          path: dependencies/.cache
+          key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
+      - name: install clang 7
+        run: |
+          sudo apt update
+          sudo apt install clang-7
+      - name: Use cmake
+        run: |
+          mkdir build &&
+          cd build &&
+          cmake  -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination ..  &&
+          cmake --build .   &&
+          ctest -j --output-on-failure -LE explicitonly

.github/workflows/ubuntu18-threadsani.yml

@@ -1,12 +1,7 @@
 name: Ubuntu 18.04 CI (GCC 7) with Thread Sanitizer
 
-on:
+on: [push, pull_request]
-  push:
+
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
 
 jobs:
   ubuntu-build:
@@ -15,8 +10,8 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: ubuntu-18.04
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
-      - uses: actions/cache@v2
+      - uses: actions/cache@v3
         with:
           path: dependencies/.cache
           key: ${{ hashFiles('dependencies/CMakeLists.txt') }}

.github/workflows/ubuntu18.yml

@@ -1,12 +1,6 @@
 name: Ubuntu 18.04 CI (GCC 7)
 
-on:
+on: [push, pull_request]
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
 
 jobs:
   ubuntu-build:
@@ -15,8 +9,8 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: ubuntu-18.04
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
-      - uses: actions/cache@v2
+      - uses: actions/cache@v3
         with:
           path: dependencies/.cache
           key: ${{ hashFiles('dependencies/CMakeLists.txt') }}

.github/workflows/ubuntu20-checkperf.yml

@@ -15,8 +15,8 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
-      - uses: actions/cache@v2
+      - uses: actions/cache@v3
         with:
           path: dependencies/.cache
           key: ${{ hashFiles('dependencies/CMakeLists.txt') }}

.github/workflows/ubuntu20-gcc8.yml

@@ -0,0 +1,38 @@
+name: Ubuntu 20.04 CI (GCC 8)
+
+on: [push, pull_request]
+
+jobs:
+  ubuntu-build:
+    if: >-
+      ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
+      ! contains(toJSON(github.event.commits.*.message), '[skip github]')
+    runs-on: ubuntu-20.04
+    env:
+      CXX: g++-8
+      CC: gcc-8
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
+        with:
+          path: dependencies/.cache
+          key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
+      - name: Install GCC 8
+        run: sudo apt-get install -y g++-8
+      - name: Use cmake
+        run: |
+          mkdir builddebug &&
+          cd builddebug &&
+          cmake -DCMAKE_BUILD_TYPE=Debug -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF ..  &&
+          cmake --build .   &&
+          ctest -j --output-on-failure -LE explicitonly   &&
+          cd .. &&
+          mkdir build &&
+          cd build &&
+          cmake  -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination ..  &&
+          cmake --build .   &&
+          ctest -j --output-on-failure -LE explicitonly   &&
+          cmake --install . &&
+          echo -e '#include <simdjson.h>\nint main(int argc,char**argv) {simdjson::dom::parser parser;simdjson::dom::element tweets = parser.load(argv[1]); }' > tmp.cpp && c++ -Idestination/include -Ldestination/lib -std=c++17 -Wl,-rpath,destination/lib -o linkandrun tmp.cpp -lsimdjson && ./linkandrun jsonexamples/twitter.json &&
+          cd ../tests/installation_tests/find &&
+          mkdir build && cd build && cmake -DCMAKE_INSTALL_PREFIX:PATH=../../../build/destination .. &&  cmake --build .

.github/workflows/ubuntu20-noexcept.yml

@@ -1,12 +1,6 @@
 name: Ubuntu 20.04 CI (GCC 9) without exceptions
 
-on:
+on: [push, pull_request]
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
 
 jobs:
   ubuntu-build:
@@ -15,8 +9,8 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
-      - uses: actions/cache@v2
+      - uses: actions/cache@v3
         with:
           path: dependencies/.cache
           key: ${{ hashFiles('dependencies/CMakeLists.txt') }}

.github/workflows/ubuntu20-nothread.yml

@@ -1,12 +1,6 @@
-name: Ubuntu 20.04 CI (GCC 9) without threads
+name: Ubuntu 20.04 CI (GCC 9) Without Threads
 
-on:
+on: [push, pull_request]
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
 
 jobs:
   ubuntu-build:
@@ -15,8 +9,8 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
-      - uses: actions/cache@v2
+      - uses: actions/cache@v3
         with:
           path: dependencies/.cache
           key: ${{ hashFiles('dependencies/CMakeLists.txt') }}

.github/workflows/ubuntu20-sani.yml

@@ -0,0 +1,23 @@
+name: Ubuntu 20.04 CI (GCC 9) With Memory Sanitizer
+
+on: [push, pull_request]
+
+jobs:
+  ubuntu-build:
+    if: >-
+      ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
+      ! contains(toJSON(github.event.commits.*.message), '[skip github]')
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
+        with:
+          path: dependencies/.cache
+          key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
+      - name: Use cmake
+        run: |
+          mkdir builddebug &&
+          cd builddebug &&
+          cmake -DSIMDJSON_SANITIZE=ON -DCMAKE_BUILD_TYPE=Debug -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF ..  &&
+          cmake --build .   &&
+          ctest -j --output-on-failure -LE explicitonly

.github/workflows/ubuntu20-threadsani.yml

@@ -1,12 +1,6 @@
 name: Ubuntu 20.04 CI (GCC 9) with Thread Sanitizer
 
-on:
+on: [push, pull_request]
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
 
 jobs:
   ubuntu-build:
@@ -15,8 +9,8 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
-      - uses: actions/cache@v2
+      - uses: actions/cache@v3
         with:
           path: dependencies/.cache
           key: ${{ hashFiles('dependencies/CMakeLists.txt') }}

.github/workflows/ubuntu20.yml

@@ -1,12 +1,6 @@
 name: Ubuntu 20.04 CI (GCC 9)
 
-on:
+on: [push, pull_request]
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
 
 jobs:
   ubuntu-build:
@@ -15,8 +9,8 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
-      - uses: actions/cache@v2
+      - uses: actions/cache@v3
         with:
           path: dependencies/.cache
           key: ${{ hashFiles('dependencies/CMakeLists.txt') }}

.github/workflows/ubuntu22-clang13.yml

@@ -0,0 +1,25 @@
+name: Ubuntu 22.04 CI (CLANG 13)
+
+on: [push, pull_request]
+
+jobs:
+  ubuntu-build:
+    if: >-
+      ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
+      ! contains(toJSON(github.event.commits.*.message), '[skip github]')
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
+        with:
+          path: dependencies/.cache
+          key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
+      - name: Install clang++-13
+        run: sudo apt-get install -y clang++-13
+      - name: Use cmake
+        run: |
+          mkdir build &&
+          cd build &&
+          CXX=clang++-13 cmake -DSIMDJSON_DEVELOPER_MODE=ON ..  &&
+          cmake --build .   &&
+          ctest -j --output-on-failure -LE explicitonly

.github/workflows/ubuntu22-gcc12.yml

@@ -0,0 +1,25 @@
+name: Ubuntu 22.04 CI (GCC 12)
+
+on: [push, pull_request]
+
+jobs:
+  ubuntu-build:
+    if: >-
+      ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
+      ! contains(toJSON(github.event.commits.*.message), '[skip github]')
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
+        with:
+          path: dependencies/.cache
+          key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
+      - name: Install gcc12
+        run: sudo apt-get install -y g++-12
+      - name: Use cmake
+        run: |
+          mkdir build &&
+          cd build &&
+          CXX=g++-12 cmake -DSIMDJSON_DEVELOPER_MODE=ON ..  &&
+          cmake --build .   &&
+          ctest -j --output-on-failure -LE explicitonly

.github/workflows/ubuntu22.yml

@@ -0,0 +1,33 @@
+name: Ubuntu 22.04 CI (GCC 11)
+
+on: [push, pull_request]
+
+jobs:
+  ubuntu-build:
+    if: >-
+      ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
+      ! contains(toJSON(github.event.commits.*.message), '[skip github]')
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
+        with:
+          path: dependencies/.cache
+          key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
+      - name: Use cmake
+        run: |
+          mkdir builddebug &&
+          cd builddebug &&
+          cmake -DCMAKE_BUILD_TYPE=Debug -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF ..  &&
+          cmake --build .   &&
+          ctest -j --output-on-failure -LE explicitonly   &&
+          cd .. &&
+          mkdir build &&
+          cd build &&
+          cmake  -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination ..  &&
+          cmake --build .   &&
+          ctest -j --output-on-failure -LE explicitonly   &&
+          cmake --install . &&
+          echo -e '#include <simdjson.h>\nint main(int argc,char**argv) {simdjson::dom::parser parser;simdjson::dom::element tweets = parser.load(argv[1]); }' > tmp.cpp && c++ -Idestination/include -Ldestination/lib -std=c++17 -Wl,-rpath,destination/lib -o linkandrun tmp.cpp -lsimdjson && ./linkandrun jsonexamples/twitter.json &&
+          cd ../tests/installation_tests/find &&
+          mkdir build && cd build && cmake -DCMAKE_INSTALL_PREFIX:PATH=../../../build/destination .. &&  cmake --build .

.github/workflows/vs16-clang-ci.yml

@@ -1,37 +0,0 @@
-name: VS16-CLANG-CI
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
-
-jobs:
-  ci:
-    if: >-
-      ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
-      ! contains(toJSON(github.event.commits.*.message), '[skip github]')
-    name: windows-vs16
-    runs-on: windows-latest
-    steps:
-    - uses: actions/checkout@v2
-    - uses: actions/cache@v2
-      with:
-        path: dependencies/.cache
-        key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
-    - name: 'Run CMake with VS16 Clang'
-      uses: lukka/run-cmake@v3
-      with:
-        cmakeListsOrSettingsJson: CMakeListsTxtAdvanced
-        cmakeListsTxtPath: '${{ github.workspace }}/CMakeLists.txt'
-        buildDirectory: "${{ github.workspace }}/../../_temp/windows"
-        cmakeBuildType: Release
-        buildWithCMake: true
-        cmakeAppendedArgs:  -T ClangCL -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF
-        buildWithCMakeArgs: --config Release
-    
-    - name: 'Run CTest'
-      run: ctest -C Release -LE explicitonly  --output-on-failure
-      working-directory: "${{ github.workspace }}/../../_temp/windows"

.github/workflows/vs16-ninja-ci.yml

@@ -1,51 +0,0 @@
-name: VS16-Ninja-CI
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
-
-jobs:
-  ci:
-    if: >-
-      ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
-      ! contains(toJSON(github.event.commits.*.message), '[skip github]')
-    name: windows-vs16
-    runs-on: windows-latest
-    steps:
-    - uses: actions/checkout@v2
-    - uses: actions/cache@v2
-      with:
-        path: dependencies/.cache
-        key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
-    - name: 'Run CMake with VS16'
-      uses: lukka/run-cmake@v2
-      with:
-        cmakeListsOrSettingsJson: CMakeListsTxtAdvanced
-        cmakeListsTxtPath: '${{ github.workspace }}/CMakeLists.txt'
-        buildDirectory: "${{ github.workspace }}/../../_temp/windows"
-        cmakeBuildType: Release
-        buildWithCMake: true
-        cmakeAppendedArgs:  -G Ninja -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF
-        buildWithCMakeArgs: --config Release
-    - name: 'Run CTest'
-      run: ctest -C Release  -LE explicitonly  --output-on-failure
-      working-directory: "${{ github.workspace }}/../../_temp/windows"
-    - name: 'Install with CMake'
-      uses: lukka/run-cmake@v3
-      with:
-        cmakeListsTxtPath: '${{ github.workspace }}/CMakeLists.txt'
-        buildWithCMakeArgs: '--target install'
-    - name: 'Test Installation with CMake'
-      uses: lukka/run-cmake@v3
-      with:
-        cmakeListsOrSettingsJson: CMakeListsTxtAdvanced
-        cmakeListsTxtPath: '${{ github.workspace }}/tests/installation_tests/find/CMakeLists.txt'
-        cmakeBuildType: Release
-        buildWithCMake: true
-        buildDirectory: '${{ github.workspace }}/tests/installation_tests/find/buildDirectory'
-        cmakeAppendedArgs:  -G Ninja
-        buildWithCMakeArgs: '--config Release --verbose'

.github/workflows/vs17-arm-ci.yml

@@ -1,10 +1,10 @@
-name: VS16-ARM-CI
+name: VS17-ARM-CI
 
 on: [push, pull_request]
 
 jobs:
    ci:
-     name: windows-vs16
+     name: windows-vs17
      runs-on: windows-latest
      strategy:
        fail-fast: false
@@ -14,7 +14,7 @@ jobs:
            - {arch: ARM64}
      steps:
        - name: checkout
-         uses: actions/checkout@v2
+         uses: actions/checkout@v3
        - name: Use cmake
          run: |
            cmake -A ${{ matrix.arch }} -DCMAKE_CROSSCOMPILING=1 -DSIMDJSON_DEVELOPER_MODE=ON -D SIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_EXCEPTIONS=OFF -B build  &&

.github/workflows/vs17-ci.yml

@@ -1,34 +1,28 @@
-name: VS16-CI
+name: VS17-CI
 
-on:
+on: [push, pull_request]
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
 
 jobs:
   ci:
     if: >-
       ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
-    name: windows-vs16
+    name: windows-vs17
     runs-on: windows-latest
     strategy:
       fail-fast: false
       matrix:
         include:
-          - {gen: Visual Studio 16 2019, arch: Win32, static: ON}
+          - {gen: Visual Studio 17 2022, arch: Win32, shared: ON}
-          - {gen: Visual Studio 16 2019, arch: Win32, static: OFF}
+          - {gen: Visual Studio 17 2022, arch: Win32, shared: OFF}
-          - {gen: Visual Studio 16 2019, arch: x64, static: ON}
+          - {gen: Visual Studio 17 2022, arch: x64, shared: ON}
-          - {gen: Visual Studio 16 2019, arch: x64, static: OFF}
+          - {gen: Visual Studio 17 2022, arch: x64, shared: OFF}
     steps:
     - name: checkout
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
     - name: Configure
       run: |
-        cmake -G "${{matrix.gen}}" -A ${{matrix.arch}} -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_BUILD_STATIC=${{matrix.static}} -B build
+        cmake -G "${{matrix.gen}}" -A ${{matrix.arch}} -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -DBUILD_SHARED_LIBS=${{matrix.shared}} -B build
     - name: Build Debug
       run: cmake --build build --config Debug --verbose
     - name: Build Release

.github/workflows/vs17-clang-ci.yml

@@ -1,34 +1,25 @@
-name: VS15-CI
+name: VS17-CLANG-CI
 
-on:
+on: [push, pull_request]
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
 
 jobs:
   ci:
     if: >-
       ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
-    name: windows-vs15
+    name: windows-vs17
-    runs-on: windows-2016
+    runs-on: windows-latest
     strategy:
       fail-fast: false
       matrix:
         include:
-          - {gen: Visual Studio 15 2017, arch: Win32, static: ON}
+          - {gen: Visual Studio 17 2022, arch: x64}
-          - {gen: Visual Studio 15 2017, arch: Win32, static: OFF}
-          - {gen: Visual Studio 15 2017, arch: x64, static: ON}
-          - {gen: Visual Studio 15 2017, arch: x64, static: OFF}
     steps:
     - name: checkout
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
     - name: Configure
       run: |
-        cmake -G "${{matrix.gen}}" -A ${{matrix.arch}} -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_BUILD_STATIC=${{matrix.static}} -B build
+        cmake -G "${{matrix.gen}}" -A ${{matrix.arch}}  -T ClangCL -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -B build
     - name: Build Debug
       run: cmake --build build --config Debug --verbose
     - name: Build Release

.github/workflows/vs17-noexcept-ci.yml

@@ -1,24 +1,18 @@
-name: VS16-NoExcept-CI
+name: VS17-NoExcept-CI
 
-on:
+on: [push, pull_request]
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
 
 jobs:
   ci:
-    name: windows-vs16
+    name: windows-vs17
     runs-on: windows-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
-    - uses: actions/cache@v2
+    - uses: actions/cache@v3
       with:
         path: dependencies/.cache
         key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
-    - name: 'Run CMake with VS16'
+    - name: 'Run CMake with VS17'
       uses: lukka/run-cmake@v3
       with:
         cmakeListsOrSettingsJson: CMakeListsTxtAdvanced

.vscode/settings.json

@@ -17,7 +17,6 @@
         "__errc": "cpp",
         "__functional_base": "cpp",
         "__hash_table": "cpp",
-        "__locale": "cpp",
         "__mutex_base": "cpp",
         "__node_handle": "cpp",
         "__nullptr": "cpp",
@@ -85,6 +84,8 @@
         "utility": "cpp",
         "valarray": "cpp",
         "vector": "cpp",
-        "*.ipp": "cpp"
+        "*.ipp": "cpp",
+        "__functional_base_03": "cpp",
+        "filesystem": "cpp"
     }
 }

CMakeLists.txt

@@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.14)
 project(
     simdjson
     # The version number is modified by tools/release.py
-    VERSION 0.9.1
+    VERSION 2.0.4
     DESCRIPTION "Parsing gigabytes of JSON per second"
     HOMEPAGE_URL "https://simdjson.org/"
     LANGUAGES CXX C
@@ -20,8 +20,8 @@ string(
 # ---- Options, variables ----
 
 # These version numbers are modified by tools/release.py
-set(SIMDJSON_LIB_VERSION "8.0.0" CACHE STRING "simdjson library version")
+set(SIMDJSON_LIB_VERSION "11.0.0" CACHE STRING "simdjson library version")
-set(SIMDJSON_LIB_SOVERSION "8" CACHE STRING "simdjson library soversion")
+set(SIMDJSON_LIB_SOVERSION "11" CACHE STRING "simdjson library soversion")
 
 option(SIMDJSON_ENABLE_THREADS "Link with thread support" ON)
 
@@ -49,19 +49,10 @@ endif()
 if(is_top_project)
   option(SIMDJSON_DEVELOPER_MODE "Enable targets for developing simdjson" OFF)
   option(BUILD_SHARED_LIBS "Build simdjson as a shared library" OFF)
-
-  if("$ENV{CI}")
-    set(SIMDJSON_DEVELOPER_MODE ON CACHE INTERNAL "")
-  endif()
 endif()
 
 include(cmake/handle-deprecations.cmake)
-
+include(cmake/developer-options.cmake)
-if(SIMDJSON_DEVELOPER_MODE)
-  include(cmake/developer-options.cmake)
-else()
-  message(STATUS "Building only the library. Advanced users may want to turn SIMDJSON_DEVELOPER_MODE to ON, e.g., via -D SIMDJSON_DEVELOPER_MODE=ON.")
-endif()
 
 # ---- simdjson library ----
 
@@ -180,6 +171,7 @@ endif()
 # ---- Developer mode extras ----
 
 if(NOT SIMDJSON_DEVELOPER_MODE)
+  message(STATUS "Building only the library. Advanced users may want to turn SIMDJSON_DEVELOPER_MODE to ON, e.g., via -D SIMDJSON_DEVELOPER_MODE=ON.")
   return()
 elseif(NOT is_top_project)
   message(AUTHOR_WARNING "Developer mode is intended for developers of simdjson")

CONTRIBUTING.md

@@ -62,13 +62,13 @@ Pull Requests
 
 Pull requests are always invited. However, we ask that you follow these guidelines:
 
-- It is wiser to discuss your ideas first as part of an issue before you start coding. If you omit this step and code first, be prepare to have your code receive scrutiny and be dropped.
+- It is wise to discuss your ideas first as part of an issue before you start coding. If you omit this step and code first, be prepared to have your code receive scrutiny and be dropped.
-- Users should provide a rationale for their changes. Does it improve performance? Does it add a feature? Does it improve maintainability? Does fix a bug? This must be explicitly stated as part of the pull request. Do not propose changes based on taste or intuition. We do not delegate programming to tools: that some tool suggested a code change is not reason enough to change the code.
+- Users should provide a rationale for their changes. Does it improve performance? Does it add a feature? Does it improve maintainability? Does it fix a bug? This must be explicitly stated as part of the pull request. Do not propose changes based on taste or intuition. We do not delegate programming to tools: that some tool suggested a code change is not reason enough to change the code.
    1. When your code improves performance, please document the gains with a benchmark using hard numbers.
-   2. If your code fixes a bug, please be either fix a failing test, or propose a new test.
+   2. If your code fixes a bug, please either fix a failing test, or propose a new test.
    3. Other types of changes must be clearly motivated. We openly discourage changes with no identifiable benefits.
 - Changes should be focused and minimal. You should change as few lines of code as possible. Please do not reformat or touch files needlessly.
-- New features must be accompanied of new tests, in general.
+- New features must be accompanied by new tests, in general.
 - Your code should pass our continuous-integration tests. It is your responsibility to ensure that your proposal pass the tests. We do not merge pull requests that would break our build.
    - An exception to this would be changes to non-code files, such as documentation and assets, or trivial changes to code, such as comments, where it is encouraged to explicitly ask for skipping a CI run using the `[skip ci]` prefix in your Pull Request title **and** in the first line of the most recent commit in a push. Example for such a commit: `[skip ci] Fixed typo in power_of_ten's docs`
    This benefits the project in such a way that the CI pipeline is not burdened by running jobs on changes that don't change any behavior in the code, which reduces wait times for other Pull Requests that do change behavior and require testing.

CONTRIBUTORS

@@ -37,5 +37,9 @@ Furkan Taşkale
 Brendan Knapp
 Danila Kutenin
 Pavel Pavlov
+Hao Chen
+Nicolas Boyer
+Kim Walisch and Jatin Bhateja (AVX-512 bitset decoder)
+Fangzheng Zhang and Weiqiang Wan (AVX-512 kernel)
 # if you have contributed to the project and your name does not
 # appear in this list, please let us know!

Doxyfile

@@ -38,7 +38,7 @@ PROJECT_NAME           = simdjson
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = "0.9.1"
+PROJECT_NUMBER         = "2.0.4"
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a

README.md

@@ -3,7 +3,7 @@
 [![Ubuntu 20.04 CI](https://github.com/simdjson/simdjson/workflows/Ubuntu%2020.04%20CI%20(GCC%209)/badge.svg)](https://simdjson.org/plots.html)
 ![VS16-CI](https://github.com/simdjson/simdjson/workflows/VS16-CI/badge.svg)
 ![MinGW64-CI](https://github.com/simdjson/simdjson/workflows/MinGW64-CI/badge.svg)
-[![][license img]][license]  [![Doxygen Documentation](https://img.shields.io/badge/docs-doxygen-green.svg)](https://simdjson.org/api/0.9.0/index.html)
+[![][license img]][license]  [![Doxygen Documentation](https://img.shields.io/badge/docs-doxygen-green.svg)](https://simdjson.org/api/2.0.0/index.html)
 
 simdjson : Parsing gigabytes of JSON per second
 ===============================================
@@ -53,6 +53,7 @@ The simdjson library is easily consumable with a single .h and .cpp file.
 2. Create `quickstart.cpp`:
 
 ```c++
+#include <iostream>
 #include "simdjson.h"
 using namespace simdjson;
 int main(void) {
@@ -78,7 +79,7 @@ Usage documentation is available:
 * [Performance](doc/performance.md) shows some more advanced scenarios and how to tune for them.
 * [Implementation Selection](doc/implementation-selection.md) describes runtime CPU detection and
   how you can work with it.
-* [API](https://simdjson.org/api/0.9.0/annotated.html) contains the automatically generated API documentation.
+* [API](https://simdjson.org/api/1.0.0/annotated.html) contains the automatically generated API documentation.
 
 Performance results
 -------------------
@@ -137,6 +138,7 @@ We distinguish between "bindings" (which just wrap the C++ code) and a port to a
 - [rcppsimdjson](https://github.com/eddelbuettel/rcppsimdjson): R bindings.
 - [simdjson_erlang](https://github.com/ChomperT/simdjson_erlang): erlang bindings.
 - [lua-simdjson](https://github.com/FourierTransformer/lua-simdjson): lua bindings.
+- [hermes-json](https://hackage.haskell.org/package/hermes-json): haskell bindings.
 
 
 About simdjson
@@ -158,7 +160,7 @@ We also have an informal [blog post providing some background and context](https
 
 For the video inclined, <br />
 [![simdjson at QCon San Francisco 2019](http://img.youtube.com/vi/wlvKAT7SZIQ/0.jpg)](http://www.youtube.com/watch?v=wlvKAT7SZIQ)<br />
-(it was the best voted talk, we're kinda proud of it).
+(It was the best voted talk, we're kinda proud of it.)
 
 Funding
 -------

benchmark/amazon_cellphones/amazon_cellphones.h

@@ -4,8 +4,12 @@
 #include <map>
 #include <string>
 
+
 namespace amazon_cellphones {
 
+const bool UNTHREADED = false;
+const bool THREADED = true;
+
 using namespace json_benchmark;
 
 struct brand {
@@ -59,10 +63,11 @@ struct runner : public file_runner<I> {
     }
 };
 
+template<bool threaded>
 struct simdjson_dom;
 
 template<typename I> simdjson_really_inline static void amazon_cellphones(benchmark::State &state) {
-  run_json_benchmark<runner<I>, runner<simdjson_dom>>(state);
+  run_json_benchmark<runner<I>, runner<simdjson_dom<UNTHREADED>>>(state);
 }
 
 }   // namespace amazon_cellphones

benchmark/amazon_cellphones/simdjson_dom.h

@@ -8,12 +8,16 @@ namespace amazon_cellphones {
 
 using namespace simdjson;
 
+template<bool threaded>
 struct simdjson_dom {
   using StringType = std::string;
 
   dom::parser parser{};
 
   bool run(simdjson::padded_string &json, std::map<StringType, brand> &result) {
+#ifdef SIMDJSON_THREADS_ENABLED
+    parser.threaded = threaded;
+#endif
     auto stream = parser.parse_many(json);
     auto i = stream.begin();
     ++i;  // Skip first line
@@ -37,7 +41,10 @@ struct simdjson_dom {
 
 };
 
-BENCHMARK_TEMPLATE(amazon_cellphones, simdjson_dom)->UseManualTime();
+BENCHMARK_TEMPLATE(amazon_cellphones, simdjson_dom<UNTHREADED>)->UseManualTime();
+#ifdef SIMDJSON_THREADS_ENABLED
+BENCHMARK_TEMPLATE(amazon_cellphones, simdjson_dom<THREADED>)->UseManualTime();
+#endif
 
 } // namespace amazon_cellphones
 

benchmark/amazon_cellphones/simdjson_ondemand.h

@@ -8,17 +8,21 @@ namespace amazon_cellphones {
 
 using namespace simdjson;
 
+template<bool threaded>
 struct simdjson_ondemand {
   using StringType = std::string;
 
   ondemand::parser parser{};
 
   bool run(simdjson::padded_string &json, std::map<StringType, brand> &result) {
+#ifdef SIMDJSON_THREADS_ENABLED
+    parser.threaded = threaded;
+#endif
     ondemand::document_stream stream = parser.iterate_many(json);
     ondemand::document_stream::iterator i = stream.begin();
     ++i;  // Skip first line
     for (;i != stream.end(); ++i) {
-      auto & doc = *i;
+      auto doc = *i;
       size_t index{0};
       StringType copy;
       double rating;
@@ -58,7 +62,10 @@ struct simdjson_ondemand {
 
 };
 
-BENCHMARK_TEMPLATE(amazon_cellphones, simdjson_ondemand)->UseManualTime();
+BENCHMARK_TEMPLATE(amazon_cellphones, simdjson_ondemand<UNTHREADED>)->UseManualTime();
+#ifdef SIMDJSON_THREADS_ENABLED
+BENCHMARK_TEMPLATE(amazon_cellphones, simdjson_ondemand<THREADED>)->UseManualTime();
+#endif
 
 } // namespace amazon_cellphones
 

benchmark/bench_dom_api.cpp

@@ -1,4 +1,5 @@
 #include <benchmark/benchmark.h>
+#include <iostream>
 #include "simdjson.h"
 #include <sstream>
 

benchmark/bench_parse_call.cpp

@@ -1,4 +1,5 @@
 #include <benchmark/benchmark.h>
+#include <iostream>
 #include "simdjson.h"
 using namespace simdjson;
 using namespace benchmark;

benchmark/benchfeatures.cpp

@@ -97,9 +97,9 @@ struct option_struct {
         verbose = true;
         break;
       case 'a': {
-          auto impl = simdjson::available_implementations[optarg];
+          auto impl = simdjson::get_available_implementations()[optarg];
           if(impl && impl->supported_by_runtime_system()) {
-            simdjson::active_implementation = impl;
+            simdjson::get_active_implementation() = impl;
           } else {
             std::cerr << "implementation " << optarg << " not found or not supported " << std::endl;
           }
@@ -204,10 +204,13 @@ struct feature_benchmarker {
   }
   // Rate of 1-7-structural misses per 8-structural flip
   double struct1_7_miss_rate(BenchmarkStage stage) const {
+#if SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
+    return 1;
+#else
     if (!has_events()) { return 1; }
     return struct7_miss[stage].best.branch_misses() - struct7[stage].best.branch_misses() / double(struct7_miss.stats->blocks_with_1_structural_flipped);
+#endif
   }
-
   // Extra cost of an 8-15 structural block over a 1-7 structural block
   double struct8_15_cost(BenchmarkStage stage) const {
     return cost_per_block(stage, struct15, struct15.stats->blocks_with_8_structurals, struct7);
@@ -218,8 +221,12 @@ struct feature_benchmarker {
   }
   // Rate of 8-15-structural misses per 8-structural flip
   double struct8_15_miss_rate(BenchmarkStage stage) const {
+#if SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
+    return 1;
+#else
     if (!has_events()) { return 1; }
     return double(struct15_miss[stage].best.branch_misses() - struct15[stage].best.branch_misses()) / double(struct15_miss.stats->blocks_with_8_structurals_flipped);
+#endif
   }
 
   // Extra cost of a 16+-structural block over an 8-15 structural block (actual varies based on # of structurals!)
@@ -232,10 +239,15 @@ struct feature_benchmarker {
   }
   // Rate of 16-structural misses per 16-structural flip
   double struct16_miss_rate(BenchmarkStage stage) const {
+#if SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
+    return 1;
+#else
     if (!has_events()) { return 1; }
     return double(struct23_miss[stage].best.branch_misses() - struct23[stage].best.branch_misses()) / double(struct23_miss.stats->blocks_with_16_structurals_flipped);
+#endif
   }
 
+
   // Extra cost of having UTF-8 in a block
   double utf8_cost(BenchmarkStage stage) const {
     return cost_per_block(stage, utf8, utf8.stats->blocks_with_utf8, struct7_full);
@@ -246,10 +258,13 @@ struct feature_benchmarker {
   }
   // Rate of UTF-8 misses per UTF-8 flip
   double utf8_miss_rate(BenchmarkStage stage) const {
+#if SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
+    return 1;
+#else
     if (!has_events()) { return 1; }
     return double(utf8_miss[stage].best.branch_misses() - utf8[stage].best.branch_misses()) / double(utf8_miss.stats->blocks_with_utf8_flipped);
+#endif
   }
-
   // Extra cost of having escapes in a block
   double escape_cost(BenchmarkStage stage) const {
     return cost_per_block(stage, escape, escape.stats->blocks_with_escapes, struct7_full);
@@ -260,10 +275,15 @@ struct feature_benchmarker {
   }
   // Rate of escape misses per escape flip
   double escape_miss_rate(BenchmarkStage stage) const {
+#if SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
+    return 1;
+#else
     if (!has_events()) { return 1; }
     return double(escape_miss[stage].best.branch_misses() - escape[stage].best.branch_misses()) / double(escape_miss.stats->blocks_with_escapes_flipped);
+#endif
   }
 
+
   double calc_expected_feature_cost(BenchmarkStage stage, const benchmarker& file) const {
     // Expected base ns/block (empty)
     json_stats& stats = *file.stats;
@@ -300,7 +320,6 @@ struct feature_benchmarker {
   double calc_expected(BenchmarkStage stage, const benchmarker& file) const {
     return calc_expected_feature_cost(stage, file) + calc_expected_miss_cost(stage, file);
   }
-
   void print(const option_struct& options) const {
     printf("\n");
     printf("Features in ns/block (64 bytes):\n");
@@ -359,6 +378,22 @@ struct feature_benchmarker {
   }
 };
 
+#if SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
+void print_file_effectiveness(BenchmarkStage stage, const char* filename, const benchmarker& results, const feature_benchmarker& features) {
+  double actual = results[stage].best.elapsed_ns() / double(results.stats->blocks);
+  double calc = features.calc_expected(stage, results);
+  double calc_misses = features.calc_expected_misses(stage, results);
+  double calc_miss_cost = features.calc_expected_miss_cost(stage, results);
+  printf("        | %-8s ", benchmark_stage_name(stage));
+  printf("| %-15s ",   filename);
+  printf("|    %8.3g ", features.calc_expected_feature_cost(stage, results));
+  printf("|    %8.3g ", calc_miss_cost);
+  printf("| %8.3g ",  calc);
+  printf("| %8.3g ",  actual);
+  printf("| %+8.3g ", actual - calc);
+  printf("| %13llu ", (long long unsigned)(calc_misses));
+}
+#else
 void print_file_effectiveness(BenchmarkStage stage, const char* filename, const benchmarker& results, const feature_benchmarker& features) {
   double actual = results[stage].best.elapsed_ns() / double(results.stats->blocks);
   double calc = features.calc_expected(stage, results);
@@ -382,6 +417,7 @@ void print_file_effectiveness(BenchmarkStage stage, const char* filename, const
   }
   printf("|\n");
 }
+#endif
 
 int main(int argc, char *argv[]) {
   // Read options

benchmark/benchmarker.h

@@ -2,7 +2,7 @@
 #define __BENCHMARKER_H
 
 #include "event_counter.h"
-#include "simdjson.h" // For SIMDJSON_DISABLE_DEPRECATED_WARNINGS
+#include "simdjson.h"
 
 #include <cassert>
 #include <cctype>
@@ -423,7 +423,7 @@ struct benchmarker {
         stage.instructions() / static_cast<double>(stats->structurals),
         stage.instructions() / static_cast<double>(stage.cycles())
       );
-
+#if !SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
       // NOTE: removed cycles/miss because it is a somewhat misleading stat
       printf("%s%-13s: %7.0f branch misses (%6.2f%%) - %.0f cache misses (%6.2f%%) - %.2f cache references\n",
         prefix,
@@ -434,6 +434,7 @@ struct benchmarker {
         percent(stage.cache_misses(), all_stages_without_allocation.cache_misses()),
         stage.cache_references()
       );
+#endif
     }
   }
 

benchmark/dom/parse.cpp

@@ -66,7 +66,7 @@ void print_usage(ostream& out) {
   out << "-H           - Make the buffers hot (reduce page allocation and related OS tasks during parsing) [default]" << endl;
   out << "-a IMPL      - Use the given parser implementation. By default, detects the most advanced" << endl;
   out << "               implementation supported on the host machine." << endl;
-  for (auto impl : simdjson::available_implementations) {
+  for (auto impl : simdjson::get_available_implementations()) {
     if(impl->supported_by_runtime_system()) {
       out << "-a " << std::left << std::setw(9) << impl->name() << " - Use the " << impl->description() << " parser implementation." << endl;
     }
@@ -116,10 +116,10 @@ struct option_struct {
         verbose = true;
         break;
       case 'a': {
-        const implementation *impl = simdjson::available_implementations[optarg];
+        const implementation *impl = simdjson::get_available_implementations()[optarg];
         if ((!impl) || (!impl->supported_by_runtime_system())) {
           std::string exit_message = string("Unsupported option value -a ") + optarg + ": expected -a  with one of ";
-          for (auto imple : simdjson::available_implementations) {
+          for (auto imple : simdjson::get_available_implementations()) {
             if(imple->supported_by_runtime_system()) {
               exit_message += imple->name();
               exit_message += " ";
@@ -127,7 +127,7 @@ struct option_struct {
           }
           exit_usage(exit_message);
         }
-        simdjson::active_implementation = impl;
+        simdjson::get_active_implementation() = impl;
         break;
       }
       case 'C':
@@ -175,7 +175,7 @@ int main(int argc, char *argv[]) {
   option_struct options(argc, argv);
   if (options.verbose) {
     verbose_stream = &cout;
-    verbose() << "Implementation: " << simdjson::active_implementation->name() << endl;
+    verbose() << "Implementation: " << simdjson::get_active_implementation()->name() << endl;
   }
 
   // Start collecting events. We put this early so if it prints an error message, it's the

benchmark/dom/statisticalmodel.cpp

@@ -50,9 +50,9 @@ simdjson_really_inline void simdjson_process_atom(stat_t &s,
   } else if(element.is<double>()) {
     s.float_count++;
   } else if (element.is<bool>()) {
-    simdjson::error_code err;
     bool v;
-    err = element.get(v);
+    simdjson::error_code error;
+    if ((error = element.get(v))) { std::cerr << error << std::endl; abort(); }
     if (v) {
       s.true_count++;
     } else {

benchmark/event_counter.h

@@ -1,6 +1,15 @@
 #ifndef __EVENT_COUNTER_H
 #define __EVENT_COUNTER_H
 
+#ifndef SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
+#ifdef __aarch64__
+// on ARM, we use just cycles and instructions
+#define SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS 1
+#else
+// elsewhere, we try to use four counters.
+#define SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS 0
+#endif
+#endif
 #include <cassert>
 #include <cctype>
 #ifndef _MSC_VER
@@ -46,6 +55,12 @@ struct event_count {
   event_count(const event_count& other): elapsed(other.elapsed), event_counts(other.event_counts) { }
 
   // The types of counters (so we can read the getter more easily)
+  #if SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
+  enum event_counter_types {
+    CPU_CYCLES,
+    INSTRUCTIONS
+  };
+  #else
   enum event_counter_types {
     CPU_CYCLES,
     INSTRUCTIONS,
@@ -53,15 +68,16 @@ struct event_count {
     CACHE_REFERENCES,
     CACHE_MISSES
   };
-
+  #endif
   double elapsed_sec() const { return duration<double>(elapsed).count(); }
   double elapsed_ns() const { return duration<double, std::nano>(elapsed).count(); }
   double cycles() const { return static_cast<double>(event_counts[CPU_CYCLES]); }
   double instructions() const { return static_cast<double>(event_counts[INSTRUCTIONS]); }
+#if !SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
   double branch_misses() const { return static_cast<double>(event_counts[BRANCH_MISSES]); }
   double cache_references() const { return static_cast<double>(event_counts[CACHE_REFERENCES]); }
   double cache_misses() const { return static_cast<double>(event_counts[CACHE_MISSES]); }
-
+#endif
   event_count& operator=(const event_count& other) {
     this->elapsed = other.elapsed;
     this->event_counts = other.event_counts;
@@ -105,9 +121,11 @@ struct event_aggregate {
   double elapsed_ns() const { return total.elapsed_ns() / iterations; }
   double cycles() const { return total.cycles() / iterations; }
   double instructions() const { return total.instructions() / iterations; }
+#if !SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
   double branch_misses() const { return total.branch_misses() / iterations; }
   double cache_references() const { return total.cache_references() / iterations; }
   double cache_misses() const { return total.cache_misses() / iterations; }
+#endif
 };
 
 struct event_collector {
@@ -117,11 +135,16 @@ struct event_collector {
 #if defined(__linux__)
   LinuxEvents<PERF_TYPE_HARDWARE> linux_events;
   event_collector(bool quiet = false) : linux_events(vector<int>{
+  #if SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
+    PERF_COUNT_HW_CPU_CYCLES,
+    PERF_COUNT_HW_INSTRUCTIONS,
+  #else
     PERF_COUNT_HW_CPU_CYCLES,
     PERF_COUNT_HW_INSTRUCTIONS,
     PERF_COUNT_HW_BRANCH_MISSES,
     PERF_COUNT_HW_CACHE_REFERENCES,
     PERF_COUNT_HW_CACHE_MISSES
+  #endif
   }, quiet) {}
   bool has_events() {
     return linux_events.is_working();

benchmark/json_benchmark/run_json_benchmark.h

@@ -10,8 +10,8 @@ void maybe_display_implementation() {
   static bool displayed_implementation = false;
   if(!displayed_implementation) {
     displayed_implementation = true;
-    std::cout << "simdjson::dom implementation:      " << simdjson::active_implementation->name() << std::endl;
+    std::cout << "simdjson::dom implementation:      " << simdjson::get_active_implementation()->name() << std::endl;
-    std::cout << "simdjson::ondemand implementation (stage 1): " << simdjson::active_implementation->name() << std::endl;
+    std::cout << "simdjson::ondemand implementation (stage 1): " << simdjson::get_active_implementation()->name() << std::endl;
     std::cout << "simdjson::ondemand implementation (stage 2): " << simdjson::builtin_implementation()->name() << std::endl;
   }
 }
@@ -58,10 +58,11 @@ template<typename B, typename R> static void run_json_benchmark(benchmark::State
   if (collector.has_events()) {
     state.counters["instructions"] = events.instructions();
     state.counters["cycles"]       = events.cycles();
+#if !SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
     state.counters["branch_miss"]  = events.branch_misses();
     state.counters["cache_miss"]   = events.cache_misses();
     state.counters["cache_ref"]    = events.cache_references();
-
+#endif
     state.counters["instructions_per_byte"]  = events.instructions() / double(bench.bytes_per_iteration());
     state.counters["instructions_per_cycle"] = events.instructions() / events.cycles();
     state.counters["cycles_per_byte"]        = events.cycles()       / double(bench.bytes_per_iteration());
@@ -69,9 +70,11 @@ template<typename B, typename R> static void run_json_benchmark(benchmark::State
 
     state.counters["best_instructions"] = events.best.instructions();
     state.counters["best_cycles"]       = events.best.cycles();
+#if !SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
     state.counters["best_branch_miss"]  = events.best.branch_misses();
     state.counters["best_cache_miss"]   = events.best.cache_misses();
     state.counters["best_cache_ref"]    = events.best.cache_references();
+#endif
 
     state.counters["best_instructions_per_byte"]  = events.best.instructions() / double(bench.bytes_per_iteration());
     state.counters["best_instructions_per_cycle"] = events.best.instructions() / events.best.cycles();
@@ -92,9 +95,11 @@ template<typename B, typename R> static void run_json_benchmark(benchmark::State
   if (collector.has_events()) {
     label << " instructions=" << setw(12) << uint64_t(events.best.instructions())     << setw(0);
     label << " cycles="       << setw(12) << uint64_t(events.best.cycles())           << setw(0);
+#if !SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
     label << " branch_miss="  << setw(8)  << uint64_t(events.best.branch_misses())    << setw(0);
     label << " cache_miss="   << setw(8)  << uint64_t(events.best.cache_misses())     << setw(0);
     label << " cache_ref="    << setw(10) << uint64_t(events.best.cache_references()) << setw(0);
+#endif
   }
 
   label << " items=" << setw(10) << bench.items_per_iteration() << setw(0);

benchmark/large_amazon_cellphones/large_amazon_cellphones.h

@@ -6,6 +6,9 @@
 
 namespace large_amazon_cellphones {
 
+const bool UNTHREADED = false;
+const bool THREADED = true;
+
 static const simdjson::padded_string &get_built_json();
 
 using namespace json_benchmark;
@@ -81,11 +84,11 @@ static const simdjson::padded_string &get_built_json() {
   return json;
 }
 
-
+template<bool threaded>
 struct simdjson_dom;
 
 template<typename I> simdjson_really_inline static void large_amazon_cellphones(benchmark::State &state) {
-  run_json_benchmark<runner<I>, runner<simdjson_dom>>(state);
+  run_json_benchmark<runner<I>, runner<simdjson_dom<UNTHREADED>>>(state);
 }
 
 }   // namespace large_amazon_cellphones

benchmark/large_amazon_cellphones/simdjson_dom.h

@@ -9,12 +9,16 @@ namespace large_amazon_cellphones {
 
 using namespace simdjson;
 
+template<bool threaded>
 struct simdjson_dom {
   using StringType = std::string;
 
   dom::parser parser{};
 
   bool run(simdjson::padded_string &json, std::map<StringType, brand> &result) {
+#ifdef SIMDJSON_THREADS_ENABLED
+    parser.threaded = threaded;
+#endif
     auto stream = parser.parse_many(json);
     auto i = stream.begin();
     ++i;  // Skip first line
@@ -38,7 +42,10 @@ struct simdjson_dom {
 
 };
 
-BENCHMARK_TEMPLATE(large_amazon_cellphones, simdjson_dom)->UseManualTime();
+BENCHMARK_TEMPLATE(large_amazon_cellphones, simdjson_dom<UNTHREADED>)->UseManualTime();
+#ifdef SIMDJSON_THREADS_ENABLED
+BENCHMARK_TEMPLATE(large_amazon_cellphones, simdjson_dom<THREADED>)->UseManualTime();
+#endif
 
 } // namespace large_amazon_cellphones
 

benchmark/large_amazon_cellphones/simdjson_ondemand.h

@@ -8,17 +8,21 @@ namespace large_amazon_cellphones {
 
 using namespace simdjson;
 
+template<bool threaded>
 struct simdjson_ondemand {
   using StringType = std::string;
 
   ondemand::parser parser{};
 
   bool run(simdjson::padded_string &json, std::map<StringType, brand> &result) {
+#ifdef SIMDJSON_THREADS_ENABLED
+    parser.threaded = threaded;
+#endif
     ondemand::document_stream stream = parser.iterate_many(json);
     ondemand::document_stream::iterator i = stream.begin();
     ++i;  // Skip first line
     for (;i != stream.end(); ++i) {
-      auto & doc = *i;
+      auto doc = *i;
       size_t index{0};
       StringType copy;
       double rating;
@@ -58,7 +62,10 @@ struct simdjson_ondemand {
 
 };
 
-BENCHMARK_TEMPLATE(large_amazon_cellphones, simdjson_ondemand)->UseManualTime();
+BENCHMARK_TEMPLATE(large_amazon_cellphones, simdjson_ondemand<UNTHREADED>)->UseManualTime();
+#ifdef SIMDJSON_THREADS_ENABLED
+BENCHMARK_TEMPLATE(large_amazon_cellphones, simdjson_ondemand<THREADED>)->UseManualTime();
+#endif
 
 } // namespace amazon_cellphones
 

cmake/developer-options.cmake

@@ -3,6 +3,12 @@
 #
 add_library(simdjson-internal-flags INTERFACE)
 
+
+option(SIMDJSON_CHECK_EOF "Check for the end of the input buffer. The setting is unnecessary since we require padding of the inputs. You should expect tests to fail with this option turned on." OFF)
+if(SIMDJSON_CHECK_EOF)
+  add_compile_definitions(SIMDJSON_CHECK_EOF=1)
+endif()
+
 option(SIMDJSON_SANITIZE_UNDEFINED "Sanitize undefined behavior" OFF)
 if(SIMDJSON_SANITIZE_UNDEFINED)
   add_compile_options(-fsanitize=undefined -fno-sanitize-recover=all)
@@ -169,6 +175,18 @@ if(CMAKE_C_COMPILER_ID MATCHES "Intel")
   set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -static-intel")
 endif()
 
+option(
+    SIMDJSON_AVX512_ALLOWED
+    "Enable AVX-512 instructions (only affects processors and compilers with AVX-512 support)."
+    ON
+)
+if(SIMDJSON_AVX512_ALLOWED)
+  add_compile_definitions(SIMDJSON_AVX512_ALLOWED=1)
+else()
+  add_compile_definitions(SIMDJSON_AVX512_ALLOWED=0)
+  message(STATUS "AVX-512 instructions are not allowed.")
+endif()
+
 include(CheckSymbolExists)
 check_symbol_exists(fork unistd.h HAVE_POSIX_FORK)
 check_symbol_exists(wait sys/wait.h HAVE_POSIX_WAIT)

cmake/implementation-flags.cmake

@@ -1,7 +1,7 @@
 #
 # Implementation selection
 #
-set(SIMDJSON_ALL_IMPLEMENTATIONS fallback westmere haswell arm64 ppc64)
+set(SIMDJSON_ALL_IMPLEMENTATIONS fallback westmere haswell icelake arm64 ppc64)
 
 set(
     SIMDJSON_IMPLEMENTATION ""
@@ -15,7 +15,7 @@ set(
     SIMDJSON_EXCLUDE_IMPLEMENTATION ""
     CACHE STRING "\
 Semicolon-separated list of implementations to exclude \
-(haswell/westmere/arm64/ppc64/fallback). By default, excludes any \
+(icelake/haswell/westmere/arm64/ppc64/fallback). By default, excludes any \
 implementations that are unsupported at compile time or cannot be selected at \
 runtime."
 )

dependencies/CMakeLists.txt

@@ -13,8 +13,9 @@ cmake_dependent_option(SIMDJSON_GOOGLE_BENCHMARKS "compile the Google Benchmark
 if(SIMDJSON_GOOGLE_BENCHMARKS)
   set_off(BENCHMARK_ENABLE_TESTING)
   set_off(BENCHMARK_ENABLE_INSTALL)
+  set_off(BENCHMARK_ENABLE_WERROR)
 
-  import_dependency(google_benchmarks google/benchmark 8982e1e)
+  import_dependency(google_benchmarks google/benchmark f91b6b4)
   add_dependency(google_benchmarks)
 endif()
 

doc/basics_doxygen.md

@@ -1,3 +1,5 @@
+We take our documentation seriously. Please start reading the documentation before you attempt to use simdjson. We hope you will enjoy reading us.
+
 * Basics: https://github.com/simdjson/simdjson/blob/master/doc/basics.md is an overview of how to use simdjson and its APIs.
 * parse_many: https://github.com/simdjson/simdjson/blob/master/doc/parse_many.md describes an interface providing features to work with files or streams containing multiple small JSON documents. As fast and convenient as possible.
 * Performance: https://github.com/simdjson/simdjson/blob/master/doc/performance.md shows some more advanced scenarios and how to tune for them.

doc/dom.md

@@ -3,8 +3,8 @@ The Document-Object-Model (DOM) front-end
 
 An overview of what you need to know to use simdjson, with examples.
 
-* [DOM vs On Demand](#dom-vs-ondemand)
+* [DOM vs On Demand](#dom-vs-on-demand)
-* [The Basics: Loading and Parsing JSON Documents](#the-basics-loading-and-parsing-json-documents)
+* [The Basics: Loading and Parsing JSON Documents](#the-basics-loading-and-parsing-json-documents-using-the-dom-front-end)
 * [Using the Parsed JSON](#using-the-parsed-json)
 * [C++17 Support](#c17-support)
 * [JSON Pointer](#json-pointer)
@@ -64,7 +64,7 @@ Once you have an element, you can navigate it with idiomatic C++ iterators, oper
 
 * **Extracting Values (with exceptions):** You can cast a JSON element to a native type: `double(element)` or
   `double x = json_element`. This works for double, uint64_t, int64_t, bool,
-  dom::object and dom::array. An exception is thrown if the cast is not possible.
+  dom::object and dom::array. An exception (`simdjson::simdjson_error`) is thrown if the cast is not possible.
 * **Extracting Values (without exceptions):** You can use a variant usage of `get()` with error codes to avoid exceptions. You first declare the variable of the appropriate type (`double`, `uint64_t`, `int64_t`, `bool`,
   `dom::object` and `dom::array`) and pass it by reference to `get()` which gives you back an error code: e.g.,
   ```c++
@@ -81,7 +81,7 @@ Once you have an element, you can navigate it with idiomatic C++ iterators, oper
   know the type of the value, you can cast it right there, too! `for (double value : array) { ... }`
 * **Object Iteration:** You can iterate through an object's fields, too: `for (auto [key, value] : object)`
 * **Array Index:** To get at an array value by index, use the at() method: `array.at(0)` gets the
-  first element.
+  first element. The at() method has linear-time complexity so it should not be used to iterate over the values of an array.
   > Note that array[0] does not compile, because implementing [] gives the impression indexing is a
   > O(1) operation, which it is not presently in simdjson. Instead, you should iterate over the elements
   > using a for-loop, as in our examples.
@@ -249,7 +249,11 @@ auto error = parser.parse(json).get(doc);
 if (error) { cerr << error << endl; exit(1); }
 ```
 
-When you use the code this way, it is your responsibility to check for error before using the
+When there is no error, the error code simdjson::SUCCESS is returned: it evaluates as false as a Boolean.
+We have several error codes to indicate errors, they all evaluate to true as a Boolean: your software should not generally not depend on exact
+error codes. We may change the error codes in future releases and the exact error codes could vary depending on your system.
+
+When you use the code without exceptions, it is your responsibility to check for error before using the
 result: if there is an error, the result value will not be valid and using it will caused undefined
 behavior.
 
@@ -274,6 +278,7 @@ Our program loads the file, selects value corresponding to key "search_metadata"
 it selects the key "count" within that object.
 
 ```C++
+#include <iostream>
 #include "simdjson.h"
 
 int main(void) {
@@ -301,6 +306,7 @@ triggering exceptions. To do this, we use `["statuses"].at(0)["id"]`. We break t
 Observe how we use the `at` method when querying an index into an array, and not the bracket operator.
 
 ```C++
+#include <iostream>
 #include "simdjson.h"
 
 int main(void) {
@@ -316,6 +322,9 @@ int main(void) {
 }
 ```
 
+The `at()` method has linear-time complexity: it should not be used to iterate
+over the content of an array.
+
 ### Error Handling Example
 
 This is how the example in "Using the Parsed JSON" could be written using only error code checking:
@@ -457,6 +466,7 @@ program from continuing if there was an error.
 If one is willing to trigger exceptions, it is possible to write simpler code:
 
 ```C++
+#include <iostream>
 #include "simdjson.h"
 
 int main(void) {

doc/implementation-selection.md

@@ -50,9 +50,9 @@ Inspecting the Detected Implementation
 You can check what implementation is running with `active_implementation`:
 
 ```c++
-cout << "simdjson v" << STRINGIFY(SIMDJSON_VERSION) << endl;
+cout << "simdjson v" << SIMDJSON_STRINGIFY(SIMDJSON_VERSION) << endl;
-cout << "Detected the best implementation for your machine: " << simdjson::active_implementation->name();
+cout << "Detected the best implementation for your machine: " << simdjson::get_active_implementation()->name();
-cout << "(" << simdjson::active_implementation->description() << ")" << endl;
+cout << "(" << simdjson::get_active_implementation()->description() << ")" << endl;
 ```
 
 Implementation detection will happen in this case when you first call `name()`.
@@ -63,7 +63,7 @@ Querying Available Implementations
 You can list all available implementations, regardless of which one was selected:
 
 ```c++
-for (auto implementation : simdjson::available_implementations) {
+for (auto implementation : simdjson::get_available_implementations()) {
   cout << implementation->name() << ": " << implementation->description() << endl;
 }
 ```
@@ -71,10 +71,10 @@ for (auto implementation : simdjson::available_implementations) {
 And look them up by name:
 
 ```c++
-cout << simdjson::available_implementations["fallback"]->description() << endl;
+cout << simdjson::get_available_implementations()["fallback"]->description() << endl;
 ```
 Though the fallback implementation should always be available, others might be missing. When
-an implementation is not available, the bracket call `simdjson::available_implementations[name]`
+an implementation is not available, the bracket call `simdjson::get_available_implementations()[name]`
 will return the null pointer.
 
 The available implementations have been compiled but may not necessarily be run safely on your system
@@ -90,18 +90,18 @@ can select the CPU architecture yourself:
 
 ```c++
 // Use the fallback implementation, even though my machine is fast enough for anything
-simdjson::active_implementation = simdjson::available_implementations["fallback"];
+simdjson::get_active_implementation() = simdjson::get_available_implementations()["fallback"];
 ```
 
 You are responsible for ensuring that the requirements of the selected implementation match your current system.
-Furthermore, you should check that the implementation is available before setting it to `simdjson::active_implementation`
+Furthermore, you should check that the implementation is available before setting it to `simdjson::get_active_implementation()`
 by comparing it with the null pointer.
 
 ```c++
-auto my_implementation = simdjson::available_implementations["haswell"];
+auto my_implementation = simdjson::get_available_implementations()["haswell"];
 if(! my_implementation) { exit(1); }
 if(! my_implementation->supported_by_runtime_system()) { exit(1); }
-simdjson::active_implementation = my_implementation;
+simdjson::get_active_implementation() = my_implementation;
 ```
 
 Checking that an Implementation can Run on your System
@@ -110,12 +110,12 @@ Checking that an Implementation can Run on your System
 You should call `supported_by_runtime_system()` to compare the processor's features with the need of the implementation.
 
 ```c++
-for (auto implementation : simdjson::available_implementations) {
+for (auto implementation : simdjson::get_available_implementations()) {
   if(implementation->supported_by_runtime_system()) {
     cout << implementation->name() << ": " << implementation->description() << endl;
   }
 }
 ```
 
-The call to `supported_by_runtime_system()` maybe relatively expensive. Do not call  `supported_by_runtime_system()` each
+The call to `supported_by_runtime_system()` may be relatively expensive. Do not call  `supported_by_runtime_system()` each
 time you parse a JSON input (for example). It is meant to be called a handful of times at most in the life of a program.

doc/iterate_many.md

@@ -1,8 +1,16 @@
 iterate_many
 ==========
 
-An interface providing features to work with files or streams containing multiple small JSON documents.
+When serializing large databases, it is often better to write out many independent JSON
-As fast and convenient as possible.
+documents, instead of one large monolithic document containing many records. The simdjson
+library provides high-speed access to files or streams containing multiple small JSON documents separated by ASCII white-space characters. Given an input such as
+```JSON
+{"text":"a"}
+{"text":"b"}
+{"text":"c"}
+...
+```
+... you want to read the entries (individual JSON documents) as quickly and as conveniently as possible. Importantly, the input might span several gigabytes, but you want to use a small (fixed) amount of memory. Ideally, you'd also like the parallelize the processing (using more than one core) to speed up the process.
 
 Contents
 --------
@@ -108,7 +116,9 @@ Whitespace Characters:
 - **Linefeed**
 - **Carriage return**
 - **Horizontal tab**
-- **Nothing**
+
+If your documents are all objects or arrays, then you may even have nothing between them.
+E.g., `[1,2]{"32":1}` is recognized as two documents.
 
 Some official formats **(non-exhaustive list)**:
 - [Newline-Delimited JSON (NDJSON)](http://ndjson.org/)
@@ -175,7 +185,7 @@ Let us illustrate the idea with code:
     auto i = stream.begin();
 	 size_t count{0};
     for(; i != stream.end(); ++i) {
-        auto & doc = *i;
+        auto doc = *i;
         if(!i.error()) {
           std::cout << "got full document at " << i.current_index() << std::endl;
           std::cout << i.source() << std::endl;

doc/ondemand_design.md

@@ -709,42 +709,6 @@ in production systems:
     }
 ```
 
-
-### Long-Running Processes and Memory Capacity
-
-The On Demand approach also automatically expands its memory capacity when larger documents are parsed. However, for longer processes where very large files are processed (such as server loops), this capacity is not resized down. Similarly to the DOM-based approach (see [here](https://github.com/simdjson/simdjson/blob/master/doc/dom.md#server-loops-long-running-processes-and-memory-capacity)]), On Demand also lets you adjust the maximal capacity that the parser can process:
-
-* You can set an upper bound (*max_capacity*) when construction the parser:
-```C++
-    ondemand::parser parser(1000*1000);  // Never grows past documents > 1 MB
-    auto doc = parser.iterate(json);
-    for (web_request request : listen()) {
-      padded_string json;
-      padded_string json = padded_string::load(request.body);
-      auto error = parser.iterate(json);
-      // If the document was above our limit, emit 413 = payload too large
-      if (error == CAPACITY) { request.respond(413); continue; }
-      // ...
-    }
-```
-
-The capacity will grow as the parser encounters larger documents up to 1 MB.
-
-* You can also allocate a *fixed capacity* that will never grow:
-```C++
-    ondemand::parser parser(1000*1000);
-    parser.allocate(1000*1000)  // Fix the capacity to 1 MB
-    auto doc = parser.iterate(json);
-    for (web_request request : listen()) {
-      padded_string json;
-      padded_string json = padded_string::load(request.body);
-      auto error = parser.iterate(json);
-      // If the document was above our limit, emit 413 = payload too large
-      if (error == CAPACITY) { request.respond(413); continue; }
-      // ...
-    }
-```
-You can also manually set the maximal capacity using the method `set_max_capacity()`.
 ### Benefits of the On Demand Approach
 
 We expect that the On Demand approach has many of the performance benefits of the schema-based approach, while providing a flexibility that is similar to that of the DOM-based approach.
@@ -764,7 +728,6 @@ There are currently additional technical limitations which we expect to resolve
 
 * The simdjson library offers runtime dispatching which allows you to compile one binary and have it run at full speed on different processors, taking advantage of the specific features of the processor. The On Demand API has limited runtime dispatch support. Under x64 systems, to fully benefit from the On Demand API, we recommend that you compile your code for a specific processor. E.g., if your processor supports AVX2 instructions, you should compile your binary executable with AVX2 instruction support (by using your compiler's commands). If you are sufficiently technically proficient, you can implement runtime dispatching within your application, by compiling your On Demand code for different processors.
 * There is an initial phase which scans the entire document quickly, irrespective of the size of the document. We plan to break this phase into distinct steps for large files in a future release as we have done with other components of our API (e.g., `parse_many`).
-* The On Demand API does not support JSON Pointer. This capability is currently limited to our core API.
 
 ### Applicability of the On Demand Approach
 

doc/parse_many.md

@@ -1,8 +1,14 @@
 parse_many
 ==========
 
-An interface providing features to work with files or streams containing multiple small JSON documents.
+An interface providing features to work with files or streams containing multiple small JSON documents. Given an input such as
-As fast and convenient as possible.
+```JSON
+{"text":"a"}
+{"text":"b"}
+{"text":"c"}
+...
+```
+... you want to read the entries (individual JSON documents) as quickly and as conveniently as possible. Importantly, the input might span several gigabytes, but you want to use a small (fixed) amount of memory. Ideally, you'd also like the parallelize the processing (using more than one core) to speed up the process.
 
 Contents
 --------

doc/performance.md

@@ -10,7 +10,7 @@ are still some scenarios where tuning can enhance performance.
 * [Large files and huge page support](#large-files-and-huge-page-support)
 * [Number parsing](#number-parsing)
 * [Visual Studio](#visual-studio)
-* [Downclocking](#downclocking)
+* [Power Usage and Downclocking](#power-usage-and-downclocking)
 
 Reusing the parser for maximum efficiency
 -----------------------------------------
@@ -56,6 +56,43 @@ or simply
 ```
 
 
+Server Loops: Long-Running Processes and Memory Capacity
+---------------------------------
+
+The On Demand approach also automatically expands its memory capacity when larger documents are parsed. However, for longer processes where very large files are processed (such as server loops), this capacity is not resized down. On Demand also lets you adjust the maximal capacity that the parser can process:
+
+* You can set an upper bound (*max_capacity*) when construction the parser:
+```C++
+    ondemand::parser parser(1000*1000);  // Never grows past documents > 1 MB
+    auto doc = parser.iterate(json);
+    for (web_request request : listen()) {
+      padded_string json;
+      padded_string json = padded_string::load(request.body);
+      auto error = parser.iterate(json);
+      // If the document was above our limit, emit 413 = payload too large
+      if (error == CAPACITY) { request.respond(413); continue; }
+      // ...
+    }
+```
+
+The capacity will grow as the parser encounters larger documents up to 1 MB.
+
+* You can also allocate a *fixed capacity* that will never grow:
+```C++
+    ondemand::parser parser(1000*1000);
+    parser.allocate(1000*1000)  // Fix the capacity to 1 MB
+    auto doc = parser.iterate(json);
+    for (web_request request : listen()) {
+      padded_string json;
+      padded_string json = padded_string::load(request.body);
+      auto error = parser.iterate(json);
+      // If the document was above our limit, emit 413 = payload too large
+      if (error == CAPACITY) { request.respond(413); continue; }
+      // ...
+    }
+```
+You can also manually set the maximal capacity using the method `set_max_capacity()`.
+
 Large files and huge page support
 ---------------------------------
 
@@ -110,23 +147,21 @@ Recent versions of Microsoft Visual Studio on Windows provides support for the L
 Under Windows, we also support the GNU GCC compiler via MSYS2. The performance of 64-bit MSYS2 under Windows excellent (on par with Linux).
 
 
-Downclocking
+Power Usage and Downclocking
 --------------
 
-
+The simdjson library relies on SIMD instructions. SIMD instructions are the public transportation of computing. Instead of using 4 distinct instructions to add numbers, you can replace them with a single instruction that does the same work. Though the one instruction is slightly more expensive, the energy used per unit of work is much less with SIMD. If you can increase your speed using SIMD instructions (NEON, SSE, AVX), you should expect to reduce your power usage.
-
-SIMD instructions are the public transportation of computing. Instead of using 4 distinct instructions to add numbers, you can replace them with a single instruction that does the same work. Though the one instruction is slightly more expensive, the energy used per unit of work is much less with SIMD. If you can increase your speed using SIMD instructions (NEON, SSE, AVX), you should expect to reduce your power usage.
 
 The SIMD instructions that simdjson relies upon (SSE and AVX under x64, NEON under ARM, ALTIVEC under PPC) are routinely part of runtime libraries (e.g., [Go](https://golang.org/src/runtime/memmove_amd64.s), [Glibc](https://github.com/ihtsae/glibc/commit/5f3d0b78e011d2a72f9e88b0e9ef5bc081d18f97), [LLVM](https://github.com/llvm/llvm-project/blob/96f3ea0d21b48ca088355db10d4d1a2e9bc9f884/lldb/tools/debugserver/source/MacOSX/i386/DNBArchImplI386.cpp), [Rust](https://github.com/rust-lang/rust/commit/070fad1701fb36b112853b0a6a9787a7bb7ff34c), [Java](http://hg.openjdk.java.net/jdk8u/jdk8u/hotspot/file/c1374141598c/src/cpu/x86/vm/stubGenerator_x86_64.cpp#l1297), [PHP](https://github.com/php/php-src/blob/e5cb53ec68603d4dbdd780fd3ecfca943b4fd383/ext/standard/string.c)). What distinguishes the simdjson library is that it is built from the ground up to benefit from these instructions.
 
 
-You should not expect the simdjson library to cause *downclocking* of your recent Intel CPU cores.
+You should not expect the simdjson library to cause *downclocking* of your recent Intel CPU cores. On some Intel processors, using SIMD instructions in a sustained manner on the same CPU core may result in a phenomenon called downclocking whereas the processor initially runs these instructions at a slow speed before reducing the frequency of the core for a short time (milliseconds). Intel refers to these states as licenses. On some current Intel processors, it occurs under two scenarios:
-
-On some Intel processors, using SIMD instructions in a sustained manner on the same CPU core may result in a phenomenon called downclocking whereas the processor initially runs these instructions at a slow speed before reducing the frequency of the core for a short time (milliseconds). Intel refers to these states as licenses. On some current Intel processors, it occurs under two scenarios:
 
 - [Whenever 512-bit AVX-512 instructions are used](https://lemire.me/blog/2018/09/07/avx-512-when-and-how-to-use-these-new-instructions/).
 - Whenever heavy 256-bit or wider instructions are used. Heavy instructions are those involving floating point operations or integer multiplications (since these execute on the floating point unit).
 
-The simdjson library does not currently support AVX-512 instructions and it does not make use of heavy 256-bit instructions. We do use vectorized multiplications, but only using 128-bit registers. Thus there should be no downclocking due to simdjson on recent processors.
+The simdjson library does not make use of heavy 256-bit instructions. We do use vectorized multiplications, but only using 128-bit registers. Thus there should be no downclocking due to simdjson on recent processors, except when AVX-512 is allowed and
+detected. However, we only allow AVX-512 on recent processors (Ice Lake/Tiger Lake or better) where [little to no frequency throttling is expected](https://travisdowns.github.io/blog/2020/08/19/icl-avx512-freq.html). If you can still concerned, you can easily disable AVX-512 with the CMake option `SIMDJSON_AVX512_ALLOWED` set to `OFF` (e.g., `cmake -D SIMDJSON_AVX512_ALLOWED=OFF -B build && cmake --build build`) or by setting
+the macro `SIMDJSON_AVX512_ALLOWED` to `0` in C++ prior to importing the headers.
 
 You may still be worried about which SIMD instruction set is used by simdjson.  Thankfully,  [you can always determine and change which architecture-specific implementation is used](implementation-selection.md) by simdjson. Thus even if your CPU supports AVX2, you do not need to use AVX2. You are in control.

examples/quickstart/quickstart.cpp

@@ -1,3 +1,4 @@
+#include <iostream>
 #include "simdjson.h"
 
 int main(void) {

examples/quickstart/quickstart2.cpp

@@ -1,3 +1,4 @@
+#include <iostream>
 #include "simdjson.h"
 
 int main(void) {

examples/quickstart/quickstart2_noexceptions.cpp

@@ -1,3 +1,4 @@
+#include <iostream>
 #include "simdjson.h"
 
 int main(void) {

examples/quickstart/quickstart_noexceptions.cpp

@@ -1,3 +1,4 @@
+#include <iostream>
 #include "simdjson.h"
 
 int main(void) {

examples/quickstart/quickstart_ondemand.cpp

@@ -1,3 +1,4 @@
+#include <iostream>
 #include "simdjson.h"
 using namespace simdjson;
 int main(void) {

examples/quickstart/quickstart_ondemand_noexceptions.cpp

@@ -1,3 +1,4 @@
+#include <iostream>
 #include "simdjson.h"
 using namespace simdjson;
 int main(void) {

fuzz/Fuzzing.md

@@ -18,7 +18,7 @@ The fuzzers are used in several ways.
 * oss-fuzz - heavy duty 24/7 fuzzing provided by the google driven oss-fuzz project
 
 ## Local fuzzing
-Just invoke fuzz/quick_check.sh, it will download the latest corpus from bintray (kept up to date by the CI fuzzers) and run the fuzzers for a short time. In case you want to run the fuzzers for longer, modify the timeout value in the script or invoke the fuzzer directly.
+Just invoke fuzz/quick_check.sh, it will download the latest corpus (kept up to date by the CI fuzzers) and run the fuzzers for a short time. In case you want to run the fuzzers for longer, modify the timeout value in the script or invoke the fuzzer directly.
 
 This requires linux with clang and cmake installed (recent Debian and Ubuntu are known to work fine).
 
@@ -69,7 +69,7 @@ As little code as possible is kept at oss-fuzz since it is inconvenient to chang
 
 The simdjson library does not benefit from a corpus as much as other projects, because the library is very fast and explores the input space very well. With that said, it is still beneficial to have one. The CI job stores the corpus on a remote server between runs, and is available at [www.pauldreik.se](https://readonly:readonly@www.pauldreik.se/fuzzdata/index.php?project=simdjson).
 
-One can also grab the corpus as an artifact from the github actions job. Pick a run, then go to artifacts and download.
+One can also grab the corpus as an artifact from the github actions job if you are logged in at github. Pick a run, then go to artifacts and download.
 
 ## Fuzzing coverage
 

fuzz/fuzz_implementations.cpp

@@ -11,6 +11,7 @@
 #include <cstddef>
 #include <cstdint>
 #include <cstdlib>
+#include <iostream>
 #include <string>
 #include <array>
 #include "supported_implementations.h"
@@ -92,7 +93,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
     std::size_t nerrors=0;
     for(std::size_t i=0; i<Nimplementations; ++i) {
         auto& e=implementations[i];
-        simdjson::active_implementation=e.impl;
+        simdjson::get_active_implementation()=e.impl;
         e.error=e.parser.parse(Data,Size).get(e.element);
         if(e.error) {
             ++nerrors;

fuzz/fuzz_minifyimpl.cpp

@@ -12,6 +12,7 @@
 #include "simdjson.h"
 #include <cstddef>
 #include <cstdlib>
+#include <iostream>
 #include <vector>
 #include "supported_implementations.h"
 

fuzz/fuzz_ndjson.cpp

@@ -8,24 +8,26 @@
 
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
   FuzzData fd(Data, Size);
-  const auto batch_size=static_cast<size_t>(fd.getInt<0,1000>());
+  const auto batch_size = static_cast<size_t>(fd.getInt<0,1000>());
-  const auto json=simdjson::padded_string{fd.remainder_as_stringview()};
+  const auto json = simdjson::padded_string{fd.remainder_as_stringview()};
   simdjson::dom::parser parser;
-#if SIMDJSON_EXCEPTIONS
+  simdjson::dom::document_stream docs;
-  try {
+  if(parser.parse_many(json,batch_size).get(docs)) { return 0; }
-#endif
+  size_t bool_count1 = 0;
-    simdjson::dom::document_stream docs;
+  size_t total_count1 = 0;
-    if(parser.parse_many(json,batch_size).get(docs)) {
-      return 0;
-    }
-
-  size_t bool_count=0;
   for (auto doc : docs) {
-    bool_count+=doc.is_bool();
+    total_count1++;
+    bool_count1 += doc.is_bool();
   }
-#if SIMDJSON_EXCEPTIONS
+  // Restart, if we made it this far, the document *must* be accessible.
-  } catch(...) {
+  if(parser.parse_many(json,batch_size).get(docs)) { return EXIT_FAILURE; }
+  size_t bool_count2 = 0;
+  size_t total_count2 = 0;
+  for (auto doc : docs) {
+    total_count2++;
+    bool_count2 += doc.is_bool();
   }
-#endif
+  // They should agree!!!
+  if((total_count2 != total_count1) || (bool_count2 != bool_count1)) { return EXIT_FAILURE; }
   return 0;
 }

fuzz/fuzz_utf8.cpp

@@ -10,6 +10,7 @@
 #include "simdjson.h"
 #include <cstddef>
 #include <cstdlib>
+#include <iostream>
 #include "supported_implementations.h"
 
 extern "C" int VerboseTestOneInput(const uint8_t *Data, size_t Size) {

fuzz/quick_check.sh

@@ -4,7 +4,7 @@
 # good when working locally developing the fuzzers or making
 # sure code changes still pass the fuzzers.
 #
-# It will download the corpus from bintray (kept up to date
+# It will download the corpus from external store (kept up to date
 # by the crontab github actions) unless a local out/ directory
 # already exists.
 #
@@ -23,10 +23,9 @@ done
 
 #download the corpus if it does not already exist
 if [ ! -d out ] ; then
-  # ideally, we would download the github artifact but that requires being logged in which can not
+  # the corpus is also available for download from the artifacts page on https://github.com/simdjson/simdjson/actions/workflows/fuzzers.yml
-  # easily be fixed from this shell script.
+  # but that requires being logged in so can not be easily done from this script.
-  echo "NOTE! please go to the artifacts page on https://github.com/simdjson/simdjson/actions/workflows/fuzzers.yml and download the latest corpus.tar.zip artifact manually to speed up fuzzing"
+  wget -O - https://readonly:readonly@www.pauldreik.se/fuzzdata/index.php?project=simdjson |tar xzf -
-  sleep 5s
 fi
 
 # By default, use the debug friendly variant since this script is intended

fuzz/supported_implementations.h

@@ -17,7 +17,7 @@
 std::vector<const simdjson::implementation*>
 get_runtime_supported_implementations() {
     std::vector<const simdjson::implementation*> ret;
-    for(auto& e: simdjson::available_implementations) {
+    for(auto& e: simdjson::get_available_implementations()) {
         if(e->supported_by_runtime_system()) {
             ret.emplace_back(e);
         }

include/simdjson.h

@@ -34,6 +34,7 @@
     }
  */
 
+#include "simdjson/simdjson_version.h"
 #include "simdjson/dom.h"
 #include "simdjson/builtin.h"
 

include/simdjson/arm64/bitmanipulation.h

@@ -8,7 +8,7 @@ namespace {
 // We sometimes call trailing_zero on inputs that are zero,
 // but the algorithms do not end up using the returned value.
 // Sadly, sanitizers are not smart enough to figure it out.
-NO_SANITIZE_UNDEFINED
+SIMDJSON_NO_SANITIZE_UNDEFINED
 simdjson_really_inline int trailing_zeroes(uint64_t input_num) {
 #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
   unsigned long ret;
@@ -73,7 +73,7 @@ simdjson_really_inline uint64_t reverse_bits(uint64_t input_num) {
  * greating or equal to 63 in which case we trigger undefined behavior, but the output
  * of such undefined behavior is never used.
  **/
-NO_SANITIZE_UNDEFINED
+SIMDJSON_NO_SANITIZE_UNDEFINED
 simdjson_really_inline uint64_t zero_leading_bit(uint64_t rev_bits, int leading_zeroes) {
   return rev_bits ^ (uint64_t(0x8000000000000000) >> leading_zeroes);
 }

include/simdjson/arm64/implementation.h

@@ -22,6 +22,7 @@ public:
   ) const noexcept final;
   simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final;
   simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final;
+  simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst) const noexcept final;
 };
 
 } // namespace arm64

include/simdjson/arm64/numberparsing.h

@@ -19,7 +19,7 @@ static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t
 } // namespace SIMDJSON_IMPLEMENTATION
 } // namespace simdjson
 
-#define SWAR_NUMBER_PARSING
+#define SIMDJSON_SWAR_NUMBER_PARSING 1
 
 #include "simdjson/generic/numberparsing.h"
 

include/simdjson/arm64/simd.h

@@ -57,6 +57,19 @@ simdjson_really_inline uint8x16_t make_uint8x16_t(uint8_t x1,  uint8_t x2,  uint
   return x;
 }
 
+simdjson_really_inline uint8x8_t make_uint8x8_t(uint8_t x1,  uint8_t x2,  uint8_t x3,  uint8_t x4,
+                                         uint8_t x5,  uint8_t x6,  uint8_t x7,  uint8_t x8) {
+  uint8x8_t x{};
+  x = vset_lane_u8(x1, x, 0);
+  x = vset_lane_u8(x2, x, 1);
+  x = vset_lane_u8(x3, x, 2);
+  x = vset_lane_u8(x4, x, 3);
+  x = vset_lane_u8(x5, x, 4);
+  x = vset_lane_u8(x6, x, 5);
+  x = vset_lane_u8(x7, x, 6);
+  x = vset_lane_u8(x8, x, 7);
+  return x;
+}
 
 // We have to do the same work for make_int8x16_t
 simdjson_really_inline int8x16_t make_int8x16_t(int8_t x1,  int8_t x2,  int8_t x3,  int8_t x4,
@@ -119,7 +132,7 @@ simdjson_really_inline int8x16_t make_int8x16_t(int8_t x1,  int8_t x2,  int8_t x
     simdjson_really_inline simd8<T>& operator&=(const simd8<T> other) { auto this_cast = static_cast<simd8<T>*>(this); *this_cast = *this_cast & other; return *this_cast; }
     simdjson_really_inline simd8<T>& operator^=(const simd8<T> other) { auto this_cast = static_cast<simd8<T>*>(this); *this_cast = *this_cast ^ other; return *this_cast; }
 
-    simdjson_really_inline Mask operator==(const simd8<T> other) const { return vceqq_u8(*this, other); }
+    friend simdjson_really_inline Mask operator==(const simd8<T> lhs, const simd8<T> rhs) { return vceqq_u8(lhs, rhs); }
 
     template<int N=1>
     simdjson_really_inline simd8<T> prev(const simd8<T> prev_chunk) const {
@@ -289,6 +302,27 @@ simdjson_really_inline int8x16_t make_int8x16_t(int8_t x1,  int8_t x2,  int8_t x
       vst1q_u8(reinterpret_cast<uint8_t*>(output), answer);
     }
 
+    // Copies all bytes corresponding to a 0 in the low half of the mask (interpreted as a
+    // bitset) to output1, then those corresponding to a 0 in the high half to output2.
+    template<typename L>
+    simdjson_really_inline void compress_halves(uint16_t mask, L *output1, L *output2) const {
+      using internal::thintable_epi8;
+      uint8_t mask1 = uint8_t(mask); // least significant 8 bits
+      uint8_t mask2 = uint8_t(mask >> 8); // most significant 8 bits
+      uint8x8_t compactmask1 = vcreate_u8(thintable_epi8[mask1]);
+      uint8x8_t compactmask2 = vcreate_u8(thintable_epi8[mask2]);
+      // we increment by 0x08 the second half of the mask
+#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
+      uint8x8_t inc = make_uint8x8_t(0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08);
+#else
+      uint8x8_t inc = {0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08};
+#endif
+      compactmask2 = vadd_u8(compactmask2, inc);
+      // store each result (with the second store possibly overlapping the first)
+      vst1_u8((uint8_t*)output1, vqtbl1_u8(*this, compactmask1));
+      vst1_u8((uint8_t*)output2, vqtbl1_u8(*this, compactmask2));
+    }
+
     template<typename L>
     simdjson_really_inline simd8<L> lookup_16(
         L replace0,  L replace1,  L replace2,  L replace3,
@@ -439,11 +473,15 @@ simdjson_really_inline int8x16_t make_int8x16_t(int8_t x1,  int8_t x2,  int8_t x
     }
 
 
-    simdjson_really_inline void compress(uint64_t mask, T * output) const {
+    simdjson_really_inline uint64_t compress(uint64_t mask, T * output) const {
-      this->chunks[0].compress(uint16_t(mask), output);
+      uint64_t popcounts = vget_lane_u64(vreinterpret_u64_u8(vcnt_u8(vcreate_u8(~mask))), 0);
-      this->chunks[1].compress(uint16_t(mask >> 16), output + 16 - count_ones(mask & 0xFFFF));
+      // compute the prefix sum of the popcounts of each byte
-      this->chunks[2].compress(uint16_t(mask >> 32), output + 32 - count_ones(mask & 0xFFFFFFFF));
+      uint64_t offsets = popcounts * 0x0101010101010101;
-      this->chunks[3].compress(uint16_t(mask >> 48), output + 48 - count_ones(mask & 0xFFFFFFFFFFFF));
+      this->chunks[0].compress_halves(uint16_t(mask), output, &output[popcounts & 0xFF]);
+      this->chunks[1].compress_halves(uint16_t(mask >> 16), &output[(offsets >> 8) & 0xFF], &output[(offsets >> 16) & 0xFF]);
+      this->chunks[2].compress_halves(uint16_t(mask >> 32), &output[(offsets >> 24) & 0xFF], &output[(offsets >> 32) & 0xFF]);
+      this->chunks[3].compress_halves(uint16_t(mask >> 48), &output[(offsets >> 40) & 0xFF], &output[(offsets >> 48) & 0xFF]);
+      return offsets >> 56;
     }
 
     simdjson_really_inline uint64_t to_bitmask() const {

include/simdjson/builtin.h

@@ -5,7 +5,9 @@
 
 // Determine the best builtin implementation
 #ifndef SIMDJSON_BUILTIN_IMPLEMENTATION
-#if SIMDJSON_CAN_ALWAYS_RUN_HASWELL
+#if SIMDJSON_CAN_ALWAYS_RUN_ICELAKE
+#define SIMDJSON_BUILTIN_IMPLEMENTATION icelake
+#elif SIMDJSON_CAN_ALWAYS_RUN_HASWELL
 #define SIMDJSON_BUILTIN_IMPLEMENTATION haswell
 #elif SIMDJSON_CAN_ALWAYS_RUN_WESTMERE
 #define SIMDJSON_BUILTIN_IMPLEMENTATION westmere

include/simdjson/common_defs.h

@@ -122,6 +122,9 @@ constexpr size_t DEFAULT_MAX_DEPTH = 1024;
 
   #define SIMDJSON_PUSH_DISABLE_WARNINGS _Pragma("GCC diagnostic push")
   // gcc doesn't seem to disable all warnings with all and extra, add warnings here as necessary
+  // We do it separately for clang since it has different warnings.
+  #ifdef __clang__
+  // clang is missing -Wmaybe-uninitialized.
   #define SIMDJSON_PUSH_DISABLE_ALL_WARNINGS SIMDJSON_PUSH_DISABLE_WARNINGS \
     SIMDJSON_DISABLE_GCC_WARNING(-Weffc++) \
     SIMDJSON_DISABLE_GCC_WARNING(-Wall) \
@@ -134,6 +137,22 @@ constexpr size_t DEFAULT_MAX_DEPTH = 1024;
     SIMDJSON_DISABLE_GCC_WARNING(-Wshadow) \
     SIMDJSON_DISABLE_GCC_WARNING(-Wunused-parameter) \
     SIMDJSON_DISABLE_GCC_WARNING(-Wunused-variable)
+  #else // __clang__
+  #define SIMDJSON_PUSH_DISABLE_ALL_WARNINGS SIMDJSON_PUSH_DISABLE_WARNINGS \
+    SIMDJSON_DISABLE_GCC_WARNING(-Weffc++) \
+    SIMDJSON_DISABLE_GCC_WARNING(-Wall) \
+    SIMDJSON_DISABLE_GCC_WARNING(-Wconversion) \
+    SIMDJSON_DISABLE_GCC_WARNING(-Wextra) \
+    SIMDJSON_DISABLE_GCC_WARNING(-Wattributes) \
+    SIMDJSON_DISABLE_GCC_WARNING(-Wimplicit-fallthrough) \
+    SIMDJSON_DISABLE_GCC_WARNING(-Wnon-virtual-dtor) \
+    SIMDJSON_DISABLE_GCC_WARNING(-Wreturn-type) \
+    SIMDJSON_DISABLE_GCC_WARNING(-Wshadow) \
+    SIMDJSON_DISABLE_GCC_WARNING(-Wunused-parameter) \
+    SIMDJSON_DISABLE_GCC_WARNING(-Wunused-variable) \
+    SIMDJSON_DISABLE_GCC_WARNING(-Wmaybe-uninitialized)
+  #endif // __clang__
+
   #define SIMDJSON_PRAGMA(P) _Pragma(#P)
   #define SIMDJSON_DISABLE_GCC_WARNING(WARNING) SIMDJSON_PRAGMA(GCC diagnostic ignored #WARNING)
   #if defined(SIMDJSON_CLANG_VISUAL_STUDIO)
@@ -253,11 +272,8 @@ namespace std {
 #endif
 #endif
 
-// Feature flag for partially-implemented "don't require padding" feature
+// The SIMDJSON_CHECK_EOF macro is a feature flag for the "don't require padding"
-// TODO remove once feature complete.
+// feature.
-#ifndef __SIMDJSON_CHECK_EOF
-# define __SIMDJSON_CHECK_EOF 1
-#endif
 
 #if SIMDJSON_CPLUSPLUS17
 // if we have C++, then fallthrough is a default attribute
@@ -268,11 +284,11 @@ namespace std {
 #if __has_attribute(__fallthrough__)
 // we are good to go:
 # define simdjson_fallthrough                    __attribute__((__fallthrough__))
-#endif
+#endif // __has_attribute(__fallthrough__)
-#endif
+#endif // SIMDJSON_CPLUSPLUS17
 // on some systems, we simply do not have support for fallthrough, so use a default:
 #ifndef simdjson_fallthrough
 # define simdjson_fallthrough do {} while (0)  /* fallthrough */
-#endif
+#endif // simdjson_fallthrough
 
 #endif // SIMDJSON_COMMON_DEFS_H

include/simdjson/dom/document_stream-inl.h

@@ -256,6 +256,7 @@ inline size_t document_stream::size_in_bytes() const noexcept {
 }
 
 inline size_t document_stream::truncated_bytes() const noexcept {
+  if(error == CAPACITY) { return len - batch_start; }
   return parser->implementation->structural_indexes[parser->implementation->n_structural_indexes] - parser->implementation->structural_indexes[parser->implementation->n_structural_indexes + 1];
 }
 

include/simdjson/dom/parsedjson_iterator.h

@@ -5,7 +5,7 @@
 
 #include <cstring>
 #include <string>
-#include <iostream>
+#include <ostream>
 #include <iterator>
 #include <limits>
 #include <stdexcept>

include/simdjson/dom/parser-inl.h

@@ -176,7 +176,7 @@ inline error_code parser::allocate(size_t capacity, size_t max_depth) noexcept {
   if (implementation) {
     err = implementation->allocate(capacity, max_depth);
   } else {
-    err = simdjson::active_implementation->create_dom_parser_implementation(capacity, max_depth, implementation);
+    err = simdjson::get_active_implementation()->create_dom_parser_implementation(capacity, max_depth, implementation);
   }
   if (err) { return err; }
   return SUCCESS;

include/simdjson/dom/serialization-inl.h

@@ -157,7 +157,10 @@ simdjson_really_inline void mini_formatter::string(std::string_view unescaped) {
   size_t i = 0;
   // Fast path for the case where we have no control character, no ", and no backslash.
   // This should include most keys.
-  constexpr static bool needs_escaping[] = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  //
+  // We would like to use 'bool' but some compilers take offense to bitwise operation
+  // with bool types.
+  constexpr static char needs_escaping[] = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0,
     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0,
@@ -169,6 +172,8 @@ simdjson_really_inline void mini_formatter::string(std::string_view unescaped) {
     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
   for(;i + 8 <= unescaped.length(); i += 8) {
     // Poor's man vectorization. This could get much faster if we used SIMD.
+    //
+    // It is not the case that replacing '|' with '||' would be neutral performance-wise.
     if(needs_escaping[uint8_t(unescaped[i])] | needs_escaping[uint8_t(unescaped[i+1])]
       | needs_escaping[uint8_t(unescaped[i+2])] | needs_escaping[uint8_t(unescaped[i+3])]
       | needs_escaping[uint8_t(unescaped[i+4])] | needs_escaping[uint8_t(unescaped[i+5])]

include/simdjson/error.h

@@ -7,7 +7,12 @@
 namespace simdjson {
 
 /**
- * All possible errors returned by simdjson.
+ * All possible errors returned by simdjson. These error codes are subject to change
+ * and not all simdjson kernel returns the same error code given the same input: it is not
+ * well defined which error a given input should produce.
+ *
+ * Only SUCCESS evaluates to false as a Boolean. All other error codes will evaluate
+ * to true as a Boolean.
  */
 enum error_code {
   SUCCESS = 0,                ///< No error
@@ -38,6 +43,8 @@ enum error_code {
   OUT_OF_ORDER_ITERATION,     ///< tried to iterate an array or object out of order
   INSUFFICIENT_PADDING,       ///< The JSON doesn't have enough padding for simdjson to safely parse it.
   INCOMPLETE_ARRAY_OR_OBJECT, ///< The document ends early.
+  SCALAR_DOCUMENT_AS_VALUE,   ///< A scalar document is treated as a value.
+  OUT_OF_BOUNDS,              ///< Attempted to access location outside of document.
   NUM_ERROR_CODES
 };
 
@@ -287,7 +294,7 @@ struct simdjson_result : public internal::simdjson_result_base<T> {
 #if SIMDJSON_EXCEPTIONS
 
 template<typename T>
-inline std::ostream& operator<<(std::ostream& out, simdjson_result<T> value) noexcept { return out << value.value(); }
+inline std::ostream& operator<<(std::ostream& out, simdjson_result<T> value) { return out << value.value(); }
 #endif // SIMDJSON_EXCEPTIONS
 
 #ifndef SIMDJSON_DISABLE_DEPRECATED_API

include/simdjson/fallback/implementation.h

@@ -25,6 +25,7 @@ public:
   ) const noexcept final;
   simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final;
   simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final;
+  simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst) const noexcept final;
 };
 
 } // namespace fallback

include/simdjson/fallback/numberparsing.h

@@ -27,7 +27,8 @@ static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t
 } // namespace SIMDJSON_IMPLEMENTATION
 } // namespace simdjson
 
-#define SWAR_NUMBER_PARSING
+#define SIMDJSON_SWAR_NUMBER_PARSING 1
+
 #include "simdjson/generic/numberparsing.h"
 
 #endif // SIMDJSON_FALLBACK_NUMBERPARSING_H

include/simdjson/generic/implementation_simdjson_result_base-inl.h

@@ -49,18 +49,23 @@ simdjson_really_inline implementation_simdjson_result_base<T>::operator T&&() &&
   return std::forward<implementation_simdjson_result_base<T>>(*this).take_value();
 }
 
+#endif // SIMDJSON_EXCEPTIONS
+
 template<typename T>
 simdjson_really_inline const T& implementation_simdjson_result_base<T>::value_unsafe() const& noexcept {
   return this->first;
 }
 
+template<typename T>
+simdjson_really_inline T& implementation_simdjson_result_base<T>::value_unsafe() & noexcept {
+  return this->first;
+}
+
 template<typename T>
 simdjson_really_inline T&& implementation_simdjson_result_base<T>::value_unsafe() && noexcept {
   return std::forward<T>(this->first);
 }
 
-#endif // SIMDJSON_EXCEPTIONS
-
 template<typename T>
 simdjson_really_inline implementation_simdjson_result_base<T>::implementation_simdjson_result_base(T &&value, error_code error) noexcept
     : first{std::forward<T>(value)}, second{error} {}

include/simdjson/generic/implementation_simdjson_result_base.h

@@ -97,22 +97,28 @@ struct implementation_simdjson_result_base {
    */
   simdjson_really_inline operator T&&() && noexcept(false);
 
+
+#endif // SIMDJSON_EXCEPTIONS
+
   /**
    * Get the result value. This function is safe if and only
    * the error() method returns a value that evaluates to false.
    */
   simdjson_really_inline const T& value_unsafe() const& noexcept;
-
+  /**
+   * Get the result value. This function is safe if and only
+   * the error() method returns a value that evaluates to false.
+   */
+  simdjson_really_inline T& value_unsafe() & noexcept;
   /**
    * Take the result value (move it). This function is safe if and only
    * the error() method returns a value that evaluates to false.
    */
   simdjson_really_inline T&& value_unsafe() && noexcept;
-
+protected:
-#endif // SIMDJSON_EXCEPTIONS
+  /** users should never directly access first and second. **/
-
+  T first{}; /** Users should never directly access 'first'. **/
-  T first{};
+  error_code second{UNINITIALIZED}; /** Users should never directly access 'second'. **/
-  error_code second{UNINITIALIZED};
 }; // struct implementation_simdjson_result_base
 
 } // namespace SIMDJSON_IMPLEMENTATION

include/simdjson/generic/numberparsing.h

@@ -3,6 +3,18 @@
 
 namespace simdjson {
 namespace SIMDJSON_IMPLEMENTATION {
+
+namespace ondemand {
+/**
+ * The type of a JSON number
+ */
+enum class number_type {
+    floating_point_number=1, /// a binary64 number
+    signed_integer,          /// a signed integer that fits in a 64-bit word using two's complement
+    unsigned_integer         /// a positive integer larger or equal to 1<<63
+};
+}
+
 namespace {
 /// @private
 namespace numberparsing {
@@ -349,7 +361,7 @@ error_code slow_float_parsing(simdjson_unused const uint8_t * src, W writer) {
 }
 
 template<typename I>
-NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later
+SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later
 simdjson_really_inline bool parse_digit(const uint8_t c, I &i) {
   const uint8_t digit = static_cast<uint8_t>(c - '0');
   if (digit > 9) {
@@ -367,14 +379,16 @@ simdjson_really_inline error_code parse_decimal(simdjson_unused const uint8_t *c
   // the integer into a float in a lossless manner.
   const uint8_t *const first_after_period = p;
 
-#ifdef SWAR_NUMBER_PARSING
+#ifdef SIMDJSON_SWAR_NUMBER_PARSING
+#if SIMDJSON_SWAR_NUMBER_PARSING
   // this helps if we have lots of decimals!
   // this turns out to be frequent enough.
   if (is_made_of_eight_digits_fast(p)) {
     i = i * 100000000 + parse_eight_digits_unrolled(p);
     p += 8;
   }
-#endif
+#endif // SIMDJSON_SWAR_NUMBER_PARSING
+#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING
   // Unrolling the first digit makes a small difference on some implementations (e.g. westmere)
   if (parse_digit(*p, i)) { ++p; }
   while (parse_digit(*p, i)) { p++; }
@@ -441,9 +455,7 @@ simdjson_really_inline size_t significant_digits(const uint8_t * start_digits, s
   // It is possible that the integer had an overflow.
   // We have to handle the case where we have 0.0000somenumber.
   const uint8_t *start = start_digits;
-  while ((*start == '0') || (*start == '.')) {
+  while ((*start == '0') || (*start == '.')) { ++start; }
-    start++;
-  }
   // we over-decrement by one when there is a '.'
   return digit_count - size_t(start - start_digits);
 }
@@ -513,7 +525,12 @@ simdjson_really_inline error_code parse_number(const uint8_t *const, W &writer)
 simdjson_unused simdjson_really_inline simdjson_result<uint64_t> parse_unsigned(const uint8_t * const src) noexcept { return 0; }
 simdjson_unused simdjson_really_inline simdjson_result<int64_t> parse_integer(const uint8_t * const src) noexcept { return 0; }
 simdjson_unused simdjson_really_inline simdjson_result<double> parse_double(const uint8_t * const src) noexcept { return 0; }
-
+simdjson_unused simdjson_really_inline simdjson_result<uint64_t> parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; }
+simdjson_unused simdjson_really_inline simdjson_result<int64_t> parse_integer_in_string(const uint8_t * const src) noexcept { return 0; }
+simdjson_unused simdjson_really_inline simdjson_result<double> parse_double_in_string(const uint8_t * const src) noexcept { return 0; }
+simdjson_unused simdjson_really_inline bool is_negative(const uint8_t * src) noexcept  { return false; }
+simdjson_unused simdjson_really_inline simdjson_result<bool> is_integer(const uint8_t * src) noexcept  { return false; }
+simdjson_unused simdjson_really_inline simdjson_result<ondemand::number_type> get_number_type(const uint8_t * src) noexcept { return ondemand::number_type::signed_integer; }
 #else
 
 // parse the number at src
@@ -592,7 +609,7 @@ simdjson_really_inline error_code parse_number(const uint8_t *const src, W &writ
     // - That is smaller than the smallest possible 20-digit number the user could write:
     //   10,000,000,000,000,000,000.
     // - Therefore, if the number is positive and lower than that, it's overflow.
-    // - The value we are looking at is less than or equal to 9,223,372,036,854,775,808 (INT64_MAX).
+    // - The value we are looking at is less than or equal to INT64_MAX.
     //
     }  else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); }
   }
@@ -715,7 +732,7 @@ simdjson_unused simdjson_really_inline simdjson_result<uint64_t> parse_unsigned(
     // - That is smaller than the smallest possible 20-digit number the user could write:
     //   10,000,000,000,000,000,000.
     // - Therefore, if the number is positive and lower than that, it's overflow.
-    // - The value we are looking at is less than or equal to 9,223,372,036,854,775,808 (INT64_MAX).
+    // - The value we are looking at is less than or equal to INT64_MAX.
     //
     if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; }
   }
@@ -765,7 +782,7 @@ simdjson_unused simdjson_really_inline simdjson_result<uint64_t> parse_unsigned(
     // - That is smaller than the smallest possible 20-digit number the user could write:
     //   10,000,000,000,000,000,000.
     // - Therefore, if the number is positive and lower than that, it's overflow.
-    // - The value we are looking at is less than or equal to 9,223,372,036,854,775,808 (INT64_MAX).
+    // - The value we are looking at is less than or equal to INT64_MAX.
     //
     if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; }
   }
@@ -773,6 +790,56 @@ simdjson_unused simdjson_really_inline simdjson_result<uint64_t> parse_unsigned(
   return i;
 }
 
+// Parse any number from 0 to 18,446,744,073,709,551,615
+simdjson_unused simdjson_really_inline simdjson_result<uint64_t> parse_unsigned_in_string(const uint8_t * const src) noexcept {
+  const uint8_t *p = src + 1;
+  //
+  // Parse the integer part.
+  //
+  // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare
+  const uint8_t *const start_digits = p;
+  uint64_t i = 0;
+  while (parse_digit(*p, i)) { p++; }
+
+  // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error.
+  // Optimization note: size_t is expected to be unsigned.
+  size_t digit_count = size_t(p - start_digits);
+  // The longest positive 64-bit number is 20 digits.
+  // We do it this way so we don't trigger this branch unless we must.
+  // Optimization note: the compiler can probably merge
+  // ((digit_count == 0) || (digit_count > 20))
+  // into a single  branch since digit_count is unsigned.
+  if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; }
+  // Here digit_count > 0.
+  if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; }
+  // We can do the following...
+  // if (!jsoncharutils::is_structural_or_whitespace(*p)) {
+  //  return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR;
+  // }
+  // as a single table lookup:
+  if (*p != '"') { return NUMBER_ERROR; }
+
+  if (digit_count == 20) {
+    // Positive overflow check:
+    // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the
+    //   biggest uint64_t.
+    // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX.
+    //   If we got here, it's a 20 digit number starting with the digit "1".
+    // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller
+    //   than 1,553,255,926,290,448,384.
+    // - That is smaller than the smallest possible 20-digit number the user could write:
+    //   10,000,000,000,000,000,000.
+    // - Therefore, if the number is positive and lower than that, it's overflow.
+    // - The value we are looking at is less than or equal to INT64_MAX.
+    //
+    // Note: we use src[1] and not src[0] because src[0] is the quote character in this
+    // instance.
+    if (src[1] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; }
+  }
+
+  return i;
+}
+
 // Parse any number from  -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807
 simdjson_unused simdjson_really_inline simdjson_result<int64_t> parse_integer(const uint8_t *src) noexcept {
   //
@@ -859,6 +926,48 @@ simdjson_unused simdjson_really_inline simdjson_result<int64_t> parse_integer(co
   return negative ? (~i+1) : i;
 }
 
+// Parse any number from  -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807
+simdjson_unused simdjson_really_inline simdjson_result<int64_t> parse_integer_in_string(const uint8_t *src) noexcept {
+  //
+  // Check for minus sign
+  //
+  bool negative = (*(src + 1) == '-');
+  const uint8_t *p = src + negative + 1;
+
+  //
+  // Parse the integer part.
+  //
+  // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare
+  const uint8_t *const start_digits = p;
+  uint64_t i = 0;
+  while (parse_digit(*p, i)) { p++; }
+
+  // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error.
+  // Optimization note: size_t is expected to be unsigned.
+  size_t digit_count = size_t(p - start_digits);
+  // We go from
+  // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807
+  // so we can never represent numbers that have more than 19 digits.
+  size_t longest_digit_count = 19;
+  // Optimization note: the compiler can probably merge
+  // ((digit_count == 0) || (digit_count > longest_digit_count))
+  // into a single  branch since digit_count is unsigned.
+  if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; }
+  // Here digit_count > 0.
+  if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; }
+  // We can do the following...
+  // if (!jsoncharutils::is_structural_or_whitespace(*p)) {
+  //  return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR;
+  // }
+  // as a single table lookup:
+  if(*p != '"') { return NUMBER_ERROR; }
+  // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX.
+  // Performance note: This check is only needed when digit_count == longest_digit_count but it is
+  // so cheap that we might as well always make it.
+  if(i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; }
+  return negative ? (~i+1) : i;
+}
+
 simdjson_unused simdjson_really_inline simdjson_result<double> parse_double(const uint8_t * src) noexcept {
   //
   // Check for minus sign
@@ -937,6 +1046,44 @@ simdjson_unused simdjson_really_inline simdjson_result<double> parse_double(cons
   return d;
 }
 
+simdjson_unused simdjson_really_inline bool is_negative(const uint8_t * src) noexcept {
+  return (*src == '-');
+}
+
+simdjson_unused simdjson_really_inline simdjson_result<bool> is_integer(const uint8_t * src) noexcept {
+  bool negative = (*src == '-');
+  src += negative;
+  const uint8_t *p = src;
+  while(static_cast<uint8_t>(*p - '0') <= 9) { p++; }
+  if ( p == src ) { return NUMBER_ERROR; }
+  if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; }
+  return false;
+}
+
+simdjson_unused simdjson_really_inline simdjson_result<ondemand::number_type> get_number_type(const uint8_t * src) noexcept {
+  bool negative = (*src == '-');
+  src += negative;
+  const uint8_t *p = src;
+  while(static_cast<uint8_t>(*p - '0') <= 9) { p++; }
+  if ( p == src ) { return NUMBER_ERROR; }
+  if (jsoncharutils::is_structural_or_whitespace(*p)) {
+    // We have an integer.
+    // If the number is negative and valid, it must be a signed integer.
+    if(negative) { return ondemand::number_type::signed_integer; }
+    // We want values larger or equal to 9223372036854775808 to be unsigned
+    // integers, and the other values to be signed integers.
+    int digit_count = int(p - src);
+    if(digit_count >= 19) {
+      const uint8_t * smaller_big_integer = reinterpret_cast<const uint8_t *>("9223372036854775808");
+      if((digit_count >= 20) || (memcmp(src, smaller_big_integer, 19) >= 0)) {
+        return ondemand::number_type::unsigned_integer;
+      }
+    }
+    return ondemand::number_type::signed_integer;
+  }
+  // Hopefully, we have 'e' or 'E' or '.'.
+  return ondemand::number_type::floating_point_number;
+}
 
 // Never read at src_end or beyond
 simdjson_unused simdjson_really_inline simdjson_result<double> parse_double(const uint8_t * src, const uint8_t * const src_end) noexcept {
@@ -1020,6 +1167,83 @@ simdjson_unused simdjson_really_inline simdjson_result<double> parse_double(cons
   return d;
 }
 
+simdjson_unused simdjson_really_inline simdjson_result<double> parse_double_in_string(const uint8_t * src) noexcept {
+  //
+  // Check for minus sign
+  //
+  bool negative = (*(src + 1) == '-');
+  src += negative + 1;
+
+  //
+  // Parse the integer part.
+  //
+  uint64_t i = 0;
+  const uint8_t *p = src;
+  p += parse_digit(*p, i);
+  bool leading_zero = (i == 0);
+  while (parse_digit(*p, i)) { p++; }
+  // no integer digits, or 0123 (zero must be solo)
+  if ( p == src ) { return INCORRECT_TYPE; }
+  if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; }
+
+  //
+  // Parse the decimal part.
+  //
+  int64_t exponent = 0;
+  bool overflow;
+  if (simdjson_likely(*p == '.')) {
+    p++;
+    const uint8_t *start_decimal_digits = p;
+    if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits
+    p++;
+    while (parse_digit(*p, i)) { p++; }
+    exponent = -(p - start_decimal_digits);
+
+    // Overflow check. More than 19 digits (minus the decimal) may be overflow.
+    overflow = p-src-1 > 19;
+    if (simdjson_unlikely(overflow && leading_zero)) {
+      // Skip leading 0.00000 and see if it still overflows
+      const uint8_t *start_digits = src + 2;
+      while (*start_digits == '0') { start_digits++; }
+      overflow = start_digits-src > 19;
+    }
+  } else {
+    overflow = p-src > 19;
+  }
+
+  //
+  // Parse the exponent
+  //
+  if (*p == 'e' || *p == 'E') {
+    p++;
+    bool exp_neg = *p == '-';
+    p += exp_neg || *p == '+';
+
+    uint64_t exp = 0;
+    const uint8_t *start_exp_digits = p;
+    while (parse_digit(*p, exp)) { p++; }
+    // no exp digits, or 20+ exp digits
+    if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; }
+
+    exponent += exp_neg ? 0-exp : exp;
+  }
+
+  if (*p != '"') { return NUMBER_ERROR; }
+
+  overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power;
+
+  //
+  // Assemble (or slow-parse) the float
+  //
+  double d;
+  if (simdjson_likely(!overflow)) {
+    if (compute_float_64(exponent, i, negative, d)) { return d; }
+  }
+  if (!parse_float_fallback(src-negative, &d)) {
+    return NUMBER_ERROR;
+  }
+  return d;
+}
 } //namespace {}
 #endif // SIMDJSON_SKIPNUMBERPARSING
 

include/simdjson/generic/ondemand/array-inl.h

@@ -82,10 +82,15 @@ simdjson_really_inline simdjson_result<std::string_view> array::raw_json() noexc
   const uint8_t * starting_point{iter.peek_start()};
   auto error = consume();
   if(error) { return error; }
-  const uint8_t * final_point{iter._json_iter->peek(0)};
+  // After 'consume()', we could be left pointing just beyond the document, but that
+  // is ok because we are not going to dereference the final pointer position, we just
+  // use it to compute the length in bytes.
+  const uint8_t * final_point{iter._json_iter->unsafe_pointer()};
   return std::string_view(reinterpret_cast<const char*>(starting_point), size_t(final_point - starting_point));
 }
 
+SIMDJSON_PUSH_DISABLE_WARNINGS
+SIMDJSON_DISABLE_STRICT_OVERFLOW_WARNING
 simdjson_really_inline simdjson_result<size_t> array::count_elements() & noexcept {
   size_t count{0};
   // Important: we do not consume any of the values.
@@ -97,6 +102,18 @@ simdjson_really_inline simdjson_result<size_t> array::count_elements() & noexcep
   iter.reset_array();
   return count;
 }
+SIMDJSON_POP_DISABLE_WARNINGS
+
+simdjson_really_inline simdjson_result<bool> array::is_empty() & noexcept {
+  bool is_not_empty;
+  auto error = iter.reset_array().get(is_not_empty);
+  if(error) { return error; }
+  return !is_not_empty;
+}
+
+inline simdjson_result<bool> array::reset() & noexcept {
+  return iter.reset_array();
+}
 
 inline simdjson_result<value> array::at_pointer(std::string_view json_pointer) noexcept {
   if (json_pointer[0] != '/') { return INVALID_JSON_POINTER; }
@@ -176,6 +193,10 @@ simdjson_really_inline  simdjson_result<size_t> simdjson_result<SIMDJSON_IMPLEME
   if (error()) { return error(); }
   return first.count_elements();
 }
+simdjson_really_inline  simdjson_result<bool> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array>::is_empty() & noexcept {
+  if (error()) { return error(); }
+  return first.is_empty();
+}
 simdjson_really_inline  simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array>::at(size_t index) noexcept {
   if (error()) { return error(); }
   return first.at(index);

include/simdjson/generic/ondemand/array.h

@@ -41,9 +41,32 @@ public:
    * beginning as if it had never been accessed. If the JSON is malformed (e.g.,
    * there is a missing comma), then an error is returned and it is no longer
    * safe to continue.
+   *
+   * To check that an array is empty, it is more performant to use
+   * the is_empty() method.
    */
   simdjson_really_inline simdjson_result<size_t> count_elements() & noexcept;
-
+  /**
+   * This method scans the beginning of the array and checks whether the
+   * array is empty.
+   * The runtime complexity is constant time. After
+   * calling this function, if successful, the array is 'rewinded' at its
+   * beginning as if it had never been accessed. If the JSON is malformed (e.g.,
+   * there is a missing comma), then an error is returned and it is no longer
+   * safe to continue.
+   */
+  simdjson_really_inline simdjson_result<bool> is_empty() & noexcept;
+  /**
+   * Reset the iterator so that we are pointing back at the
+   * beginning of the array. You should still consume values only once even if you
+   * can iterate through the array more than once. If you unescape a string
+   * within the array more than once, you have unsafe code. Note that rewinding
+   * an array means that you may need to reparse it anew: it is not a free
+   * operation.
+   *
+   * @returns true if the array contains some elements (not empty)
+   */
+  inline simdjson_result<bool> reset() & noexcept;
   /**
    * Get the value associated with the given JSON pointer.  We use the RFC 6901
    * https://tools.ietf.org/html/rfc6901 standard, interpreting the current node
@@ -79,6 +102,14 @@ public:
    */
   simdjson_really_inline simdjson_result<std::string_view> raw_json() noexcept;
 
+  /**
+   * Get the value at the given index. This function has linear-time complexity.
+   * This function should only be called once on an array instance since the array iterator is not reset between each call.
+   *
+   * @return The value at the given index, or:
+   *         - INDEX_OUT_OF_BOUNDS if the array index is larger than an array length
+   */
+  simdjson_really_inline simdjson_result<value> at(size_t index) noexcept;
 protected:
   /**
    * Go to the end of the array, no matter where you are right now.
@@ -121,15 +152,6 @@ protected:
    */
   simdjson_really_inline array(const value_iterator &iter) noexcept;
 
-  /**
-   * Get the value at the given index. This function has linear-time complexity.
-   * This function should only be called once as the array iterator is not reset between each call.
-   *
-   * @return The value at the given index, or:
-   *         - INDEX_OUT_OF_BOUNDS if the array index is larger than an array length
-   */
-  simdjson_really_inline simdjson_result<value> at(size_t index) noexcept;
-
   /**
    * Iterator marking current position.
    *
@@ -159,7 +181,9 @@ public:
 
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> begin() noexcept;
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> end() noexcept;
-  simdjson_really_inline simdjson_result<size_t> count_elements() & noexcept;
+  inline simdjson_result<size_t> count_elements() & noexcept;
+  inline simdjson_result<bool> is_empty() & noexcept;
+  inline simdjson_result<bool> reset() & noexcept;
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> at(size_t index) noexcept;
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> at_pointer(std::string_view json_pointer) noexcept;
 };

include/simdjson/generic/ondemand/document-inl.h

@@ -20,6 +20,17 @@ inline std::string document::to_debug_string() noexcept {
   return iter.to_string();
 }
 
+inline simdjson_result<const char *> document::current_location() noexcept {
+  return iter.current_location();
+}
+
+inline int32_t document::current_depth() const noexcept {
+  return iter.depth();
+}
+
+inline bool document::is_alive() noexcept {
+  return iter.is_alive();
+}
 simdjson_really_inline value_iterator document::resume_value_iterator() noexcept {
   return value_iterator(&iter, 1, iter.root_position());
 }
@@ -33,24 +44,19 @@ simdjson_really_inline simdjson_result<object> document::start_or_resume_object(
     return object::resume(resume_value_iterator());
   }
 }
-simdjson_really_inline simdjson_result<value> document::get_value_unsafe() noexcept {
+simdjson_really_inline simdjson_result<value> document::get_value() noexcept {
   // Make sure we start any arrays or objects before returning, so that start_root_<object/array>()
   // gets called.
+  iter.assert_at_document_depth();
   switch (*iter.peek()) {
-    case '[': {
+    case '[':
-      array result;
+    case '{':
-      SIMDJSON_TRY( get_array().get(result) );
-      return value(result.iter);
-    }
-    case '{': {
-      object result;
-      SIMDJSON_TRY( get_object().get(result) );
-      return value(result.iter);
-    }
-    default:
-      // TODO it is still wrong to convert this to a value! get_root_bool / etc. will not be
-      // called if you do this.
       return value(get_root_value_iterator());
+    default:
+      // Unfortunately, scalar documents are a special case in simdjson and they cannot
+      // be safely converted to value instances.
+      return SCALAR_DOCUMENT_AS_VALUE;
+      // return value(get_root_value_iterator());
   }
 }
 simdjson_really_inline simdjson_result<array> document::get_array() & noexcept {
@@ -64,12 +70,21 @@ simdjson_really_inline simdjson_result<object> document::get_object() & noexcept
 simdjson_really_inline simdjson_result<uint64_t> document::get_uint64() noexcept {
   return get_root_value_iterator().get_root_uint64();
 }
+simdjson_really_inline simdjson_result<uint64_t> document::get_uint64_in_string() noexcept {
+  return get_root_value_iterator().get_root_uint64_in_string();
+}
 simdjson_really_inline simdjson_result<int64_t> document::get_int64() noexcept {
   return get_root_value_iterator().get_root_int64();
 }
+simdjson_really_inline simdjson_result<int64_t> document::get_int64_in_string() noexcept {
+  return get_root_value_iterator().get_root_int64_in_string();
+}
 simdjson_really_inline simdjson_result<double> document::get_double() noexcept {
   return get_root_value_iterator().get_root_double();
 }
+simdjson_really_inline simdjson_result<double> document::get_double_in_string() noexcept {
+  return get_root_value_iterator().get_root_double_in_string();
+}
 simdjson_really_inline simdjson_result<std::string_view> document::get_string() noexcept {
   return get_root_value_iterator().get_root_string();
 }
@@ -91,6 +106,7 @@ template<> simdjson_really_inline simdjson_result<double> document::get() & noex
 template<> simdjson_really_inline simdjson_result<uint64_t> document::get() & noexcept { return get_uint64(); }
 template<> simdjson_really_inline simdjson_result<int64_t> document::get() & noexcept { return get_int64(); }
 template<> simdjson_really_inline simdjson_result<bool> document::get() & noexcept { return get_bool(); }
+template<> simdjson_really_inline simdjson_result<value> document::get() & noexcept { return get_value(); }
 
 template<> simdjson_really_inline simdjson_result<raw_json_string> document::get() && noexcept { return get_raw_json_string(); }
 template<> simdjson_really_inline simdjson_result<std::string_view> document::get() && noexcept { return get_string(); }
@@ -98,6 +114,7 @@ template<> simdjson_really_inline simdjson_result<double> document::get() && noe
 template<> simdjson_really_inline simdjson_result<uint64_t> document::get() && noexcept { return std::forward<document>(*this).get_uint64(); }
 template<> simdjson_really_inline simdjson_result<int64_t> document::get() && noexcept { return std::forward<document>(*this).get_int64(); }
 template<> simdjson_really_inline simdjson_result<bool> document::get() && noexcept { return std::forward<document>(*this).get_bool(); }
+template<> simdjson_really_inline simdjson_result<value> document::get() && noexcept { return get_value(); }
 
 template<typename T> simdjson_really_inline error_code document::get(T &out) & noexcept {
   return get<T>().get(out);
@@ -115,14 +132,33 @@ simdjson_really_inline document::operator double() noexcept(false) { return get_
 simdjson_really_inline document::operator std::string_view() noexcept(false) { return get_string(); }
 simdjson_really_inline document::operator raw_json_string() noexcept(false) { return get_raw_json_string(); }
 simdjson_really_inline document::operator bool() noexcept(false) { return get_bool(); }
+simdjson_really_inline document::operator value() noexcept(false) { return get_value(); }
+
 #endif
 simdjson_really_inline simdjson_result<size_t> document::count_elements() & noexcept {
   auto a = get_array();
   simdjson_result<size_t> answer = a.count_elements();
   /* If there was an array, we are now left pointing at its first element. */
-  if(answer.error() == SUCCESS) { iter._depth -= 1 ; /* undoing the increment so we go back at the doc depth.*/ }
+  if(answer.error() == SUCCESS) {
+    iter._depth = 1 ; /* undoing the increment so we go back at the doc depth.*/
+    iter.assert_at_document_depth();
+  }
   return answer;
 }
+simdjson_really_inline simdjson_result<size_t> document::count_fields() & noexcept {
+  auto a = get_object();
+  simdjson_result<size_t> answer = a.count_fields();
+  /* If there was an array, we are now left pointing at its first element. */
+  if(answer.error() == SUCCESS) {
+    iter._depth = 1 ; /* undoing the increment so we go back at the doc depth.*/
+    iter.assert_at_document_depth();
+  }
+  return answer;
+}
+simdjson_really_inline simdjson_result<value> document::at(size_t index) & noexcept {
+  auto a = get_array();
+  return a.at(index);
+}
 simdjson_really_inline simdjson_result<array_iterator> document::begin() & noexcept {
   return get_array().begin();
 }
@@ -156,12 +192,14 @@ simdjson_really_inline error_code document::consume() noexcept {
 }
 
 simdjson_really_inline simdjson_result<std::string_view> document::raw_json() noexcept {
-  printf("document::raw_json()\n");
   auto _iter = get_root_value_iterator();
   const uint8_t * starting_point{_iter.peek_start()};
   auto error = consume();
   if(error) { return error; }
-  const uint8_t * final_point{iter.peek(0)};
+  // After 'consume()', we could be left pointing just beyond the document, but that
+  // is ok because we are not going to dereference the final pointer position, we just
+  // use it to compute the length in bytes.
+  const uint8_t * final_point{iter.unsafe_pointer()};
   return std::string_view(reinterpret_cast<const char*>(starting_point), size_t(final_point - starting_point));
 }
 
@@ -169,6 +207,30 @@ simdjson_really_inline simdjson_result<json_type> document::type() noexcept {
   return get_root_value_iterator().type();
 }
 
+simdjson_really_inline simdjson_result<bool> document::is_scalar() noexcept {
+  json_type this_type;
+  auto error = type().get(this_type);
+  if(error) { return error; }
+  return ! ((this_type == json_type::array) || (this_type == json_type::object));
+}
+
+simdjson_really_inline bool document::is_negative() noexcept {
+  return get_root_value_iterator().is_root_negative();
+}
+
+simdjson_really_inline simdjson_result<bool> document::is_integer() noexcept {
+  return get_root_value_iterator().is_root_integer();
+}
+
+simdjson_really_inline simdjson_result<number_type> document::get_number_type() noexcept {
+  return get_root_value_iterator().get_root_number_type();
+}
+
+simdjson_really_inline simdjson_result<number> document::get_number() noexcept {
+  return get_root_value_iterator().get_root_number();
+}
+
+
 simdjson_really_inline simdjson_result<std::string_view> document::raw_json_token() noexcept {
   auto _iter = get_root_value_iterator();
   return std::string_view(reinterpret_cast<const char*>(_iter.peek_start()), _iter.peek_start_length());
@@ -177,7 +239,7 @@ simdjson_really_inline simdjson_result<std::string_view> document::raw_json_toke
 simdjson_really_inline simdjson_result<value> document::at_pointer(std::string_view json_pointer) noexcept {
   rewind(); // Rewind the document each time at_pointer is called
   if (json_pointer.empty()) {
-    return this->get_value_unsafe();
+    return this->get_value();
   }
   json_type t;
   SIMDJSON_TRY(type().get(t));
@@ -218,6 +280,14 @@ simdjson_really_inline simdjson_result<size_t> simdjson_result<SIMDJSON_IMPLEMEN
   if (error()) { return error(); }
   return first.count_elements();
 }
+simdjson_really_inline simdjson_result<size_t> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::count_fields() & noexcept {
+  if (error()) { return error(); }
+  return first.count_fields();
+}
+simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::at(size_t index) & noexcept {
+  if (error()) { return error(); }
+  return first.at(index);
+}
 simdjson_really_inline error_code simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::rewind() noexcept {
   if (error()) { return error(); }
   first.rewind();
@@ -286,6 +356,10 @@ simdjson_really_inline simdjson_result<bool> simdjson_result<SIMDJSON_IMPLEMENTA
   if (error()) { return error(); }
   return first.get_bool();
 }
+simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::get_value() noexcept {
+  if (error()) { return error(); }
+  return first.get_value();
+}
 simdjson_really_inline bool simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::is_null() noexcept {
   if (error()) { return error(); }
   return first.is_null();
@@ -329,6 +403,33 @@ simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::json_t
   return first.type();
 }
 
+simdjson_really_inline simdjson_result<bool> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::is_scalar() noexcept {
+  if (error()) { return error(); }
+  return first.is_scalar();
+}
+
+
+simdjson_really_inline bool simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::is_negative() noexcept {
+  if (error()) { return error(); }
+  return first.is_negative();
+}
+
+simdjson_really_inline simdjson_result<bool> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::is_integer() noexcept {
+  if (error()) { return error(); }
+  return first.is_integer();
+}
+
+simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number_type> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::get_number_type() noexcept {
+  if (error()) { return error(); }
+  return first.get_number_type();
+}
+
+simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::get_number() noexcept {
+  if (error()) { return error(); }
+  return first.get_number();
+}
+
+
 #if SIMDJSON_EXCEPTIONS
 simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::operator SIMDJSON_IMPLEMENTATION::ondemand::array() & noexcept(false) {
   if (error()) { throw simdjson_error(error()); }
@@ -362,8 +463,23 @@ simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::docume
   if (error()) { throw simdjson_error(error()); }
   return first;
 }
+simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::operator SIMDJSON_IMPLEMENTATION::ondemand::value() noexcept(false) {
+  if (error()) { throw simdjson_error(error()); }
+  return first;
+}
 #endif
 
+
+simdjson_really_inline simdjson_result<const char *> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::current_location() noexcept {
+  if (error()) { return error(); }
+  return first.current_location();
+}
+
+simdjson_really_inline int32_t simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::current_depth() const noexcept {
+  if (error()) { return error(); }
+  return first.current_depth();
+}
+
 simdjson_really_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::raw_json_token() noexcept {
   if (error()) { return error(); }
   return first.raw_json_token();
@@ -374,4 +490,239 @@ simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>
   return first.at_pointer(json_pointer);
 }
 
+
+} // namespace simdjson
+
+
+namespace simdjson {
+namespace SIMDJSON_IMPLEMENTATION {
+namespace ondemand {
+
+simdjson_really_inline document_reference::document_reference() noexcept : doc{nullptr} {}
+simdjson_really_inline document_reference::document_reference(document &d) noexcept : doc(&d) {}
+simdjson_really_inline void document_reference::rewind() noexcept { doc->rewind(); }
+simdjson_really_inline simdjson_result<array> document_reference::get_array() & noexcept { return doc->get_array(); }
+simdjson_really_inline simdjson_result<object> document_reference::get_object() & noexcept { return doc->get_object(); }
+simdjson_really_inline simdjson_result<uint64_t> document_reference::get_uint64() noexcept { return doc->get_uint64(); }
+simdjson_really_inline simdjson_result<int64_t> document_reference::get_int64() noexcept { return doc->get_int64(); }
+simdjson_really_inline simdjson_result<double> document_reference::get_double() noexcept { return doc->get_double(); }
+simdjson_really_inline simdjson_result<std::string_view> document_reference::get_string() noexcept { return doc->get_string(); }
+simdjson_really_inline simdjson_result<raw_json_string> document_reference::get_raw_json_string() noexcept { return doc->get_raw_json_string(); }
+simdjson_really_inline simdjson_result<bool> document_reference::get_bool() noexcept { return doc->get_bool(); }
+simdjson_really_inline simdjson_result<value> document_reference::get_value() noexcept { return doc->get_value(); }
+simdjson_really_inline bool document_reference::is_null() noexcept { return doc->is_null(); }
+
+#if SIMDJSON_EXCEPTIONS
+simdjson_really_inline document_reference::operator array() & noexcept(false) { return array(*doc); }
+simdjson_really_inline document_reference::operator object() & noexcept(false) { return object(*doc); }
+simdjson_really_inline document_reference::operator uint64_t() noexcept(false) { return uint64_t(*doc); }
+simdjson_really_inline document_reference::operator int64_t() noexcept(false) { return int64_t(*doc); }
+simdjson_really_inline document_reference::operator double() noexcept(false) { return double(*doc); }
+simdjson_really_inline document_reference::operator std::string_view() noexcept(false) { return std::string_view(*doc); }
+simdjson_really_inline document_reference::operator raw_json_string() noexcept(false) { return raw_json_string(*doc); }
+simdjson_really_inline document_reference::operator bool() noexcept(false) { return bool(*doc); }
+simdjson_really_inline document_reference::operator value() noexcept(false) { return value(*doc); }
+#endif
+simdjson_really_inline simdjson_result<size_t> document_reference::count_elements() & noexcept { return doc->count_elements(); }
+simdjson_really_inline simdjson_result<size_t> document_reference::count_fields() & noexcept { return doc->count_fields(); }
+simdjson_really_inline simdjson_result<value> document_reference::at(size_t index) & noexcept { return doc->at(index); }
+simdjson_really_inline simdjson_result<array_iterator> document_reference::begin() & noexcept { return doc->begin(); }
+simdjson_really_inline simdjson_result<array_iterator> document_reference::end() & noexcept { return doc->end(); }
+simdjson_really_inline simdjson_result<value> document_reference::find_field(std::string_view key) & noexcept { return doc->find_field(key); }
+simdjson_really_inline simdjson_result<value> document_reference::find_field(const char *key) & noexcept { return doc->find_field(key); }
+simdjson_really_inline simdjson_result<value> document_reference::operator[](std::string_view key) & noexcept { return (*doc)[key]; }
+simdjson_really_inline simdjson_result<value> document_reference::operator[](const char *key) & noexcept { return (*doc)[key]; }
+simdjson_really_inline simdjson_result<value> document_reference::find_field_unordered(std::string_view key) & noexcept { return doc->find_field_unordered(key); }
+simdjson_really_inline simdjson_result<value> document_reference::find_field_unordered(const char *key) & noexcept { return doc->find_field_unordered(key); }
+simdjson_really_inline simdjson_result<json_type> document_reference::type() noexcept { return doc->type(); }
+simdjson_really_inline simdjson_result<bool> document_reference::is_scalar() noexcept { return doc->is_scalar(); }
+simdjson_really_inline simdjson_result<const char *> document_reference::current_location() noexcept { return doc->current_location(); }
+simdjson_really_inline int32_t document_reference::current_depth() const noexcept { return doc->current_depth(); }
+simdjson_really_inline bool document_reference::is_negative() noexcept { return doc->is_negative(); }
+simdjson_really_inline simdjson_result<bool> document_reference::is_integer() noexcept { return doc->is_integer(); }
+simdjson_really_inline simdjson_result<number_type> document_reference::get_number_type() noexcept { return doc->get_number_type(); }
+simdjson_really_inline simdjson_result<number> document_reference::get_number() noexcept { return doc->get_number(); }
+simdjson_really_inline simdjson_result<std::string_view> document_reference::raw_json_token() noexcept { return doc->raw_json_token(); }
+simdjson_really_inline simdjson_result<value> document_reference::at_pointer(std::string_view json_pointer) noexcept { return doc->at_pointer(json_pointer); }
+simdjson_really_inline simdjson_result<std::string_view> document_reference::raw_json() noexcept { return doc->raw_json();}
+simdjson_really_inline document_reference::operator document&() const noexcept { return *doc; }
+
+} // namespace ondemand
+} // namespace SIMDJSON_IMPLEMENTATION
+} // namespace simdjson
+
+
+
+namespace simdjson {
+simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::simdjson_result(SIMDJSON_IMPLEMENTATION::ondemand::document_reference value, error_code error)
+  noexcept : implementation_simdjson_result_base<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>(std::forward<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>(value), error) {}
+
+
+simdjson_really_inline simdjson_result<size_t> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::count_elements() & noexcept {
+  if (error()) { return error(); }
+  return first.count_elements();
+}
+simdjson_really_inline simdjson_result<size_t> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::count_fields() & noexcept {
+  if (error()) { return error(); }
+  return first.count_fields();
+}
+simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::at(size_t index) & noexcept {
+  if (error()) { return error(); }
+  return first.at(index);
+}
+simdjson_really_inline error_code simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::rewind() noexcept {
+  if (error()) { return error(); }
+  first.rewind();
+  return SUCCESS;
+}
+simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::begin() & noexcept {
+  if (error()) { return error(); }
+  return first.begin();
+}
+simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::end() & noexcept {
+  return {};
+}
+simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::find_field_unordered(std::string_view key) & noexcept {
+  if (error()) { return error(); }
+  return first.find_field_unordered(key);
+}
+simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::find_field_unordered(const char *key) & noexcept {
+  if (error()) { return error(); }
+  return first.find_field_unordered(key);
+}
+simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::operator[](std::string_view key) & noexcept {
+  if (error()) { return error(); }
+  return first[key];
+}
+simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::operator[](const char *key) & noexcept {
+  if (error()) { return error(); }
+  return first[key];
+}
+simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::find_field(std::string_view key) & noexcept {
+  if (error()) { return error(); }
+  return first.find_field(key);
+}
+simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::find_field(const char *key) & noexcept {
+  if (error()) { return error(); }
+  return first.find_field(key);
+}
+simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::get_array() & noexcept {
+  if (error()) { return error(); }
+  return first.get_array();
+}
+simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::object> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::get_object() & noexcept {
+  if (error()) { return error(); }
+  return first.get_object();
+}
+simdjson_really_inline simdjson_result<uint64_t> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::get_uint64() noexcept {
+  if (error()) { return error(); }
+  return first.get_uint64();
+}
+simdjson_really_inline simdjson_result<int64_t> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::get_int64() noexcept {
+  if (error()) { return error(); }
+  return first.get_int64();
+}
+simdjson_really_inline simdjson_result<double> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::get_double() noexcept {
+  if (error()) { return error(); }
+  return first.get_double();
+}
+simdjson_really_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::get_string() noexcept {
+  if (error()) { return error(); }
+  return first.get_string();
+}
+simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::raw_json_string> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::get_raw_json_string() noexcept {
+  if (error()) { return error(); }
+  return first.get_raw_json_string();
+}
+simdjson_really_inline simdjson_result<bool> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::get_bool() noexcept {
+  if (error()) { return error(); }
+  return first.get_bool();
+}
+simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::get_value() noexcept {
+  if (error()) { return error(); }
+  return first.get_value();
+}
+simdjson_really_inline bool simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::is_null() noexcept {
+  if (error()) { return error(); }
+  return first.is_null();
+}
+simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::json_type> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::type() noexcept {
+  if (error()) { return error(); }
+  return first.type();
+}
+simdjson_really_inline simdjson_result<bool> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::is_scalar() noexcept {
+  if (error()) { return error(); }
+  return first.is_scalar();
+}
+simdjson_really_inline bool simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::is_negative() noexcept {
+  if (error()) { return error(); }
+  return first.is_negative();
+}
+simdjson_really_inline simdjson_result<bool> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::is_integer() noexcept {
+  if (error()) { return error(); }
+  return first.is_integer();
+}
+simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number_type> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::get_number_type() noexcept {
+  if (error()) { return error(); }
+  return first.get_number_type();
+}
+simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::get_number() noexcept {
+  if (error()) { return error(); }
+  return first.get_number();
+}
+#if SIMDJSON_EXCEPTIONS
+simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::operator SIMDJSON_IMPLEMENTATION::ondemand::array() & noexcept(false) {
+  if (error()) { throw simdjson_error(error()); }
+  return first;
+}
+simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::operator SIMDJSON_IMPLEMENTATION::ondemand::object() & noexcept(false) {
+  if (error()) { throw simdjson_error(error()); }
+  return first;
+}
+simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::operator uint64_t() noexcept(false) {
+  if (error()) { throw simdjson_error(error()); }
+  return first;
+}
+simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::operator int64_t() noexcept(false) {
+  if (error()) { throw simdjson_error(error()); }
+  return first;
+}
+simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::operator double() noexcept(false) {
+  if (error()) { throw simdjson_error(error()); }
+  return first;
+}
+simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::operator std::string_view() noexcept(false) {
+  if (error()) { throw simdjson_error(error()); }
+  return first;
+}
+simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::operator SIMDJSON_IMPLEMENTATION::ondemand::raw_json_string() noexcept(false) {
+  if (error()) { throw simdjson_error(error()); }
+  return first;
+}
+simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::operator bool() noexcept(false) {
+  if (error()) { throw simdjson_error(error()); }
+  return first;
+}
+simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::operator SIMDJSON_IMPLEMENTATION::ondemand::value() noexcept(false) {
+  if (error()) { throw simdjson_error(error()); }
+  return first;
+}
+#endif
+
+simdjson_really_inline simdjson_result<const char *> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::current_location() noexcept {
+  if (error()) { return error(); }
+  return first.current_location();
+}
+
+simdjson_really_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::raw_json_token() noexcept {
+  if (error()) { return error(); }
+  return first.raw_json_token();
+}
+
+simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::at_pointer(std::string_view json_pointer) noexcept {
+  if (error()) { return error(); }
+  return first.at_pointer(json_pointer);
+}
+
+
 } // namespace simdjson

include/simdjson/generic/ondemand/document.h

@@ -13,7 +13,7 @@ class array_iterator;
 class document_stream;
 
 /**
- * A JSON document iteration.
+ * A JSON document. It holds a json_iterator instance.
  *
  * Used by tokens to get text, and string buffer location.
  *
@@ -27,7 +27,7 @@ public:
    * Exists so you can declare a variable and later assign to it before use.
    */
   simdjson_really_inline document() noexcept = default;
-  simdjson_really_inline document(const document &other) noexcept = delete;
+  simdjson_really_inline document(const document &other) noexcept = delete; // pass your documents by reference, not by copy
   simdjson_really_inline document(document &&other) noexcept = default;
   simdjson_really_inline document &operator=(const document &other) noexcept = delete;
   simdjson_really_inline document &operator=(document &&other) noexcept = default;
@@ -53,6 +53,13 @@ public:
    * @returns INCORRECT_TYPE If the JSON value is not a 64-bit unsigned integer.
    */
   simdjson_really_inline simdjson_result<uint64_t> get_uint64() noexcept;
+  /**
+   * Cast this JSON value (inside string) to an unsigned integer.
+   *
+   * @returns A signed 64-bit integer.
+   * @returns INCORRECT_TYPE If the JSON value is not a 64-bit unsigned integer.
+   */
+  simdjson_really_inline simdjson_result<uint64_t> get_uint64_in_string() noexcept;
   /**
    * Cast this JSON value to a signed integer.
    *
@@ -60,6 +67,13 @@ public:
    * @returns INCORRECT_TYPE If the JSON value is not a 64-bit integer.
    */
   simdjson_really_inline simdjson_result<int64_t> get_int64() noexcept;
+  /**
+   * Cast this JSON value (inside string) to a signed integer.
+   *
+   * @returns A signed 64-bit integer.
+   * @returns INCORRECT_TYPE If the JSON value is not a 64-bit integer.
+   */
+  simdjson_really_inline simdjson_result<int64_t> get_int64_in_string() noexcept;
   /**
    * Cast this JSON value to a double.
    *
@@ -67,11 +81,21 @@ public:
    * @returns INCORRECT_TYPE If the JSON value is not a valid floating-point number.
    */
   simdjson_really_inline simdjson_result<double> get_double() noexcept;
+
+  /**
+   * Cast this JSON value (inside string) to a double.
+   *
+   * @returns A double.
+   * @returns INCORRECT_TYPE If the JSON value is not a valid floating-point number.
+   */
+  simdjson_really_inline simdjson_result<double> get_double_in_string() noexcept;
   /**
    * Cast this JSON value to a string.
    *
    * The string is guaranteed to be valid UTF-8.
    *
+   * Important: Calling get_string() twice on the same document is an error.
+   *
    * @returns An UTF-8 string. The string is stored in the parser and will be invalidated the next
    *          time it parses a document or when it is destroyed.
    * @returns INCORRECT_TYPE if the JSON value is not a string.
@@ -93,6 +117,14 @@ public:
    * @returns INCORRECT_TYPE if the JSON value is not true or false.
    */
   simdjson_really_inline simdjson_result<bool> get_bool() noexcept;
+  /**
+   * Cast this JSON value to a value when the document is an object or an array.
+   *
+   * @returns A value if a JSON array or object cannot be found.
+   * @returns SCALAR_DOCUMENT_AS_VALUE error is the document is a scalar (see is_scalar() function).
+   */
+  simdjson_really_inline simdjson_result<value> get_value() noexcept;
+
   /**
    * Checks if this JSON value is null.
    *
@@ -126,7 +158,9 @@ public:
   /**
    * Get this value as the given type.
    *
-   * Supported types: object, array, raw_json_string, string_view, uint64_t, int64_t, double, bool
+   * Supported types: object, array, raw_json_string, string_view, uint64_t, int64_t, double, bool, value
+   *
+   * Be mindful that the document instance must remain in scope while you are accessing object, array and value instances.
    *
    * @param out This is set to a value of the given type, parsed from the JSON. If there is an error, this may not be initialized.
    * @returns INCORRECT_TYPE If the JSON value is not an object.
@@ -198,6 +232,13 @@ public:
    * @exception simdjson_error(INCORRECT_TYPE) if the JSON value is not true or false.
    */
   simdjson_really_inline operator bool() noexcept(false);
+  /**
+   * Cast this JSON value to a value.
+   *
+   * @returns A value value.
+   * @exception if a JSON value cannot be found
+   */
+  simdjson_really_inline operator value() noexcept(false);
 #endif
   /**
    * This method scans the array and counts the number of elements.
@@ -211,6 +252,29 @@ public:
    * safe to continue.
    */
   simdjson_really_inline simdjson_result<size_t> count_elements() & noexcept;
+   /**
+   * This method scans the object and counts the number of key-value pairs.
+   * The count_fields method should always be called before you have begun
+   * iterating through the object: it is expected that you are pointing at
+   * the beginning of the object.
+   * The runtime complexity is linear in the size of the object. After
+   * calling this function, if successful, the object is 'rewinded' at its
+   * beginning as if it had never been accessed. If the JSON is malformed (e.g.,
+   * there is a missing comma), then an error is returned and it is no longer
+   * safe to continue.
+   *
+   * To check that an object is empty, it is more performant to use
+   * the is_empty() method.
+   */
+  simdjson_really_inline simdjson_result<size_t> count_fields() & noexcept;
+  /**
+   * Get the value at the given index in the array. This function has linear-time complexity.
+   * This function should only be called once on an array instance since the array iterator is not reset between each call.
+   *
+   * @return The value at the given index, or:
+   *         - INDEX_OUT_OF_BOUNDS if the array index is larger than an array length
+   */
+  simdjson_really_inline simdjson_result<value> at(size_t index) & noexcept;
   /**
    * Begin array iteration.
    *
@@ -241,6 +305,18 @@ public:
    * **Raw Keys:** The lookup will be done against the *raw* key, and will not unescape keys.
    * e.g. `object["a"]` will match `{ "a": 1 }`, but will *not* match `{ "\u0061": 1 }`.
    *
+   *
+   * You must consume the fields on an object one at a time. A request for a new key
+   * invalidates previous field values: it makes them unsafe. E.g., the array
+   * given by content["bids"].get_array() should not be accessed after you have called
+   * content["asks"].get_array(). You can detect such mistakes by first compiling and running
+   * the code in Debug mode (or with the macro `SIMDJSON_DEVELOPMENT_CHECKS` set to 1): an
+   * OUT_OF_ORDER_ITERATION error is generated.
+   *
+   * You are expected to access keys only once. You should access the value corresponding to
+   * a key a single time. Doing object["mykey"].to_string()and then again object["mykey"].to_string()
+   * is an error.
+   *
    * @param key The key to look up.
    * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object.
    */
@@ -264,6 +340,17 @@ public:
    * Use find_field() if you are sure fields will be in order (or are willing to treat it as if the
    * field wasn't there when they aren't).
    *
+   * You must consume the fields on an object one at a time. A request for a new key
+   * invalidates previous field values: it makes them unsafe. E.g., the array
+   * given by content["bids"].get_array() should not be accessed after you have called
+   * content["asks"].get_array(). You can detect such mistakes by first compiling and running
+   * the code in Debug mode (or with the macro `SIMDJSON_DEVELOPMENT_CHECKS` set to 1): an
+   * OUT_OF_ORDER_ITERATION error is generated.
+   *
+   * You are expected to access keys only once. You should access the value corresponding to a key
+   * a single time. Doing object["mykey"].to_string() and then again object["mykey"].to_string()
+   * is an error.
+   *
    * @param key The key to look up.
    * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object.
    */
@@ -286,6 +373,83 @@ public:
    */
   simdjson_really_inline simdjson_result<json_type> type() noexcept;
 
+  /**
+   * Checks whether the document is a scalar (string, number, null, Boolean).
+   * Returns false when there it is an array or object.
+   *
+   * @returns true if the type is string, number, null, Boolean
+   * @error TAPE_ERROR when the JSON value is a bad token like "}" "," or "alse".
+   */
+  simdjson_really_inline simdjson_result<bool> is_scalar() noexcept;
+
+  /**
+   * Checks whether the document is a negative number.
+   *
+   * @returns true if the number if negative.
+   */
+  simdjson_really_inline bool is_negative() noexcept;
+  /**
+   * Checks whether the document is an integer number. Note that
+   * this requires to partially parse the number string. If
+   * the value is determined to be an integer, it may still
+   * not parse properly as an integer in subsequent steps
+   * (e.g., it might overflow).
+   *
+   * @returns true if the number if negative.
+   */
+  simdjson_really_inline simdjson_result<bool> is_integer() noexcept;
+  /**
+   * Determine the number type (integer or floating-point number) as quickly
+   * as possible. This function does not fully validate the input. It is
+   * useful when you only need to classify the numbers, without parsing them.
+   *
+   * If you are planning to retrieve the value or you need full validation,
+   * consider using the get_number() method instead: it will fully parse
+   * and validate the input, and give you access to the type:
+   * get_number().get_number_type().
+   *
+   * get_number_type() is number_type::unsigned_integer if we have
+   * an integer greater or equal to 9223372036854775808
+   * get_number_type() is number_type::signed_integer if we have an
+   * integer that is less than 9223372036854775808
+   * Otherwise, get_number_type() has value number_type::floating_point_number
+   *
+   * This function requires processing the number string, but it is expected
+   * to be faster than get_number().get_number_type() because it is does not
+   * parse the number value.
+   *
+   * @returns the type of the number
+   */
+  simdjson_really_inline simdjson_result<number_type> get_number_type() noexcept;
+
+  /**
+   * Attempt to parse an ondemand::number. An ondemand::number may
+   * contain an integer value or a floating-point value, the simdjson
+   * library will autodetect the type. Thus it is a dynamically typed
+   * number. Before accessing the value, you must determine the detected
+   * type.
+   *
+   * number.get_number_type() is number_type::signed_integer if we have
+   * an integer in [-9223372036854775808,9223372036854775808)
+   * You can recover the value by calling number.get_int64() and you
+   * have that number.is_int64() is true.
+   *
+   * number.get_number_type() is number_type::unsigned_integer if we have
+   * an integer in [9223372036854775808,18446744073709551616)
+   * You can recover the value by calling number.get_uint64() and you
+   * have that number.is_uint64() is true.
+   *
+   * Otherwise, number.get_number_type() has value number_type::floating_point_number
+   * and we have a binary64 number.
+   * You can recover the value by calling number.get_double() and you
+   * have that number.is_double() is true.
+   *
+   * You must check the type before accessing the value: it is an error
+   * to call "get_int64()" when number.get_number_type() is not
+   * number_type::signed_integer and when number.is_int64() is false.
+   */
+  simdjson_warn_unused simdjson_really_inline simdjson_result<number> get_number() noexcept;
+
   /**
    * Get the raw JSON for this token.
    *
@@ -320,6 +484,27 @@ public:
    * Returns debugging information.
    */
   inline std::string to_debug_string() noexcept;
+  /**
+   * Some unrecoverable error conditions may render the document instance unusable.
+   * The is_alive() method returns true when the document is still suitable.
+   */
+  inline bool is_alive() noexcept;
+
+  /**
+   * Returns the current location in the document if in bounds.
+   */
+  inline simdjson_result<const char *> current_location() noexcept;
+
+  /**
+   * Returns the current depth in the document if in bounds.
+   *
+   * E.g.,
+   *  0 = finished with document
+   *  1 = document root value (could be [ or {, not yet known)
+   *  2 = , or } inside root array/object
+   *  3 = key or value inside root array/object.
+   */
+  simdjson_really_inline int32_t current_depth() const noexcept;
 
   /**
    * Get the value associated with the given JSON pointer.  We use the RFC 6901
@@ -350,11 +535,12 @@ public:
    *         - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length
    *         - INCORRECT_TYPE if a non-integer is used to access an array
    *         - INVALID_JSON_POINTER if the JSON pointer is invalid and cannot be parsed
+   *         - SCALAR_DOCUMENT_AS_VALUE if the json_pointer is empty and the document is not a scalar (see is_scalar() function).
    */
   simdjson_really_inline simdjson_result<value> at_pointer(std::string_view json_pointer) noexcept;
   /**
    * Consumes the document and returns a string_view instance corresponding to the
-   * document as represented in JSON. It points inside the original byte array containg
+   * document as represented in JSON. It points inside the original byte array containing
    * the JSON document.
    */
   simdjson_really_inline simdjson_result<std::string_view> raw_json() noexcept;
@@ -369,7 +555,6 @@ protected:
 
   simdjson_really_inline value_iterator resume_value_iterator() noexcept;
   simdjson_really_inline value_iterator get_root_value_iterator() noexcept;
-  simdjson_really_inline simdjson_result<value> get_value_unsafe() noexcept;
   simdjson_really_inline simdjson_result<object> start_or_resume_object() noexcept;
   static simdjson_really_inline document start(ondemand::json_iterator &&iter) noexcept;
 
@@ -389,6 +574,68 @@ protected:
   friend class document_stream;
 };
 
+
+/**
+ * A document_reference is a thin wrapper around a document reference instance.
+ */
+class document_reference {
+public:
+  simdjson_really_inline document_reference() noexcept;
+  simdjson_really_inline document_reference(document &d) noexcept;
+  simdjson_really_inline document_reference(const document_reference &other) noexcept = default;
+  simdjson_really_inline document_reference& operator=(const document_reference &other) noexcept = default;
+  simdjson_really_inline void rewind() noexcept;
+  simdjson_really_inline simdjson_result<array> get_array() & noexcept;
+  simdjson_really_inline simdjson_result<object> get_object() & noexcept;
+  simdjson_really_inline simdjson_result<uint64_t> get_uint64() noexcept;
+  simdjson_really_inline simdjson_result<int64_t> get_int64() noexcept;
+  simdjson_really_inline simdjson_result<double> get_double() noexcept;
+  simdjson_really_inline simdjson_result<std::string_view> get_string() noexcept;
+  simdjson_really_inline simdjson_result<raw_json_string> get_raw_json_string() noexcept;
+  simdjson_really_inline simdjson_result<bool> get_bool() noexcept;
+  simdjson_really_inline simdjson_result<value> get_value() noexcept;
+
+  simdjson_really_inline bool is_null() noexcept;
+  simdjson_really_inline simdjson_result<std::string_view> raw_json() noexcept;
+  simdjson_really_inline operator document&() const noexcept;
+
+#if SIMDJSON_EXCEPTIONS
+  simdjson_really_inline operator array() & noexcept(false);
+  simdjson_really_inline operator object() & noexcept(false);
+  simdjson_really_inline operator uint64_t() noexcept(false);
+  simdjson_really_inline operator int64_t() noexcept(false);
+  simdjson_really_inline operator double() noexcept(false);
+  simdjson_really_inline operator std::string_view() noexcept(false);
+  simdjson_really_inline operator raw_json_string() noexcept(false);
+  simdjson_really_inline operator bool() noexcept(false);
+  simdjson_really_inline operator value() noexcept(false);
+#endif
+  simdjson_really_inline simdjson_result<size_t> count_elements() & noexcept;
+  simdjson_really_inline simdjson_result<size_t> count_fields() & noexcept;
+  simdjson_really_inline simdjson_result<value> at(size_t index) & noexcept;
+  simdjson_really_inline simdjson_result<array_iterator> begin() & noexcept;
+  simdjson_really_inline simdjson_result<array_iterator> end() & noexcept;
+  simdjson_really_inline simdjson_result<value> find_field(std::string_view key) & noexcept;
+  simdjson_really_inline simdjson_result<value> find_field(const char *key) & noexcept;
+  simdjson_really_inline simdjson_result<value> operator[](std::string_view key) & noexcept;
+  simdjson_really_inline simdjson_result<value> operator[](const char *key) & noexcept;
+  simdjson_really_inline simdjson_result<value> find_field_unordered(std::string_view key) & noexcept;
+  simdjson_really_inline simdjson_result<value> find_field_unordered(const char *key) & noexcept;
+
+  simdjson_really_inline simdjson_result<json_type> type() noexcept;
+  simdjson_really_inline simdjson_result<bool> is_scalar() noexcept;
+
+  simdjson_really_inline simdjson_result<const char *> current_location() noexcept;
+  simdjson_really_inline int32_t current_depth() const noexcept;
+  simdjson_really_inline bool is_negative() noexcept;
+  simdjson_really_inline simdjson_result<bool> is_integer() noexcept;
+  simdjson_really_inline simdjson_result<number_type> get_number_type() noexcept;
+  simdjson_really_inline simdjson_result<number> get_number() noexcept;
+  simdjson_really_inline simdjson_result<std::string_view> raw_json_token() noexcept;
+  simdjson_really_inline simdjson_result<value> at_pointer(std::string_view json_pointer) noexcept;
+private:
+  document *doc{nullptr};
+};
 } // namespace ondemand
 } // namespace SIMDJSON_IMPLEMENTATION
 } // namespace simdjson
@@ -408,9 +655,11 @@ public:
   simdjson_really_inline simdjson_result<uint64_t> get_uint64() noexcept;
   simdjson_really_inline simdjson_result<int64_t> get_int64() noexcept;
   simdjson_really_inline simdjson_result<double> get_double() noexcept;
+  simdjson_really_inline simdjson_result<double> get_double_from_string() noexcept;
   simdjson_really_inline simdjson_result<std::string_view> get_string() noexcept;
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::raw_json_string> get_raw_json_string() noexcept;
   simdjson_really_inline simdjson_result<bool> get_bool() noexcept;
+  simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> get_value() noexcept;
   simdjson_really_inline bool is_null() noexcept;
 
   template<typename T> simdjson_really_inline simdjson_result<T> get() & noexcept;
@@ -428,8 +677,11 @@ public:
   simdjson_really_inline operator std::string_view() noexcept(false);
   simdjson_really_inline operator SIMDJSON_IMPLEMENTATION::ondemand::raw_json_string() noexcept(false);
   simdjson_really_inline operator bool() noexcept(false);
+  simdjson_really_inline operator SIMDJSON_IMPLEMENTATION::ondemand::value() noexcept(false);
 #endif
   simdjson_really_inline simdjson_result<size_t> count_elements() & noexcept;
+  simdjson_really_inline simdjson_result<size_t> count_fields() & noexcept;
+  simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> at(size_t index) & noexcept;
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> begin() & noexcept;
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> end() & noexcept;
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field(std::string_view key) & noexcept;
@@ -438,13 +690,80 @@ public:
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> operator[](const char *key) & noexcept;
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field_unordered(std::string_view key) & noexcept;
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field_unordered(const char *key) & noexcept;
-
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::json_type> type() noexcept;
-
+  simdjson_really_inline simdjson_result<bool> is_scalar() noexcept;
+  simdjson_really_inline simdjson_result<const char *> current_location() noexcept;
+  simdjson_really_inline int32_t current_depth() const noexcept;
+  simdjson_really_inline bool is_negative() noexcept;
+  simdjson_really_inline simdjson_result<bool> is_integer() noexcept;
+  simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number_type> get_number_type() noexcept;
+  simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number> get_number() noexcept;
   /** @copydoc simdjson_really_inline std::string_view document::raw_json_token() const noexcept */
   simdjson_really_inline simdjson_result<std::string_view> raw_json_token() noexcept;
 
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> at_pointer(std::string_view json_pointer) noexcept;
 };
 
+
+} // namespace simdjson
+
+
+
+namespace simdjson {
+
+template<>
+struct simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference> : public SIMDJSON_IMPLEMENTATION::implementation_simdjson_result_base<SIMDJSON_IMPLEMENTATION::ondemand::document_reference> {
+public:
+  simdjson_really_inline simdjson_result(SIMDJSON_IMPLEMENTATION::ondemand::document_reference value, error_code error) noexcept;
+  simdjson_really_inline simdjson_result() noexcept = default;
+  simdjson_really_inline error_code rewind() noexcept;
+
+  simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array> get_array() & noexcept;
+  simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::object> get_object() & noexcept;
+  simdjson_really_inline simdjson_result<uint64_t> get_uint64() noexcept;
+  simdjson_really_inline simdjson_result<int64_t> get_int64() noexcept;
+  simdjson_really_inline simdjson_result<double> get_double() noexcept;
+  simdjson_really_inline simdjson_result<std::string_view> get_string() noexcept;
+  simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::raw_json_string> get_raw_json_string() noexcept;
+  simdjson_really_inline simdjson_result<bool> get_bool() noexcept;
+  simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> get_value() noexcept;
+  simdjson_really_inline bool is_null() noexcept;
+
+#if SIMDJSON_EXCEPTIONS
+  simdjson_really_inline operator SIMDJSON_IMPLEMENTATION::ondemand::array() & noexcept(false);
+  simdjson_really_inline operator SIMDJSON_IMPLEMENTATION::ondemand::object() & noexcept(false);
+  simdjson_really_inline operator uint64_t() noexcept(false);
+  simdjson_really_inline operator int64_t() noexcept(false);
+  simdjson_really_inline operator double() noexcept(false);
+  simdjson_really_inline operator std::string_view() noexcept(false);
+  simdjson_really_inline operator SIMDJSON_IMPLEMENTATION::ondemand::raw_json_string() noexcept(false);
+  simdjson_really_inline operator bool() noexcept(false);
+  simdjson_really_inline operator SIMDJSON_IMPLEMENTATION::ondemand::value() noexcept(false);
+#endif
+  simdjson_really_inline simdjson_result<size_t> count_elements() & noexcept;
+  simdjson_really_inline simdjson_result<size_t> count_fields() & noexcept;
+  simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> at(size_t index) & noexcept;
+  simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> begin() & noexcept;
+  simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> end() & noexcept;
+  simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field(std::string_view key) & noexcept;
+  simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field(const char *key) & noexcept;
+  simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> operator[](std::string_view key) & noexcept;
+  simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> operator[](const char *key) & noexcept;
+  simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field_unordered(std::string_view key) & noexcept;
+  simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field_unordered(const char *key) & noexcept;
+  simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::json_type> type() noexcept;
+  simdjson_really_inline simdjson_result<bool> is_scalar() noexcept;
+  simdjson_really_inline simdjson_result<const char *> current_location() noexcept;
+  simdjson_really_inline int32_t current_depth() const noexcept;
+  simdjson_really_inline bool is_negative() noexcept;
+  simdjson_really_inline simdjson_result<bool> is_integer() noexcept;
+  simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number_type> get_number_type() noexcept;
+  simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number> get_number() noexcept;
+  /** @copydoc simdjson_really_inline std::string_view document_reference::raw_json_token() const noexcept */
+  simdjson_really_inline simdjson_result<std::string_view> raw_json_token() noexcept;
+
+  simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> at_pointer(std::string_view json_pointer) noexcept;
+};
+
+
 } // namespace simdjson

include/simdjson/generic/ondemand/document_stream-inl.h

@@ -126,6 +126,7 @@ inline size_t document_stream::size_in_bytes() const noexcept {
 }
 
 inline size_t document_stream::truncated_bytes() const noexcept {
+  if(error == CAPACITY) { return len - batch_start; }
   return parser->implementation->structural_indexes[parser->implementation->n_structural_indexes] - parser->implementation->structural_indexes[parser->implementation->n_structural_indexes + 1];
 }
 
@@ -137,8 +138,9 @@ simdjson_really_inline document_stream::iterator::iterator(document_stream* _str
   : stream{_stream}, finished{is_end} {
 }
 
-simdjson_really_inline ondemand::document& document_stream::iterator::operator*() noexcept {
+simdjson_really_inline simdjson_result<ondemand::document_reference> document_stream::iterator::operator*() noexcept {
-  return stream->doc;
+  //if(stream->error) { return stream->error; }
+  return simdjson_result<ondemand::document_reference>(stream->doc, stream->error);
 }
 
 simdjson_really_inline document_stream::iterator& document_stream::iterator::operator++() noexcept {

include/simdjson/generic/ondemand/document_stream.h

@@ -130,7 +130,7 @@ public:
     /**
      * Get the current document (or error).
      */
-    simdjson_really_inline ondemand::document& operator*() noexcept;
+    simdjson_really_inline simdjson_result<ondemand::document_reference> operator*() noexcept;
     /**
      * Advance to the next document (prefix).
      */