New version.

* We change slightly on development checks are enabled.

* Removing garbagy code.

.drone.yml

@@ -1,46 +1,4 @@
 kind: pipeline
-name: i386-gcc # we do not support 32-bit systems, but we run tests
-platform: { os: linux, arch: amd64 }
-steps:
-- name: Build and Test
-  image: i386/ubuntu
-  environment:
-    CC: gcc
-    CXX: g++
-    BUILD_FLAGS: -- -j
-    CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF
-    CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly 
-  commands:
-    - scripts/addcmakeppa.sh "$(env -i sh -c '. /etc/os-release; echo $VERSION_CODENAME')"
-    - apt-get install -y g++ cmake gcc git
-    - mkdir build
-    - cd build
-    - cmake $CMAKE_FLAGS ..
-    - cmake --build . $BUILD_FLAGS
-    - ctest $CTEST_FLAGS 
----
-kind: pipeline
-name: i386-clang # we do not support 32-bit systems, but we run tests
-platform: { os: linux, arch: amd64 }
-steps:
-- name: Build and Test
-  image: i386/ubuntu
-  environment:
-    CC: clang-6.0
-    CXX: clang++-6.0
-    BUILD_FLAGS: -- -j
-    CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF
-    CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly 
-  commands:
-    - scripts/addcmakeppa.sh "$(env -i sh -c '. /etc/os-release; echo $VERSION_CODENAME')"
-    - apt-get install -y clang++-6.0 cmake git
-    - mkdir build
-    - cd build
-    - cmake $CMAKE_FLAGS ..
-    - cmake --build . $BUILD_FLAGS
-    - ctest $CTEST_FLAGS 
----
-kind: pipeline
 name: gcc9
 platform: { os: linux, arch: amd64 }
 steps:
@@ -50,7 +8,7 @@ steps:
     CC: gcc
     CXX: g++
     BUILD_FLAGS: -- -j
-    CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback
+    CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=icelake;haswell;westmere;fallback
     CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
   commands:
     - echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list
@@ -61,6 +19,7 @@ steps:
     - cmake $CMAKE_FLAGS ..
     - cmake --build . $BUILD_FLAGS
     - ctest $CTEST_FLAGS -L acceptance -LE per_implementation
+    - SIMDJSON_FORCE_IMPLEMENTATION=icelake ctest $CTEST_FLAGS -L per_implementation
     - SIMDJSON_FORCE_IMPLEMENTATION=haswell ctest $CTEST_FLAGS -L per_implementation
     - SIMDJSON_FORCE_IMPLEMENTATION=westmere ctest $CTEST_FLAGS -L per_implementation
     - SIMDJSON_FORCE_IMPLEMENTATION=fallback ctest $CTEST_FLAGS -L per_implementation
@@ -77,7 +36,7 @@ steps:
     CC: clang-6.0
     CXX: clang++-6.0
     BUILD_FLAGS: -- -j
-    CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback
+    CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=icelake;haswell;westmere;fallback
     CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
   commands:
     - mkdir build
@@ -85,6 +44,7 @@ steps:
     - cmake $CMAKE_FLAGS ..
     - cmake --build . $BUILD_FLAGS
     - ctest $CTEST_FLAGS -L acceptance -LE per_implementation
+    - SIMDJSON_FORCE_IMPLEMENTATION=icelake ctest $CTEST_FLAGS -L per_implementation
     - SIMDJSON_FORCE_IMPLEMENTATION=haswell ctest $CTEST_FLAGS -L per_implementation
     - SIMDJSON_FORCE_IMPLEMENTATION=westmere ctest $CTEST_FLAGS -L per_implementation
     - SIMDJSON_FORCE_IMPLEMENTATION=fallback ctest $CTEST_FLAGS -L per_implementation
@@ -142,7 +102,7 @@ steps:
     CC: gcc
     CXX: g++
     BUILD_FLAGS: -- -j
-    CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback
+    CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=icelake;haswell;westmere;fallback
     CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
   commands:
     - echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list
@@ -153,6 +113,7 @@ steps:
     - cmake $CMAKE_FLAGS ..
     - cmake --build . $BUILD_FLAGS
     - ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L acceptance -LE per_implementation
+    - SIMDJSON_FORCE_IMPLEMENTATION=icelake ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation
     - SIMDJSON_FORCE_IMPLEMENTATION=haswell ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation
     - SIMDJSON_FORCE_IMPLEMENTATION=westmere ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation
     - SIMDJSON_FORCE_IMPLEMENTATION=fallback ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation
@@ -168,7 +129,7 @@ steps:
   environment:
     CC: clang-9
     CXX: clang++-9
-    CMAKE_FLAGS: -DSIMDJSON_SANITIZE=ON -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback
+    CMAKE_FLAGS: -DSIMDJSON_SANITIZE=ON -DSIMDJSON_IMPLEMENTATION=icelake;haswell;westmere;fallback
     BUILD_FLAGS: -- -j
     CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
   commands:
@@ -177,6 +138,7 @@ steps:
     - cmake $CMAKE_FLAGS ..
     - cmake --build . $BUILD_FLAGS
     - ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L acceptance -LE per_implementation
+    - SIMDJSON_FORCE_IMPLEMENTATION=icelake ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation
     - SIMDJSON_FORCE_IMPLEMENTATION=haswell ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation
     - SIMDJSON_FORCE_IMPLEMENTATION=westmere ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation
     - SIMDJSON_FORCE_IMPLEMENTATION=fallback ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation

.github/workflows/alpine.yml

@@ -9,8 +9,8 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/cache@v2
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
         with:
           path: dependencies/.cache
           key: ${{ hashFiles('dependencies/CMakeLists.txt') }}

.github/workflows/cifuzz.yml

@@ -0,0 +1,24 @@
+name: CIFuzz
+on: [pull_request]
+jobs:
+  Fuzzing:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Build Fuzzers
+      id: build
+      uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master
+      with:
+        oss-fuzz-project-name: 'simdjson'
+        dry-run: false
+    - name: Run Fuzzers
+      uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master
+      with:
+        oss-fuzz-project-name: 'simdjson'
+        fuzz-seconds: 600
+        dry-run: false
+    - name: Upload Crash
+      uses: actions/upload-artifact@v1
+      if: failure() && steps.build.outcome == 'success'
+      with:
+        name: artifacts
+        path: ./out/artifacts

.github/workflows/fix-trailing-whitespace.yml

@@ -6,7 +6,7 @@ jobs:
   whitespace:
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Remove whitespace and check the diff
         run: |
           set -eu

.github/workflows/fuzzers.yml

@@ -37,14 +37,14 @@ jobs:
         chmod +x llvm.sh
         sudo ./llvm.sh $CLANGVERSION
 
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
 
-    - uses: actions/cache@v2
+    - uses: actions/cache@v3
       with:
         path: dependencies/.cache
         key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
 
-    - uses: actions/cache@v2
+    - uses: actions/cache@v3
       id: cache-corpus
       with:
         path: out/

.github/workflows/macos-11.yml

@@ -9,14 +9,14 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: macos-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/cache@v2
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
         with:
           path: dependencies/.cache
           key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
       - name: Use cmake
         run: |
-          xcversion select 11.2.1
+          xcversion select 11.7
           mkdir builddebug &&
           cd builddebug &&
           cmake -DCMAKE_BUILD_TYPE=Debug -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF ..  &&

.github/workflows/macos.yml

@@ -9,8 +9,8 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: macos-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/cache@v2
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
         with:
           path: dependencies/.cache
           key: ${{ hashFiles('dependencies/CMakeLists.txt') }}

.github/workflows/mingw-ci.yml

@@ -1,63 +0,0 @@
-name: MinGW32-CI
-
-on: [push, pull_request]
-
-
-# Important: scoop will either install 32-bit GCC or 64-bit GCC, not both.
-
-# It is important to build static libraries because cmake is not smart enough under Windows/mingw to take care of the path. So
-# with a dynamic library, you could get failures due to the fact that the EXE can't find its DLL.
-
-jobs:
-  ci:
-    if: >-
-      ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
-      ! contains(toJSON(github.event.commits.*.message), '[skip github]')
-    name: windows-gcc
-    runs-on: windows-2016
-
-    env:
-      CMAKE_GENERATOR: Ninja # This is critical, try ' cmake  -GNinja-DBUILD_SHARED_LIBS=OFF .. ' if using the command line
-      CC: gcc
-      CXX: g++
-
-    steps: # To reproduce what is below, start a powershell with administrative rights, using scoop *is* a good idea
-      - uses: actions/checkout@v2
-
-      - uses: actions/cache@v2 # we cache the scoop setup with 32-bit GCC
-        id: cache
-        with:
-          path: |
-            C:\ProgramData\scoop
-          key: scoop32 # static key: should be good forever
-
-      - uses: actions/cache@v2
-        with:
-          path: dependencies/.cache
-          key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
-
-      - name: Setup Windows # This should almost never run if the cache works.
-        if: steps.cache.outputs.cache-hit != 'true'
-        shell: powershell
-        run: |
-          Invoke-Expression (New-Object System.Net.WebClient).DownloadString('https://get.scoop.sh')
-          scoop install sudo --global
-          sudo scoop install git --global
-          sudo scoop install ninja --global
-          sudo scoop install cmake --global
-          sudo scoop install gcc --arch 32bit --global
-          $env:path
-          Write-Host 'Everything has been installed, you are good!'
-      - name: Build and Test 32-bit x86
-        shell: powershell
-        run: |
-          $ENV:PATH="C:\ProgramData\scoop\shims;C:\ProgramData\scoop\apps\gcc\current\bin;C:\ProgramData\scoop\apps\ninja\current;$ENV:PATH"
-          g++ --version
-          cmake --version
-          ninja --version
-          git --version
-          mkdir build32
-          cd build32
-          cmake -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_ENABLE_THREADS=OFF ..
-          cmake --build . --target acceptance_tests --verbose
-          ctest -L acceptance --output-on-failure

.github/workflows/mingw64-ci.yml

@@ -1,69 +0,0 @@
-name: MinGW64-CI
-
-on: [push, pull_request]
-
-
-# Important: scoop will either install 32-bit GCC or 64-bit GCC, not both.
-
-# It is important to build static libraries because cmake is not smart enough under Windows/mingw to take care of the path. So
-# with a dynamic library, you could get failures due to the fact that the EXE can't find its DLL.
-
-jobs:
-  ci:
-    if: >-
-      ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
-      ! contains(toJSON(github.event.commits.*.message), '[skip github]')
-    name: windows-gcc
-    runs-on: windows-2016
-
-    env:
-      CMAKE_GENERATOR: Ninja # This is critical, try ' cmake  -GNinja-DBUILD_SHARED_LIBS=OFF .. ' if using the command line
-      CC: gcc
-      CXX: g++
-
-    steps: # To reproduce what is below, start a powershell with administrative rights, using scoop *is* a good idea
-      - uses: actions/checkout@v2
-
-      - uses: actions/cache@v2 # we cache the scoop setup with 64-bit GCC
-        id: cache
-        with:
-          path: |
-            C:\ProgramData\scoop
-          key: scoop64 # static key: should be good forever
-
-      - uses: actions/cache@v2
-        with:
-          path: dependencies/.cache
-          key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
-
-      - name: Setup Windows # This should almost never run if the cache works.
-        if: steps.cache.outputs.cache-hit != 'true'
-        shell: powershell
-        run: |
-          Invoke-Expression (New-Object System.Net.WebClient).DownloadString('https://get.scoop.sh')
-          scoop install sudo --global
-          sudo scoop install git --global
-          sudo scoop install ninja --global
-          sudo scoop install cmake --global
-          sudo scoop install gcc --arch 64bit --global
-          $env:path
-          Write-Host 'Everything has been installed, you are good!'
-      - name: Build and Test 64-bit x64
-        shell: powershell
-        run: |
-          $ENV:PATH="C:\ProgramData\scoop\shims;C:\ProgramData\scoop\apps\gcc\current\bin;C:\ProgramData\scoop\apps\ninja\current;$ENV:PATH"
-          g++ --version
-          cmake --version
-          ninja --version
-          git --version
-          mkdir build64
-          cd build64
-          cmake -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_ENABLE_THREADS=OFF ..
-          cmake --build . --target acceptance_tests --verbose
-          ctest -L acceptance --output-on-failure
-          cd ..
-          mkdir build64debug
-          cd build64debug
-          cmake -DSIMDJSON_DEVELOPER_MODE=ON -DCMAKE_BUILD_TYPE=Debug -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_ENABLE_THREADS=OFF ..
-          cmake --build . --target acceptance_tests --verbose
-          ctest -L acceptance --output-on-failure

.github/workflows/msys2-clang.yml

@@ -17,21 +17,15 @@ jobs:
           - msystem: "MINGW64"
             install: mingw-w64-x86_64-libxml2 mingw-w64-x86_64-cmake mingw-w64-x86_64-ninja mingw-w64-x86_64-clang
             type: Release
-          - msystem: "MINGW32"
-            install: mingw-w64-i686-libxml2 mingw-w64-i686-cmake mingw-w64-i686-ninja mingw-w64-i686-clang
-            type: Release
           - msystem: "MINGW64"
             install: mingw-w64-x86_64-libxml2 mingw-w64-x86_64-cmake mingw-w64-x86_64-ninja mingw-w64-x86_64-clang
             type: Debug
-          - msystem: "MINGW32"
-            install: mingw-w64-i686-libxml2 mingw-w64-i686-cmake mingw-w64-i686-ninja mingw-w64-i686-clang
-            type: Debug
     env:
       CMAKE_GENERATOR: Ninja
 
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/cache@v2
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
         with:
           path: dependencies/.cache
           key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
@@ -44,6 +38,6 @@ jobs:
         run: |
           mkdir build
           cd build
-          cmake -DSIMDJSON_DEVELOPER_MODE=ON -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_BUILD_TYPE=${{ matrix.type }} -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_DO_NOT_USE_THREADS_NO_MATTER_WHAT=ON ..
+          cmake -DSIMDJSON_DEVELOPER_MODE=ON -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_BUILD_TYPE=${{ matrix.type }} -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_DO_NOT_USE_THREADS_NO_MATTER_WHAT=ON ..
           cmake --build . --verbose
           ctest -j4 --output-on-failure -LE explicitonly

.github/workflows/msys2.yml

@@ -19,21 +19,15 @@ jobs:
           - msystem: "MINGW64"
             install: mingw-w64-x86_64-cmake mingw-w64-x86_64-ninja mingw-w64-x86_64-gcc
             type: Release
-          - msystem: "MINGW32"
-            install: mingw-w64-i686-cmake mingw-w64-i686-ninja mingw-w64-i686-gcc
-            type: Release
           - msystem: "MINGW64"
             install: mingw-w64-x86_64-cmake mingw-w64-x86_64-ninja mingw-w64-x86_64-gcc
             type: Debug
-          - msystem: "MINGW32"
-            install: mingw-w64-i686-cmake mingw-w64-i686-ninja mingw-w64-i686-gcc
-            type: Debug
     env:
       CMAKE_GENERATOR: Ninja
 
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/cache@v2
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
         with:
           path: dependencies/.cache
           key: ${{ hashFiles('dependencies/CMakeLists.txt') }}

.github/workflows/ubuntu18-checkperf.yml

@@ -15,8 +15,8 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: ubuntu-18.04
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/cache@v2
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
         with:
           path: dependencies/.cache
           key: ${{ hashFiles('dependencies/CMakeLists.txt') }}

.github/workflows/ubuntu18-oldclang.yml

@@ -13,8 +13,8 @@ jobs:
       CC: clang-7
       CXX: clang++-7
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/cache@v2
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
         with:
           path: dependencies/.cache
           key: ${{ hashFiles('dependencies/CMakeLists.txt') }}

.github/workflows/ubuntu18-threadsani.yml

@@ -10,8 +10,8 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: ubuntu-18.04
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/cache@v2
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
         with:
           path: dependencies/.cache
           key: ${{ hashFiles('dependencies/CMakeLists.txt') }}

.github/workflows/ubuntu18.yml

@@ -9,8 +9,8 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: ubuntu-18.04
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/cache@v2
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
         with:
           path: dependencies/.cache
           key: ${{ hashFiles('dependencies/CMakeLists.txt') }}

.github/workflows/ubuntu20-checkperf.yml

@@ -15,8 +15,8 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/cache@v2
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
         with:
           path: dependencies/.cache
           key: ${{ hashFiles('dependencies/CMakeLists.txt') }}

.github/workflows/ubuntu20-gcc8.yml

@@ -0,0 +1,38 @@
+name: Ubuntu 20.04 CI (GCC 8)
+
+on: [push, pull_request]
+
+jobs:
+  ubuntu-build:
+    if: >-
+      ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
+      ! contains(toJSON(github.event.commits.*.message), '[skip github]')
+    runs-on: ubuntu-20.04
+    env:
+      CXX: g++-8
+      CC: gcc-8
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
+        with:
+          path: dependencies/.cache
+          key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
+      - name: Install GCC 8
+        run: sudo apt-get install -y g++-8
+      - name: Use cmake
+        run: |
+          mkdir builddebug &&
+          cd builddebug &&
+          cmake -DCMAKE_BUILD_TYPE=Debug -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF ..  &&
+          cmake --build .   &&
+          ctest -j --output-on-failure -LE explicitonly   &&
+          cd .. &&
+          mkdir build &&
+          cd build &&
+          cmake  -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination ..  &&
+          cmake --build .   &&
+          ctest -j --output-on-failure -LE explicitonly   &&
+          cmake --install . &&
+          echo -e '#include <simdjson.h>\nint main(int argc,char**argv) {simdjson::dom::parser parser;simdjson::dom::element tweets = parser.load(argv[1]); }' > tmp.cpp && c++ -Idestination/include -Ldestination/lib -std=c++17 -Wl,-rpath,destination/lib -o linkandrun tmp.cpp -lsimdjson && ./linkandrun jsonexamples/twitter.json &&
+          cd ../tests/installation_tests/find &&
+          mkdir build && cd build && cmake -DCMAKE_INSTALL_PREFIX:PATH=../../../build/destination .. &&  cmake --build .

.github/workflows/ubuntu20-noexcept.yml

@@ -9,8 +9,8 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/cache@v2
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
         with:
           path: dependencies/.cache
           key: ${{ hashFiles('dependencies/CMakeLists.txt') }}

.github/workflows/ubuntu20-nothread.yml

@@ -9,8 +9,8 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/cache@v2
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
         with:
           path: dependencies/.cache
           key: ${{ hashFiles('dependencies/CMakeLists.txt') }}

.github/workflows/ubuntu20-sani.yml

@@ -9,8 +9,8 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/cache@v2
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
         with:
           path: dependencies/.cache
           key: ${{ hashFiles('dependencies/CMakeLists.txt') }}

.github/workflows/ubuntu20-threadsani.yml

@@ -9,8 +9,8 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/cache@v2
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
         with:
           path: dependencies/.cache
           key: ${{ hashFiles('dependencies/CMakeLists.txt') }}

.github/workflows/ubuntu20.yml

@@ -9,8 +9,8 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/cache@v2
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
         with:
           path: dependencies/.cache
           key: ${{ hashFiles('dependencies/CMakeLists.txt') }}

.github/workflows/ubuntu22-clang13.yml

@@ -0,0 +1,25 @@
+name: Ubuntu 22.04 CI (CLANG 13)
+
+on: [push, pull_request]
+
+jobs:
+  ubuntu-build:
+    if: >-
+      ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
+      ! contains(toJSON(github.event.commits.*.message), '[skip github]')
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
+        with:
+          path: dependencies/.cache
+          key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
+      - name: Install clang++-13
+        run: sudo apt-get install -y clang++-13
+      - name: Use cmake
+        run: |
+          mkdir build &&
+          cd build &&
+          CXX=clang++-13 cmake -DSIMDJSON_DEVELOPER_MODE=ON ..  &&
+          cmake --build .   &&
+          ctest -j --output-on-failure -LE explicitonly

.github/workflows/ubuntu22-gcc12.yml

@@ -0,0 +1,25 @@
+name: Ubuntu 22.04 CI (GCC 12)
+
+on: [push, pull_request]
+
+jobs:
+  ubuntu-build:
+    if: >-
+      ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
+      ! contains(toJSON(github.event.commits.*.message), '[skip github]')
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
+        with:
+          path: dependencies/.cache
+          key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
+      - name: Install gcc12
+        run: sudo apt-get install -y g++-12
+      - name: Use cmake
+        run: |
+          mkdir build &&
+          cd build &&
+          CXX=g++-12 cmake -DSIMDJSON_DEVELOPER_MODE=ON ..  &&
+          cmake --build .   &&
+          ctest -j --output-on-failure -LE explicitonly

.github/workflows/ubuntu22.yml

@@ -0,0 +1,33 @@
+name: Ubuntu 22.04 CI (GCC 11)
+
+on: [push, pull_request]
+
+jobs:
+  ubuntu-build:
+    if: >-
+      ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
+      ! contains(toJSON(github.event.commits.*.message), '[skip github]')
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
+        with:
+          path: dependencies/.cache
+          key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
+      - name: Use cmake
+        run: |
+          mkdir builddebug &&
+          cd builddebug &&
+          cmake -DCMAKE_BUILD_TYPE=Debug -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF ..  &&
+          cmake --build .   &&
+          ctest -j --output-on-failure -LE explicitonly   &&
+          cd .. &&
+          mkdir build &&
+          cd build &&
+          cmake  -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination ..  &&
+          cmake --build .   &&
+          ctest -j --output-on-failure -LE explicitonly   &&
+          cmake --install . &&
+          echo -e '#include <simdjson.h>\nint main(int argc,char**argv) {simdjson::dom::parser parser;simdjson::dom::element tweets = parser.load(argv[1]); }' > tmp.cpp && c++ -Idestination/include -Ldestination/lib -std=c++17 -Wl,-rpath,destination/lib -o linkandrun tmp.cpp -lsimdjson && ./linkandrun jsonexamples/twitter.json &&
+          cd ../tests/installation_tests/find &&
+          mkdir build && cd build && cmake -DCMAKE_INSTALL_PREFIX:PATH=../../../build/destination .. &&  cmake --build .

.github/workflows/vs16-clang-ci.yml

@@ -1,31 +0,0 @@
-name: VS16-CLANG-CI
-
-on: [push, pull_request]
-
-jobs:
-  ci:
-    if: >-
-      ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
-      ! contains(toJSON(github.event.commits.*.message), '[skip github]')
-    name: windows-vs16
-    runs-on: windows-latest
-    steps:
-    - uses: actions/checkout@v2
-    - uses: actions/cache@v2
-      with:
-        path: dependencies/.cache
-        key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
-    - name: 'Run CMake with VS16 Clang'
-      uses: lukka/run-cmake@v3
-      with:
-        cmakeListsOrSettingsJson: CMakeListsTxtAdvanced
-        cmakeListsTxtPath: '${{ github.workspace }}/CMakeLists.txt'
-        buildDirectory: "${{ github.workspace }}/../../_temp/windows"
-        cmakeBuildType: Release
-        buildWithCMake: true
-        cmakeAppendedArgs:  -T ClangCL -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF
-        buildWithCMakeArgs: --config Release
-
-    - name: 'Run CTest'
-      run: ctest -C Release -LE explicitonly  --output-on-failure
-      working-directory: "${{ github.workspace }}/../../_temp/windows"

.github/workflows/vs16-ninja-ci.yml

@@ -1,45 +0,0 @@
-name: VS16-Ninja-CI
-
-on: [push, pull_request]
-
-jobs:
-  ci:
-    if: >-
-      ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
-      ! contains(toJSON(github.event.commits.*.message), '[skip github]')
-    name: windows-vs16
-    runs-on: windows-latest
-    steps:
-    - uses: actions/checkout@v2
-    - uses: actions/cache@v2
-      with:
-        path: dependencies/.cache
-        key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
-    - name: 'Run CMake with VS16'
-      uses: lukka/run-cmake@v2
-      with:
-        cmakeListsOrSettingsJson: CMakeListsTxtAdvanced
-        cmakeListsTxtPath: '${{ github.workspace }}/CMakeLists.txt'
-        buildDirectory: "${{ github.workspace }}/../../_temp/windows"
-        cmakeBuildType: Release
-        buildWithCMake: true
-        cmakeAppendedArgs:  -G Ninja -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF
-        buildWithCMakeArgs: --config Release
-    - name: 'Run CTest'
-      run: ctest -C Release  -LE explicitonly  --output-on-failure
-      working-directory: "${{ github.workspace }}/../../_temp/windows"
-    - name: 'Install with CMake'
-      uses: lukka/run-cmake@v3
-      with:
-        cmakeListsTxtPath: '${{ github.workspace }}/CMakeLists.txt'
-        buildWithCMakeArgs: '--target install'
-    - name: 'Test Installation with CMake'
-      uses: lukka/run-cmake@v3
-      with:
-        cmakeListsOrSettingsJson: CMakeListsTxtAdvanced
-        cmakeListsTxtPath: '${{ github.workspace }}/tests/installation_tests/find/CMakeLists.txt'
-        cmakeBuildType: Release
-        buildWithCMake: true
-        buildDirectory: '${{ github.workspace }}/tests/installation_tests/find/buildDirectory'
-        cmakeAppendedArgs:  -G Ninja
-        buildWithCMakeArgs: '--config Release --verbose'

.github/workflows/vs17-arm-ci.yml

@@ -1,10 +1,10 @@
-name: VS16-ARM-CI
+name: VS17-ARM-CI
 
 on: [push, pull_request]
 
 jobs:
    ci:
-     name: windows-vs16
+     name: windows-vs17
      runs-on: windows-latest
      strategy:
        fail-fast: false
@@ -14,7 +14,7 @@ jobs:
            - {arch: ARM64}
      steps:
        - name: checkout
-         uses: actions/checkout@v2
+         uses: actions/checkout@v3
        - name: Use cmake
          run: |
            cmake -A ${{ matrix.arch }} -DCMAKE_CROSSCOMPILING=1 -DSIMDJSON_DEVELOPER_MODE=ON -D SIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_EXCEPTIONS=OFF -B build  &&

.github/workflows/vs17-ci.yml

@@ -1,4 +1,4 @@
-name: VS16-CI
+name: VS17-CI
 
 on: [push, pull_request]
 
@@ -7,22 +7,22 @@ jobs:
     if: >-
       ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
-    name: windows-vs16
+    name: windows-vs17
     runs-on: windows-latest
     strategy:
       fail-fast: false
       matrix:
         include:
-          - {gen: Visual Studio 16 2019, arch: Win32, static: ON}
-          - {gen: Visual Studio 16 2019, arch: Win32, static: OFF}
-          - {gen: Visual Studio 16 2019, arch: x64, static: ON}
-          - {gen: Visual Studio 16 2019, arch: x64, static: OFF}
+          - {gen: Visual Studio 17 2022, arch: Win32, shared: ON}
+          - {gen: Visual Studio 17 2022, arch: Win32, shared: OFF}
+          - {gen: Visual Studio 17 2022, arch: x64, shared: ON}
+          - {gen: Visual Studio 17 2022, arch: x64, shared: OFF}
     steps:
     - name: checkout
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
     - name: Configure
       run: |
-        cmake -G "${{matrix.gen}}" -A ${{matrix.arch}} -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_BUILD_STATIC=${{matrix.static}} -B build
+        cmake -G "${{matrix.gen}}" -A ${{matrix.arch}} -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -DBUILD_SHARED_LIBS=${{matrix.shared}} -B build
     - name: Build Debug
       run: cmake --build build --config Debug --verbose
     - name: Build Release

.github/workflows/vs17-clang-ci.yml

@@ -1,4 +1,4 @@
-name: VS15-CI
+name: VS17-CLANG-CI
 
 on: [push, pull_request]
 
@@ -7,22 +7,19 @@ jobs:
     if: >-
       ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
-    name: windows-vs15
-    runs-on: windows-2016
+    name: windows-vs17
+    runs-on: windows-latest
     strategy:
       fail-fast: false
       matrix:
         include:
-          - {gen: Visual Studio 15 2017, arch: Win32, static: ON}
-          - {gen: Visual Studio 15 2017, arch: Win32, static: OFF}
-          - {gen: Visual Studio 15 2017, arch: x64, static: ON}
-          - {gen: Visual Studio 15 2017, arch: x64, static: OFF}
+          - {gen: Visual Studio 17 2022, arch: x64}
     steps:
     - name: checkout
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
     - name: Configure
       run: |
-        cmake -G "${{matrix.gen}}" -A ${{matrix.arch}} -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_BUILD_STATIC=${{matrix.static}} -B build
+        cmake -G "${{matrix.gen}}" -A ${{matrix.arch}}  -T ClangCL -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -B build
     - name: Build Debug
       run: cmake --build build --config Debug --verbose
     - name: Build Release

.github/workflows/vs17-noexcept-ci.yml

@@ -1,18 +1,18 @@
-name: VS16-NoExcept-CI
+name: VS17-NoExcept-CI
 
 on: [push, pull_request]
 
 jobs:
   ci:
-    name: windows-vs16
+    name: windows-vs17
     runs-on: windows-latest
     steps:
-    - uses: actions/checkout@v2
-    - uses: actions/cache@v2
+    - uses: actions/checkout@v3
+    - uses: actions/cache@v3
       with:
         path: dependencies/.cache
         key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
-    - name: 'Run CMake with VS16'
+    - name: 'Run CMake with VS17'
       uses: lukka/run-cmake@v3
       with:
         cmakeListsOrSettingsJson: CMakeListsTxtAdvanced

.vscode/settings.json

@@ -86,6 +86,16 @@
         "vector": "cpp",
         "*.ipp": "cpp",
         "__functional_base_03": "cpp",
-        "filesystem": "cpp"
+        "filesystem": "cpp",
+        "*.inc": "cpp",
+        "compare": "cpp",
+        "concepts": "cpp",
+        "variant": "cpp",
+        "__bits": "cpp",
+        "csignal": "cpp",
+        "future": "cpp",
+        "queue": "cpp",
+        "shared_mutex": "cpp",
+        "ranges": "cpp"
     }
 }

CMakeLists.txt

@@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.14)
 project(
     simdjson
     # The version number is modified by tools/release.py
-    VERSION 1.0.0
+    VERSION 2.2.1
     DESCRIPTION "Parsing gigabytes of JSON per second"
     HOMEPAGE_URL "https://simdjson.org/"
     LANGUAGES CXX C
@@ -20,8 +20,8 @@ string(
 # ---- Options, variables ----
 
 # These version numbers are modified by tools/release.py
-set(SIMDJSON_LIB_VERSION "9.0.0" CACHE STRING "simdjson library version")
-set(SIMDJSON_LIB_SOVERSION "9" CACHE STRING "simdjson library soversion")
+set(SIMDJSON_LIB_VERSION "13.0.0" CACHE STRING "simdjson library version")
+set(SIMDJSON_LIB_SOVERSION "13" CACHE STRING "simdjson library soversion")
 
 option(SIMDJSON_ENABLE_THREADS "Link with thread support" ON)
 
@@ -49,19 +49,10 @@ endif()
 if(is_top_project)
   option(SIMDJSON_DEVELOPER_MODE "Enable targets for developing simdjson" OFF)
   option(BUILD_SHARED_LIBS "Build simdjson as a shared library" OFF)
-
-  if("$ENV{CI}")
-    set(SIMDJSON_DEVELOPER_MODE ON CACHE INTERNAL "")
-  endif()
 endif()
 
 include(cmake/handle-deprecations.cmake)
-
-if(SIMDJSON_DEVELOPER_MODE)
-  include(cmake/developer-options.cmake)
-else()
-  message(STATUS "Building only the library. Advanced users may want to turn SIMDJSON_DEVELOPER_MODE to ON, e.g., via -D SIMDJSON_DEVELOPER_MODE=ON.")
-endif()
+include(cmake/developer-options.cmake)
 
 # ---- simdjson library ----
 
@@ -180,6 +171,7 @@ endif()
 # ---- Developer mode extras ----
 
 if(NOT SIMDJSON_DEVELOPER_MODE)
+  message(STATUS "Building only the library. Advanced users may want to turn SIMDJSON_DEVELOPER_MODE to ON, e.g., via -D SIMDJSON_DEVELOPER_MODE=ON.")
   return()
 elseif(NOT is_top_project)
   message(AUTHOR_WARNING "Developer mode is intended for developers of simdjson")

CONTRIBUTING.md

@@ -62,13 +62,13 @@ Pull Requests
 
 Pull requests are always invited. However, we ask that you follow these guidelines:
 
-- It is wiser to discuss your ideas first as part of an issue before you start coding. If you omit this step and code first, be prepare to have your code receive scrutiny and be dropped.
-- Users should provide a rationale for their changes. Does it improve performance? Does it add a feature? Does it improve maintainability? Does fix a bug? This must be explicitly stated as part of the pull request. Do not propose changes based on taste or intuition. We do not delegate programming to tools: that some tool suggested a code change is not reason enough to change the code.
+- It is wise to discuss your ideas first as part of an issue before you start coding. If you omit this step and code first, be prepared to have your code receive scrutiny and be dropped.
+- Users should provide a rationale for their changes. Does it improve performance? Does it add a feature? Does it improve maintainability? Does it fix a bug? This must be explicitly stated as part of the pull request. Do not propose changes based on taste or intuition. We do not delegate programming to tools: that some tool suggested a code change is not reason enough to change the code.
    1. When your code improves performance, please document the gains with a benchmark using hard numbers.
-   2. If your code fixes a bug, please be either fix a failing test, or propose a new test.
+   2. If your code fixes a bug, please either fix a failing test, or propose a new test.
    3. Other types of changes must be clearly motivated. We openly discourage changes with no identifiable benefits.
 - Changes should be focused and minimal. You should change as few lines of code as possible. Please do not reformat or touch files needlessly.
-- New features must be accompanied of new tests, in general.
+- New features must be accompanied by new tests, in general.
 - Your code should pass our continuous-integration tests. It is your responsibility to ensure that your proposal pass the tests. We do not merge pull requests that would break our build.
    - An exception to this would be changes to non-code files, such as documentation and assets, or trivial changes to code, such as comments, where it is encouraged to explicitly ask for skipping a CI run using the `[skip ci]` prefix in your Pull Request title **and** in the first line of the most recent commit in a push. Example for such a commit: `[skip ci] Fixed typo in power_of_ten's docs`
    This benefits the project in such a way that the CI pipeline is not burdened by running jobs on changes that don't change any behavior in the code, which reduces wait times for other Pull Requests that do change behavior and require testing.

CONTRIBUTORS

@@ -37,5 +37,9 @@ Furkan Taşkale
 Brendan Knapp
 Danila Kutenin
 Pavel Pavlov
-# if you have contributed to the project and your name does not 
+Hao Chen
+Nicolas Boyer
+Kim Walisch and Jatin Bhateja (AVX-512 bitset decoder)
+Fangzheng Zhang and Weiqiang Wan (AVX-512 kernel)
+# if you have contributed to the project and your name does not
 # appear in this list, please let us know!

Doxyfile

@@ -38,7 +38,7 @@ PROJECT_NAME           = simdjson
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = "1.0.0"
+PROJECT_NUMBER         = "2.2.1"
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a

README.md

@@ -3,7 +3,7 @@
 [![Ubuntu 20.04 CI](https://github.com/simdjson/simdjson/workflows/Ubuntu%2020.04%20CI%20(GCC%209)/badge.svg)](https://simdjson.org/plots.html)
 ![VS16-CI](https://github.com/simdjson/simdjson/workflows/VS16-CI/badge.svg)
 ![MinGW64-CI](https://github.com/simdjson/simdjson/workflows/MinGW64-CI/badge.svg)
-[![][license img]][license]  [![Doxygen Documentation](https://img.shields.io/badge/docs-doxygen-green.svg)](https://simdjson.org/api/1.0.0/index.html)
+[![][license img]][license]  [![Doxygen Documentation](https://img.shields.io/badge/docs-doxygen-green.svg)](https://simdjson.org/api/2.0.0/index.html)
 
 simdjson : Parsing gigabytes of JSON per second
 ===============================================
@@ -111,10 +111,12 @@ For NDJSON files, we can exceed 3 GB/s with [our  multithreaded parsing function
 Real-world usage
 ----------------
 
-- [Microsoft FishStore](https://github.com/microsoft/FishStore)
-- [Yandex ClickHouse](https://github.com/yandex/ClickHouse)
+- [ClickHouse](https://github.com/ClickHouse/ClickHouse)
 - [Clang Build Analyzer](https://github.com/aras-p/ClangBuildAnalyzer)
 - [Shopify HeapProfiler](https://github.com/Shopify/heap-profiler)
+- [StarRocks](https://github.com/StarRocks/starrocks)
+- [Microsoft FishStore](https://github.com/microsoft/FishStore)
+
 
 If you are planning to use simdjson in a product, please work from one of our releases.
 
@@ -138,7 +140,8 @@ We distinguish between "bindings" (which just wrap the C++ code) and a port to a
 - [rcppsimdjson](https://github.com/eddelbuettel/rcppsimdjson): R bindings.
 - [simdjson_erlang](https://github.com/ChomperT/simdjson_erlang): erlang bindings.
 - [lua-simdjson](https://github.com/FourierTransformer/lua-simdjson): lua bindings.
-
+- [hermes-json](https://hackage.haskell.org/package/hermes-json): haskell bindings.
+- [simdjzon](https://github.com/travisstaloch/simdjzon): zig port.
 
 About simdjson
 --------------

benchmark/amazon_cellphones/amazon_cellphones.h

@@ -4,18 +4,22 @@
 #include <map>
 #include <string>
 
+
 namespace amazon_cellphones {
 
+const bool UNTHREADED = false;
+const bool THREADED = true;
+
 using namespace json_benchmark;
 
 struct brand {
     double cumulative_rating;
     uint64_t reviews_count;
-    simdjson_really_inline bool operator==(const brand &other) const {
+    simdjson_inline bool operator==(const brand &other) const {
         return cumulative_rating == other.cumulative_rating &&
             reviews_count == other.reviews_count;
     }
-    simdjson_really_inline bool operator!=(const brand &other) const { return !(*this == other); }
+    simdjson_inline bool operator!=(const brand &other) const { return !(*this == other); }
 };
 
 simdjson_unused static std::ostream &operator<<(std::ostream &o, const brand &b) {
@@ -59,10 +63,11 @@ struct runner : public file_runner<I> {
     }
 };
 
+template<bool threaded>
 struct simdjson_dom;
 
-template<typename I> simdjson_really_inline static void amazon_cellphones(benchmark::State &state) {
-  run_json_benchmark<runner<I>, runner<simdjson_dom>>(state);
+template<typename I> simdjson_inline static void amazon_cellphones(benchmark::State &state) {
+  run_json_benchmark<runner<I>, runner<simdjson_dom<UNTHREADED>>>(state);
 }
 
-}   // namespace amazon_cellphones
+}   // namespace amazon_cellphones

benchmark/amazon_cellphones/simdjson_dom.h

@@ -8,12 +8,16 @@ namespace amazon_cellphones {
 
 using namespace simdjson;
 
+template<bool threaded>
 struct simdjson_dom {
   using StringType = std::string;
 
   dom::parser parser{};
 
   bool run(simdjson::padded_string &json, std::map<StringType, brand> &result) {
+#ifdef SIMDJSON_THREADS_ENABLED
+    parser.threaded = threaded;
+#endif
     auto stream = parser.parse_many(json);
     auto i = stream.begin();
     ++i;  // Skip first line
@@ -37,7 +41,10 @@ struct simdjson_dom {
 
 };
 
-BENCHMARK_TEMPLATE(amazon_cellphones, simdjson_dom)->UseManualTime();
+BENCHMARK_TEMPLATE(amazon_cellphones, simdjson_dom<UNTHREADED>)->UseManualTime();
+#ifdef SIMDJSON_THREADS_ENABLED
+BENCHMARK_TEMPLATE(amazon_cellphones, simdjson_dom<THREADED>)->UseManualTime();
+#endif
 
 } // namespace amazon_cellphones
 

benchmark/amazon_cellphones/simdjson_ondemand.h

@@ -8,12 +8,16 @@ namespace amazon_cellphones {
 
 using namespace simdjson;
 
+template<bool threaded>
 struct simdjson_ondemand {
   using StringType = std::string;
 
   ondemand::parser parser{};
 
   bool run(simdjson::padded_string &json, std::map<StringType, brand> &result) {
+#ifdef SIMDJSON_THREADS_ENABLED
+    parser.threaded = threaded;
+#endif
     ondemand::document_stream stream = parser.iterate_many(json);
     ondemand::document_stream::iterator i = stream.begin();
     ++i;  // Skip first line
@@ -58,7 +62,10 @@ struct simdjson_ondemand {
 
 };
 
-BENCHMARK_TEMPLATE(amazon_cellphones, simdjson_ondemand)->UseManualTime();
+BENCHMARK_TEMPLATE(amazon_cellphones, simdjson_ondemand<UNTHREADED>)->UseManualTime();
+#ifdef SIMDJSON_THREADS_ENABLED
+BENCHMARK_TEMPLATE(amazon_cellphones, simdjson_ondemand<THREADED>)->UseManualTime();
+#endif
 
 } // namespace amazon_cellphones
 

benchmark/bench_ondemand.cpp

@@ -25,6 +25,12 @@ SIMDJSON_PUSH_DISABLE_ALL_WARNINGS
 #include <benchmark/benchmark.h>
 
 SIMDJSON_POP_DISABLE_WARNINGS
+#include "json2msgpack/simdjson_dom.h"
+#include "json2msgpack/simdjson_ondemand.h"
+#include "json2msgpack/rapidjson.h"
+#include "json2msgpack/yyjson.h"
+#include "json2msgpack/sajson.h"
+#include "json2msgpack/nlohmann_json.h"
 
 #include "amazon_cellphones/simdjson_dom.h"
 #include "amazon_cellphones/simdjson_ondemand.h"

benchmark/benchfeatures.cpp

@@ -97,9 +97,9 @@ struct option_struct {
         verbose = true;
         break;
       case 'a': {
-          auto impl = simdjson::available_implementations[optarg];
+          auto impl = simdjson::get_available_implementations()[optarg];
           if(impl && impl->supported_by_runtime_system()) {
-            simdjson::active_implementation = impl;
+            simdjson::get_active_implementation() = impl;
           } else {
             std::cerr << "implementation " << optarg << " not found or not supported " << std::endl;
           }
@@ -164,7 +164,7 @@ struct feature_benchmarker {
 
   }
 
-  simdjson_really_inline void run_iterations(size_t iterations, bool stage1_only=false) {
+  simdjson_inline void run_iterations(size_t iterations, bool stage1_only=false) {
     struct7.run_iterations(iterations, stage1_only);
     struct7_miss.run_iterations(iterations, stage1_only);
     struct7_full.run_iterations(iterations, stage1_only);
@@ -204,10 +204,13 @@ struct feature_benchmarker {
   }
   // Rate of 1-7-structural misses per 8-structural flip
   double struct1_7_miss_rate(BenchmarkStage stage) const {
+#if SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
+    return 1;
+#else
     if (!has_events()) { return 1; }
     return struct7_miss[stage].best.branch_misses() - struct7[stage].best.branch_misses() / double(struct7_miss.stats->blocks_with_1_structural_flipped);
+#endif
   }
-
   // Extra cost of an 8-15 structural block over a 1-7 structural block
   double struct8_15_cost(BenchmarkStage stage) const {
     return cost_per_block(stage, struct15, struct15.stats->blocks_with_8_structurals, struct7);
@@ -218,8 +221,12 @@ struct feature_benchmarker {
   }
   // Rate of 8-15-structural misses per 8-structural flip
   double struct8_15_miss_rate(BenchmarkStage stage) const {
+#if SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
+    return 1;
+#else
     if (!has_events()) { return 1; }
     return double(struct15_miss[stage].best.branch_misses() - struct15[stage].best.branch_misses()) / double(struct15_miss.stats->blocks_with_8_structurals_flipped);
+#endif
   }
 
   // Extra cost of a 16+-structural block over an 8-15 structural block (actual varies based on # of structurals!)
@@ -232,10 +239,15 @@ struct feature_benchmarker {
   }
   // Rate of 16-structural misses per 16-structural flip
   double struct16_miss_rate(BenchmarkStage stage) const {
+#if SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
+    return 1;
+#else
     if (!has_events()) { return 1; }
     return double(struct23_miss[stage].best.branch_misses() - struct23[stage].best.branch_misses()) / double(struct23_miss.stats->blocks_with_16_structurals_flipped);
+#endif
   }
 
+
   // Extra cost of having UTF-8 in a block
   double utf8_cost(BenchmarkStage stage) const {
     return cost_per_block(stage, utf8, utf8.stats->blocks_with_utf8, struct7_full);
@@ -246,10 +258,13 @@ struct feature_benchmarker {
   }
   // Rate of UTF-8 misses per UTF-8 flip
   double utf8_miss_rate(BenchmarkStage stage) const {
+#if SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
+    return 1;
+#else
     if (!has_events()) { return 1; }
     return double(utf8_miss[stage].best.branch_misses() - utf8[stage].best.branch_misses()) / double(utf8_miss.stats->blocks_with_utf8_flipped);
+#endif
   }
-
   // Extra cost of having escapes in a block
   double escape_cost(BenchmarkStage stage) const {
     return cost_per_block(stage, escape, escape.stats->blocks_with_escapes, struct7_full);
@@ -260,10 +275,15 @@ struct feature_benchmarker {
   }
   // Rate of escape misses per escape flip
   double escape_miss_rate(BenchmarkStage stage) const {
+#if SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
+    return 1;
+#else
     if (!has_events()) { return 1; }
     return double(escape_miss[stage].best.branch_misses() - escape[stage].best.branch_misses()) / double(escape_miss.stats->blocks_with_escapes_flipped);
+#endif
   }
 
+
   double calc_expected_feature_cost(BenchmarkStage stage, const benchmarker& file) const {
     // Expected base ns/block (empty)
     json_stats& stats = *file.stats;
@@ -300,7 +320,6 @@ struct feature_benchmarker {
   double calc_expected(BenchmarkStage stage, const benchmarker& file) const {
     return calc_expected_feature_cost(stage, file) + calc_expected_miss_cost(stage, file);
   }
-
   void print(const option_struct& options) const {
     printf("\n");
     printf("Features in ns/block (64 bytes):\n");
@@ -359,6 +378,22 @@ struct feature_benchmarker {
   }
 };
 
+#if SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
+void print_file_effectiveness(BenchmarkStage stage, const char* filename, const benchmarker& results, const feature_benchmarker& features) {
+  double actual = results[stage].best.elapsed_ns() / double(results.stats->blocks);
+  double calc = features.calc_expected(stage, results);
+  double calc_misses = features.calc_expected_misses(stage, results);
+  double calc_miss_cost = features.calc_expected_miss_cost(stage, results);
+  printf("        | %-8s ", benchmark_stage_name(stage));
+  printf("| %-15s ",   filename);
+  printf("|    %8.3g ", features.calc_expected_feature_cost(stage, results));
+  printf("|    %8.3g ", calc_miss_cost);
+  printf("| %8.3g ",  calc);
+  printf("| %8.3g ",  actual);
+  printf("| %+8.3g ", actual - calc);
+  printf("| %13llu ", (long long unsigned)(calc_misses));
+}
+#else
 void print_file_effectiveness(BenchmarkStage stage, const char* filename, const benchmarker& results, const feature_benchmarker& features) {
   double actual = results[stage].best.elapsed_ns() / double(results.stats->blocks);
   double calc = features.calc_expected(stage, results);
@@ -382,6 +417,7 @@ void print_file_effectiveness(BenchmarkStage stage, const char* filename, const
   }
   printf("|\n");
 }
+#endif
 
 int main(int argc, char *argv[]) {
   // Read options

benchmark/benchmarker.h

@@ -2,7 +2,7 @@
 #define __BENCHMARKER_H
 
 #include "event_counter.h"
-#include "simdjson.h" // For SIMDJSON_DISABLE_DEPRECATED_WARNINGS
+#include "simdjson.h"
 
 #include <cassert>
 #include <cctype>
@@ -308,7 +308,7 @@ struct benchmarker {
     return all_stages_without_allocation.iterations;
   }
 
-  simdjson_really_inline void run_iteration(bool stage1_only, bool hotbuffers=false) {
+  simdjson_inline void run_iteration(bool stage1_only, bool hotbuffers=false) {
     // Allocate dom::parser
     collector.start();
     dom::parser parser;
@@ -384,7 +384,7 @@ struct benchmarker {
     loop << all_loop_count;
   }
 
-  simdjson_really_inline void run_iterations(size_t iterations, bool stage1_only, bool hotbuffers=false) {
+  simdjson_inline void run_iterations(size_t iterations, bool stage1_only, bool hotbuffers=false) {
     for (size_t i = 0; i<iterations; i++) {
       run_iteration(stage1_only, hotbuffers);
     }
@@ -423,7 +423,7 @@ struct benchmarker {
         stage.instructions() / static_cast<double>(stats->structurals),
         stage.instructions() / static_cast<double>(stage.cycles())
       );
-
+#if !SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
       // NOTE: removed cycles/miss because it is a somewhat misleading stat
       printf("%s%-13s: %7.0f branch misses (%6.2f%%) - %.0f cache misses (%6.2f%%) - %.2f cache references\n",
         prefix,
@@ -434,6 +434,7 @@ struct benchmarker {
         percent(stage.cache_misses(), all_stages_without_allocation.cache_misses()),
         stage.cache_references()
       );
+#endif
     }
   }
 

benchmark/distinct_user_id/distinct_user_id.h

@@ -46,7 +46,7 @@ struct runner : public file_runner<I> {
 
 struct simdjson_dom;
 
-template<typename I> simdjson_really_inline static void distinct_user_id(benchmark::State &state) {
+template<typename I> simdjson_inline static void distinct_user_id(benchmark::State &state) {
   run_json_benchmark<runner<I>, runner<simdjson_dom>>(state);
 }
 

benchmark/distinct_user_id/rapidjson.h

@@ -48,7 +48,7 @@ BENCHMARK_TEMPLATE(distinct_user_id, rapidjson)->UseManualTime();
 
 struct rapidjson_insitu : rapidjson_base {
   bool run(simdjson::padded_string &json, std::vector<uint64_t> &result) {
-    return rapidjson_base::run(doc.ParseInsitu<kParseValidateEncodingFlag>(json.data()), result);
+    return rapidjson_base::run(doc.ParseInsitu<kParseValidateEncodingFlag|kParseInsituFlag>(json.data()), result);
   }
 };
 BENCHMARK_TEMPLATE(distinct_user_id, rapidjson_insitu)->UseManualTime();

benchmark/distinct_user_id/sajson.h

@@ -9,12 +9,14 @@ namespace distinct_user_id {
 struct sajson {
   size_t ast_buffer_size{0};
   size_t *ast_buffer{nullptr};
-  simdjson_really_inline std::string_view get_string_view(const ::sajson::value &obj, std::string_view key) {
+  ~sajson() { free(ast_buffer); }
+
+  simdjson_inline std::string_view get_string_view(const ::sajson::value &obj, std::string_view key) {
     auto val = obj.get_value_of_key({key.data(), key.length()});
     if (val.get_type() != ::sajson::TYPE_STRING) { throw "field is not a string"; }
     return { val.as_cstring(), val.get_string_length() };
   }
-  simdjson_really_inline uint64_t get_str_uint64(const ::sajson::value &obj, std::string_view key) {
+  simdjson_inline uint64_t get_str_uint64(const ::sajson::value &obj, std::string_view key) {
     // Since sajson only supports 53-bit numbers, and IDs in twitter.json can be > 53 bits, we read the corresponding id_str and parse that.
     auto val = obj.get_value_of_key({key.data(), key.length()});
     if (val.get_type() != ::sajson::TYPE_STRING) { throw "field not a string"; }

benchmark/dom/parse.cpp

@@ -66,7 +66,7 @@ void print_usage(ostream& out) {
   out << "-H           - Make the buffers hot (reduce page allocation and related OS tasks during parsing) [default]" << endl;
   out << "-a IMPL      - Use the given parser implementation. By default, detects the most advanced" << endl;
   out << "               implementation supported on the host machine." << endl;
-  for (auto impl : simdjson::available_implementations) {
+  for (auto impl : simdjson::get_available_implementations()) {
     if(impl->supported_by_runtime_system()) {
       out << "-a " << std::left << std::setw(9) << impl->name() << " - Use the " << impl->description() << " parser implementation." << endl;
     }
@@ -116,10 +116,10 @@ struct option_struct {
         verbose = true;
         break;
       case 'a': {
-        const implementation *impl = simdjson::available_implementations[optarg];
+        const implementation *impl = simdjson::get_available_implementations()[optarg];
         if ((!impl) || (!impl->supported_by_runtime_system())) {
           std::string exit_message = string("Unsupported option value -a ") + optarg + ": expected -a  with one of ";
-          for (auto imple : simdjson::available_implementations) {
+          for (auto imple : simdjson::get_available_implementations()) {
             if(imple->supported_by_runtime_system()) {
               exit_message += imple->name();
               exit_message += " ";
@@ -127,7 +127,7 @@ struct option_struct {
           }
           exit_usage(exit_message);
         }
-        simdjson::active_implementation = impl;
+        simdjson::get_active_implementation() = impl;
         break;
       }
       case 'C':
@@ -175,7 +175,7 @@ int main(int argc, char *argv[]) {
   option_struct options(argc, argv);
   if (options.verbose) {
     verbose_stream = &cout;
-    verbose() << "Implementation: " << simdjson::active_implementation->name() << endl;
+    verbose() << "Implementation: " << simdjson::get_active_implementation()->name() << endl;
   }
 
   // Start collecting events. We put this early so if it prints an error message, it's the

benchmark/dom/statisticalmodel.cpp

@@ -41,7 +41,7 @@ using stat_t = struct stat_s;
 
 
 
-simdjson_really_inline void simdjson_process_atom(stat_t &s,
+simdjson_inline void simdjson_process_atom(stat_t &s,
                                          simdjson::dom::element element) {
   if (element.is<int64_t>()) {
     s.integer_count++;
@@ -50,9 +50,9 @@ simdjson_really_inline void simdjson_process_atom(stat_t &s,
   } else if(element.is<double>()) {
     s.float_count++;
   } else if (element.is<bool>()) {
-    simdjson::error_code err;
     bool v;
-    err = element.get(v);
+    simdjson::error_code error;
+    if ((error = element.get(v))) { std::cerr << error << std::endl; abort(); }
     if (v) {
       s.true_count++;
     } else {

benchmark/event_counter.h

@@ -1,6 +1,15 @@
 #ifndef __EVENT_COUNTER_H
 #define __EVENT_COUNTER_H
 
+#ifndef SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
+#ifdef __aarch64__
+// on ARM, we use just cycles and instructions
+#define SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS 1
+#else
+// elsewhere, we try to use four counters.
+#define SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS 0
+#endif
+#endif
 #include <cassert>
 #include <cctype>
 #ifndef _MSC_VER
@@ -46,6 +55,12 @@ struct event_count {
   event_count(const event_count& other): elapsed(other.elapsed), event_counts(other.event_counts) { }
 
   // The types of counters (so we can read the getter more easily)
+  #if SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
+  enum event_counter_types {
+    CPU_CYCLES,
+    INSTRUCTIONS
+  };
+  #else
   enum event_counter_types {
     CPU_CYCLES,
     INSTRUCTIONS,
@@ -53,15 +68,16 @@ struct event_count {
     CACHE_REFERENCES,
     CACHE_MISSES
   };
-
+  #endif
   double elapsed_sec() const { return duration<double>(elapsed).count(); }
   double elapsed_ns() const { return duration<double, std::nano>(elapsed).count(); }
   double cycles() const { return static_cast<double>(event_counts[CPU_CYCLES]); }
   double instructions() const { return static_cast<double>(event_counts[INSTRUCTIONS]); }
+#if !SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
   double branch_misses() const { return static_cast<double>(event_counts[BRANCH_MISSES]); }
   double cache_references() const { return static_cast<double>(event_counts[CACHE_REFERENCES]); }
   double cache_misses() const { return static_cast<double>(event_counts[CACHE_MISSES]); }
-
+#endif
   event_count& operator=(const event_count& other) {
     this->elapsed = other.elapsed;
     this->event_counts = other.event_counts;
@@ -105,9 +121,11 @@ struct event_aggregate {
   double elapsed_ns() const { return total.elapsed_ns() / iterations; }
   double cycles() const { return total.cycles() / iterations; }
   double instructions() const { return total.instructions() / iterations; }
+#if !SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
   double branch_misses() const { return total.branch_misses() / iterations; }
   double cache_references() const { return total.cache_references() / iterations; }
   double cache_misses() const { return total.cache_misses() / iterations; }
+#endif
 };
 
 struct event_collector {
@@ -117,11 +135,16 @@ struct event_collector {
 #if defined(__linux__)
   LinuxEvents<PERF_TYPE_HARDWARE> linux_events;
   event_collector(bool quiet = false) : linux_events(vector<int>{
+  #if SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
+    PERF_COUNT_HW_CPU_CYCLES,
+    PERF_COUNT_HW_INSTRUCTIONS,
+  #else
     PERF_COUNT_HW_CPU_CYCLES,
     PERF_COUNT_HW_INSTRUCTIONS,
     PERF_COUNT_HW_BRANCH_MISSES,
     PERF_COUNT_HW_CACHE_REFERENCES,
     PERF_COUNT_HW_CACHE_MISSES
+  #endif
   }, quiet) {}
   bool has_events() {
     return linux_events.is_working();
@@ -133,13 +156,13 @@ struct event_collector {
   }
 #endif
 
-  simdjson_really_inline void start() {
+  simdjson_inline void start() {
 #if defined(__linux)
     linux_events.start();
 #endif
     start_clock = steady_clock::now();
   }
-  simdjson_really_inline event_count& end() {
+  simdjson_inline event_count& end() {
     time_point<steady_clock> end_clock = steady_clock::now();
 #if defined(__linux)
     linux_events.end(count.event_counts);

benchmark/find_tweet/find_tweet.h

@@ -33,7 +33,7 @@ struct runner : public file_runner<I> {
 
 struct simdjson_dom;
 
-template<typename I> simdjson_really_inline static void find_tweet(benchmark::State &state) {
+template<typename I> simdjson_inline static void find_tweet(benchmark::State &state) {
   run_json_benchmark<runner<I>, runner<simdjson_dom>>(state);
 }
 

benchmark/find_tweet/rapidjson.h

@@ -42,7 +42,7 @@ BENCHMARK_TEMPLATE(find_tweet, rapidjson)->UseManualTime();
 
 struct rapidjson_insitu : rapidjson_base {
   bool run(simdjson::padded_string &json, uint64_t find_id, std::string_view &result) {
-    return rapidjson_base::run(doc.ParseInsitu<kParseValidateEncodingFlag>(json.data()), find_id, result);
+    return rapidjson_base::run(doc.ParseInsitu<kParseValidateEncodingFlag|kParseInsituFlag>(json.data()), find_id, result);
   }
 };
 BENCHMARK_TEMPLATE(find_tweet, rapidjson_insitu)->UseManualTime();

benchmark/find_tweet/sajson.h

@@ -11,12 +11,14 @@ struct sajson {
 
   size_t ast_buffer_size{0};
   size_t *ast_buffer{nullptr};
-  simdjson_really_inline std::string_view get_string_view(const ::sajson::value &obj, std::string_view key) {
+  ~sajson() { free(ast_buffer); }
+
+  simdjson_inline std::string_view get_string_view(const ::sajson::value &obj, std::string_view key) {
     auto val = obj.get_value_of_key({key.data(), key.length()});
     if (val.get_type() != ::sajson::TYPE_STRING) { throw "field is not a string"; }
     return { val.as_cstring(), val.get_string_length() };
   }
-  simdjson_really_inline uint64_t get_str_uint64(const ::sajson::value &obj, std::string_view key) {
+  simdjson_inline uint64_t get_str_uint64(const ::sajson::value &obj, std::string_view key) {
     // Since sajson only supports 53-bit numbers, and IDs in twitter.json can be > 53 bits, we read the corresponding id_str and parse that.
     auto val = obj.get_value_of_key({key.data(), key.length()});
     if (val.get_type() != ::sajson::TYPE_STRING) { throw "field not a string"; }

benchmark/json2msgpack/json2msgpack.h

@@ -0,0 +1,48 @@
+#pragma once
+
+#include "json_benchmark/file_runner.h"
+
+namespace json2msgpack {
+
+using namespace json_benchmark;
+
+template <typename I> struct runner : public file_runner<I> {
+  std::string_view result;
+  std::unique_ptr<char[]> buffer;
+
+  bool setup(benchmark::State &state) {
+    bool isok = this->load_json(state, TWITTER_JSON);
+    if (isok) {
+      // Let us allocate a sizeable buffer.
+      buffer = std::unique_ptr<char[]>(new char[this->json.size() * 4 + 1024]);
+    }
+    return isok;
+  }
+
+  bool before_run(benchmark::State &state) {
+    if (!file_runner<I>::before_run(state)) {
+      return false;
+    }
+    // Clear the buffer.
+    ::memset(buffer.get(), 0, this->json.size() * 4 + 1024);
+    return true;
+  }
+
+  bool run(benchmark::State &) {
+    return this->implementation.run(this->json, buffer.get(), result);
+  }
+
+  template <typename R>
+  bool diff(benchmark::State &state, runner<R> &reference) {
+    return diff_results(state, result.size(), reference.result.size(), diff_flags::NONE);
+  }
+};
+
+struct simdjson_ondemand;
+
+template <typename I>
+simdjson_inline static void json2msgpack(benchmark::State &state) {
+  run_json_benchmark<runner<I>, runner<simdjson_ondemand>>(state);
+}
+
+} // namespace json2msgpack

benchmark/json2msgpack/nlohmann_json.h

@@ -0,0 +1,117 @@
+#pragma once
+#ifdef SIMDJSON_COMPETITION_NLOHMANN_JSON
+
+#include "json2msgpack.h"
+
+namespace json2msgpack {
+
+using namespace nlohmann;
+
+struct nlohmann_json2msgpack {
+  inline std::string_view to_msgpack(const simdjson::padded_string &json,
+                                     uint8_t *buf);
+
+private:
+  inline void write_double(const double d) noexcept;
+  inline void write_byte(const uint8_t b) noexcept;
+  inline void write_uint32(const uint32_t w) noexcept;
+  inline void write_string(const std::string& str);
+  inline void recursive_processor(basic_json<> element);
+
+  uint8_t *buff{};
+};
+
+std::string_view nlohmann_json2msgpack::to_msgpack(const simdjson::padded_string &json,
+                             uint8_t *buf) {
+  buff = buf;
+  auto val = nlohmann::json::parse(json.data(), json.data() + json.size());
+  recursive_processor(val);
+  return std::string_view(reinterpret_cast<char *>(buf), size_t(buff - buf));
+}
+
+void nlohmann_json2msgpack::write_double(const double d) noexcept {
+  *buff++ = 0xcb;
+  ::memcpy(buff, &d, sizeof(d));
+  buff += sizeof(d);
+}
+
+void nlohmann_json2msgpack::write_byte(const uint8_t b) noexcept {
+  *buff = b;
+  buff++;
+}
+
+void nlohmann_json2msgpack::write_uint32(const uint32_t w) noexcept {
+  ::memcpy(buff, &w, sizeof(w));
+  buff += sizeof(w);
+}
+
+void nlohmann_json2msgpack::write_string(const std::string & str) {
+  write_byte(0xdb);
+  write_uint32(uint32_t(str.size()));
+  ::memcpy(buff, str.data(), str.size());
+  buff += str.size();
+}
+
+void nlohmann_json2msgpack::recursive_processor(json element) {
+  switch (element.type()) {
+  case nlohmann::detail::value_t::array: {
+    uint32_t counter = 0;
+    write_byte(0xdd);
+    std::vector<json> array = element.get<std::vector<json>>();
+    write_uint32(uint32_t(array.size()));
+    for (auto child : array) {
+       recursive_processor(child);
+    }
+  } break;
+  case nlohmann::detail::value_t::object: {
+    write_byte(0xdf);
+    std::map<std::string,json> object = element.get<std::map<std::string,json>>();
+    write_uint32(uint32_t(object.size()));
+    for (auto field : object) {
+      write_string(field.first);
+      recursive_processor(field.second);
+    }
+  } break;
+
+  case nlohmann::detail::value_t::number_integer:
+  case nlohmann::detail::value_t::number_unsigned:
+  case nlohmann::detail::value_t::number_float:
+    write_double(double(element));
+    break;
+  case nlohmann::detail::value_t::string:
+    write_string(std::string(element));
+    break;
+  case nlohmann::detail::value_t::boolean:
+    write_byte(0xc2 + bool(element));
+    break;
+  case nlohmann::detail::value_t::null:
+    write_byte(0xc0);
+    break;
+  case nlohmann::detail::value_t::discarded:
+  case nlohmann::detail::value_t::binary:
+  default:
+    printf("unexpected\n");
+    break;
+  }
+}
+
+struct nlohmann_json {
+  using StringType = std::string_view;
+
+  nlohmann_json2msgpack parser{};
+
+  bool run(simdjson::padded_string &json, char *buffer,
+           std::string_view &result) {
+    result = parser.to_msgpack(json, reinterpret_cast<uint8_t *>(buffer));
+    return true;
+  }
+};
+
+BENCHMARK_TEMPLATE(json2msgpack, nlohmann_json)->UseManualTime();
+
+} // namespace json2msgpack
+
+#endif // SIMDJSON_COMPETITION_NLOHMANN_JSON
+
+
+

benchmark/json2msgpack/rapidjson.h

@@ -0,0 +1,140 @@
+
+
+#pragma once
+#ifdef SIMDJSON_COMPETITION_RAPIDJSON
+
+#include "json2msgpack.h"
+
+namespace json2msgpack {
+
+using namespace rapidjson;
+
+template <int parseflag>
+struct rapidjson2msgpack {
+  inline std::string_view to_msgpack(char *json, uint8_t *buf);
+
+private:
+  inline void write_double(const double d) noexcept;
+  inline void write_byte(const uint8_t b) noexcept;
+  inline void write_uint32(const uint32_t w) noexcept;
+  inline void write_uint32_at(const uint32_t w, uint8_t *p) noexcept;
+  void write_string(const char * s, size_t length) noexcept;
+  inline void recursive_processor(Value &v);
+
+  uint8_t *buff{};
+};
+
+template <int parseflag>
+std::string_view rapidjson2msgpack<parseflag>::to_msgpack(char *json, uint8_t *buf) {
+  buff = buf;
+  Document doc{};
+  if(parseflag & kParseInsituFlag) {
+    doc.ParseInsitu<parseflag>(json);
+  } else {
+    doc.Parse<parseflag>(json);
+  }
+  recursive_processor(doc);
+  return std::string_view(reinterpret_cast<char *>(buf), size_t(buff - buf));
+}
+
+template <int parseflag>
+void rapidjson2msgpack<parseflag>::write_double(const double d) noexcept {
+  *buff++ = 0xcb;
+  ::memcpy(buff, &d, sizeof(d));
+  buff += sizeof(d);
+}
+
+template <int parseflag>
+void rapidjson2msgpack<parseflag>::write_byte(const uint8_t b) noexcept {
+  *buff = b;
+  buff++;
+}
+
+template <int parseflag>
+void rapidjson2msgpack<parseflag>::write_string(const char * c, size_t len) noexcept {
+  write_byte(0xdb);
+  write_uint32(uint32_t(len));
+  ::memcpy(buff, c, len);
+  buff += len;
+}
+
+template <int parseflag>
+void rapidjson2msgpack<parseflag>::write_uint32(const uint32_t w) noexcept {
+  ::memcpy(buff, &w, sizeof(w));
+  buff += sizeof(w);
+}
+
+template <int parseflag>
+void rapidjson2msgpack<parseflag>::write_uint32_at(const uint32_t w, uint8_t *p) noexcept {
+  ::memcpy(p, &w, sizeof(w));
+}
+
+template <int parseflag>
+void rapidjson2msgpack<parseflag>::recursive_processor(Value &v) {
+  switch (v.GetType()) {
+  case kArrayType:
+    write_byte(0xdd);
+    write_uint32(v.Size());
+    for (Value::ValueIterator i = v.Begin(); i != v.End(); ++i) {
+      recursive_processor(*i);
+    }
+    break;
+  case kObjectType:
+    write_byte(0xdf);
+    write_uint32(uint32_t(v.MemberEnd()-v.MemberBegin()));
+    for (Value::MemberIterator m = v.MemberBegin(); m != v.MemberEnd();
+         ++m) {
+      write_string(m->name.GetString(), m->name.GetStringLength());
+      recursive_processor(m->value);
+    }
+    break;
+  case kStringType:
+    write_string(v.GetString(), v.GetStringLength());
+    break;
+  case kNumberType:
+    write_double(v.GetDouble());
+    break;
+  case kFalseType:
+    write_byte(0xc2);
+    break;
+  case kTrueType:
+    write_byte(0xc3);
+    break;
+  case kNullType:
+    write_byte(0xc0);
+    break;
+  }
+}
+
+template <int parseflag>
+struct rapidjson_base {
+  using StringType = std::string_view;
+
+  rapidjson2msgpack<parseflag> parser{};
+
+  bool run(simdjson::padded_string &json, char *buffer,
+           std::string_view &result) {
+    result =
+        parser.to_msgpack(json.data(), reinterpret_cast<uint8_t *>(buffer));
+
+    return true;
+  }
+};
+
+
+using rapidjson_lossless = rapidjson_base<kParseValidateEncodingFlag|kParseFullPrecisionFlag>;
+
+BENCHMARK_TEMPLATE(json2msgpack, rapidjson_lossless)->UseManualTime();
+
+
+using rapidjson = rapidjson_base<kParseValidateEncodingFlag>;
+
+BENCHMARK_TEMPLATE(json2msgpack, rapidjson)->UseManualTime();
+
+using rapidjson_insitu = rapidjson_base<kParseValidateEncodingFlag|kParseInsituFlag>;
+
+BENCHMARK_TEMPLATE(json2msgpack, rapidjson_insitu)->UseManualTime();
+
+} // namespace json2msgpack
+
+#endif // SIMDJSON_COMPETITION_RAPIDJSON

benchmark/json2msgpack/sajson.h

@@ -0,0 +1,131 @@
+#pragma once
+#ifdef SIMDJSON_COMPETITION_SAJSON
+
+#include "json2msgpack.h"
+
+namespace json2msgpack {
+
+using namespace sajson;
+
+
+struct sajson2msgpack {
+  inline std::string_view to_msgpack(char *json, size_t size, uint8_t *buf);
+  virtual ~sajson2msgpack() { free(ast_buffer); }
+
+private:
+  inline void write_double(const double d) noexcept;
+  inline void write_byte(const uint8_t b) noexcept;
+  inline void write_uint32(const uint32_t w) noexcept;
+  inline void write_string(const char * s, size_t length) noexcept;
+  inline void recursive_processor(const sajson::value &v);
+
+  uint8_t *buff{};
+  size_t ast_buffer_size{0};
+  size_t *ast_buffer{nullptr};
+};
+
+
+std::string_view sajson2msgpack::to_msgpack(char *json, size_t size, uint8_t *buf) {
+  buff = buf;
+
+  if (!ast_buffer) {
+    ast_buffer_size = size;
+    ast_buffer = (size_t *)std::malloc(ast_buffer_size * sizeof(size_t));
+  }
+  auto doc = parse(
+      bounded_allocation(ast_buffer, ast_buffer_size),
+      mutable_string_view(size, json)
+  );
+
+  auto root = doc.get_root();
+  recursive_processor(root);
+  return std::string_view(reinterpret_cast<char *>(buf), size_t(buff - buf));
+}
+
+void sajson2msgpack::write_string(const char * c, size_t len) noexcept {
+  write_byte(0xdb);
+  write_uint32(uint32_t(len));
+  ::memcpy(buff, c, len);
+  buff += len;
+}
+
+void sajson2msgpack::write_double(const double d) noexcept {
+  *buff++ = 0xcb;
+  ::memcpy(buff, &d, sizeof(d));
+  buff += sizeof(d);
+}
+
+void sajson2msgpack::write_byte(const uint8_t b) noexcept {
+  *buff = b;
+  buff++;
+}
+
+void sajson2msgpack::write_uint32(const uint32_t w) noexcept {
+  ::memcpy(buff, &w, sizeof(w));
+  buff += sizeof(w);
+}
+
+void sajson2msgpack::recursive_processor(const sajson::value &node) {
+  using namespace sajson;
+  switch (node.get_type()) {
+  case TYPE_NULL:
+    write_byte(0xc0);
+    break;
+  case TYPE_FALSE:
+    write_byte(0xc2);
+    break;
+  case TYPE_TRUE:
+    write_byte(0xc3);
+    break;
+  case TYPE_ARRAY: {
+    auto length = node.get_length();
+    write_byte(0xdf);
+    write_uint32(uint32_t(length));
+    for (size_t i = 0; i < length; ++i) {
+      recursive_processor(node.get_array_element(i));
+    }
+    break;
+  }
+  case TYPE_OBJECT: {
+    auto length = node.get_length();
+    write_byte(0xdd);
+    write_uint32(uint32_t(length));
+    for (auto i = 0u; i < length; ++i) {
+      auto s = node.get_object_key(i);
+      write_string(s.data(), s.length());
+      recursive_processor(node.get_object_value(i));
+    }
+    break;
+  }
+  case TYPE_STRING:
+    write_string(node.as_cstring(), node.get_string_length());
+    break;
+  case TYPE_DOUBLE:
+  case TYPE_INTEGER:
+    write_double(node.get_number_value());
+    break;
+  default:
+    assert(false && "unknown node type");
+  }
+}
+
+
+struct sajson {
+  using StringType = std::string_view;
+
+  sajson2msgpack parser{};
+
+  bool run(simdjson::padded_string &json, char *buffer,
+           std::string_view &result) {
+    result =
+        parser.to_msgpack(json.data(), json.size(), reinterpret_cast<uint8_t *>(buffer));
+
+    return true;
+  }
+};
+
+BENCHMARK_TEMPLATE(json2msgpack, sajson)->UseManualTime();
+
+} // namespace json2msgpack
+
+#endif // SIMDJSON_COMPETITION_SAJSON

benchmark/json2msgpack/simdjson_dom.h

@@ -0,0 +1,138 @@
+#pragma once
+#if SIMDJSON_EXCEPTIONS
+
+
+#include "json2msgpack.h"
+
+namespace json2msgpack {
+
+using namespace simdjson;
+
+struct simdjsondom2msgpack {
+  /**
+   * @brief Converts the provided JSON into msgpack.
+   *
+   * @param json JSON input
+   * @param buf temporary buffer (must be large enough, with simdjson::SIMDJSON_PADDING bytes
+   * of padding)
+   * @return std::string_view msgpack output, writting to the temporary buffer
+   */
+  inline std::string_view to_msgpack(const simdjson::padded_string &json,
+                                     uint8_t *buf);
+
+private:
+  simdjson_really_inline void write_double(const double d) noexcept;
+  simdjson_really_inline void write_string(const std::string_view v) noexcept;
+  simdjson_really_inline void write_byte(const uint8_t b) noexcept;
+  simdjson_really_inline void write_uint32(const uint32_t w) noexcept;
+  simdjson_really_inline uint8_t *skip_uint32() noexcept;
+  simdjson_really_inline void write_uint32_at(const uint32_t w,
+                                              uint8_t *p) noexcept;
+
+  inline void recursive_processor(simdjson::dom::element element);
+
+  dom::parser parser;
+  uint8_t *buff{};
+};
+
+std::string_view
+simdjsondom2msgpack::to_msgpack(const simdjson::padded_string &json,
+                             uint8_t *buf) {
+  buff = buf;
+
+  recursive_processor(parser.parse(json));
+  return std::string_view(reinterpret_cast<char *>(buf), size_t(buff - buf));
+}
+void simdjsondom2msgpack::write_string(const std::string_view v) noexcept {
+  write_byte(0xdb);
+  write_uint32(uint32_t(v.size()));
+  ::memcpy(buff, v.data(), v.size());
+  buff += v.size();
+}
+void simdjsondom2msgpack::write_double(const double d) noexcept {
+  *buff++ = 0xcb;
+  ::memcpy(buff, &d, sizeof(d));
+  buff += sizeof(d);
+}
+
+void simdjsondom2msgpack::write_byte(const uint8_t b) noexcept {
+  *buff = b;
+  buff++;
+}
+
+void simdjsondom2msgpack::write_uint32(const uint32_t w) noexcept {
+  ::memcpy(buff, &w, sizeof(w));
+  buff += sizeof(w);
+}
+
+uint8_t *simdjsondom2msgpack::skip_uint32() noexcept {
+  uint8_t *ret = buff;
+  buff += sizeof(uint32_t);
+  return ret;
+}
+
+void simdjsondom2msgpack::write_uint32_at(const uint32_t w, uint8_t *p) noexcept {
+  ::memcpy(p, &w, sizeof(w));
+}
+
+
+void simdjsondom2msgpack::recursive_processor(simdjson::dom::element element) {
+  switch (element.type()) {
+    case dom::element_type::ARRAY: {
+      uint32_t counter = 0;
+      write_byte(0xdd);
+      uint8_t *location = skip_uint32();
+      for (auto child : dom::array(element)) {
+        counter++;
+        recursive_processor(child);
+      }
+      write_uint32_at(counter, location);}
+      break;
+    case dom::element_type::OBJECT:{
+      uint32_t counter = 0;
+      write_byte(0xdf);
+      uint8_t *location = skip_uint32();
+      for (dom::key_value_pair field : dom::object(element)) {
+        counter++;
+        write_string(field.key);
+        recursive_processor(field.value);
+      }
+      write_uint32_at(counter, location);
+      }
+      break;
+    case dom::element_type::INT64:
+    case dom::element_type::UINT64:
+    case dom::element_type::DOUBLE:
+      write_double( double(element));
+      break;
+    case dom::element_type::STRING:
+      write_string(std::string_view(element));
+      break;
+    case dom::element_type::BOOL:
+      write_byte(0xc2 + bool(element));
+      break;
+    case dom::element_type::NULL_VALUE:
+      write_byte(0xc0);
+      break;
+    default:
+      break;
+  }
+}
+
+struct simdjson_dom {
+  using StringType = std::string_view;
+
+  simdjsondom2msgpack parser{};
+
+  bool run(simdjson::padded_string &json, char *buffer,
+           std::string_view &result) {
+    result = parser.to_msgpack(json, reinterpret_cast<uint8_t *>(buffer));
+    return true;
+  }
+};
+
+BENCHMARK_TEMPLATE(json2msgpack, simdjson_dom)->UseManualTime();
+
+} // namespace json2msgpack
+
+#endif // SIMDJSON_EXCEPTIONS

benchmark/json2msgpack/simdjson_ondemand.h

@@ -0,0 +1,182 @@
+#pragma once
+#if SIMDJSON_EXCEPTIONS
+
+#include "json2msgpack.h"
+
+namespace json2msgpack {
+
+using namespace simdjson;
+
+/**
+ * @brief The simdjson2msgpack struct is used to quickly convert
+ * JSON strings to msgpack views. You must provide a pointer to
+ * a large memory region where the msgpack gets written. The
+ * buffer should be large enough to store the msgpack output (which
+ * can never be 3x larger than the input JSON) with an additional
+ * simdjson::SIMDJSON_PADDING bytes.
+ *
+ * Recommended usage:
+ *
+ * simdjson2msgpack parser{};
+ * simdjson::padded_string json = "[1,2]"_padded; // some JSON
+ * uint8_t * buffer = new uint8_t[3*json.size() + simdjson::SIMDJSON_PADDING]; // large buffer
+ *
+ * std::string_view msgpack = parser.to_msgpack(json, buffer);
+ *
+ * The result (msgpack) is a string view to a msgpack serialization of the input JSON,
+ * it points inside the buffer you provided.
+ *
+ * You may reuse the simdjson2msgpack instance though you should use
+ * one per thread.
+ */
+struct simdjson2msgpack {
+  /**
+   * @brief Converts the provided JSON into msgpack.
+   *
+   * @param json JSON input
+   * @param buf temporary buffer (must be large enough, with simdjson::SIMDJSON_PADDING bytes
+   * of padding)
+   * @return std::string_view msgpack output, writting to the temporary buffer
+   */
+  inline std::string_view to_msgpack(const simdjson::padded_string &json,
+                                     uint8_t *buf);
+
+private:
+  simdjson_inline void write_double(const double d) noexcept;
+  simdjson_inline void write_byte(const uint8_t b) noexcept;
+  simdjson_inline void write_uint32(const uint32_t w) noexcept;
+  simdjson_inline uint8_t *skip_uint32() noexcept;
+  simdjson_inline void write_uint32_at(const uint32_t w,
+                                              uint8_t *p) noexcept;
+  simdjson_inline void
+  write_raw_string(simdjson::ondemand::raw_json_string rjs);
+  inline void recursive_processor(simdjson::ondemand::value element);
+
+  simdjson::ondemand::parser parser;
+  uint8_t *buff{};
+};
+
+std::string_view
+simdjson2msgpack::to_msgpack(const simdjson::padded_string &json,
+                             uint8_t *buf) {
+  buff = buf;
+  ondemand::document doc = parser.iterate(json);
+  if (doc.is_scalar()) {
+    // we have a special case where the JSON document is a single document...
+    switch (doc.type()) {
+    case simdjson::ondemand::json_type::number:
+      write_double(doc.get_double());
+      break;
+    case simdjson::ondemand::json_type::string:
+      write_raw_string(doc.get_raw_json_string());
+      break;
+    case simdjson::ondemand::json_type::boolean:
+      write_byte(0xc2 + doc.get_bool());
+      break;
+    case simdjson::ondemand::json_type::null:
+      write_byte(0xc0);
+      break;
+    case simdjson::ondemand::json_type::array:
+    case simdjson::ondemand::json_type::object:
+    default:
+      // impossible
+      break;
+    }
+  } else {
+    simdjson::ondemand::value val = doc;
+    recursive_processor(val);
+  }
+  return std::string_view(reinterpret_cast<char *>(buf), size_t(buff - buf));
+}
+
+void simdjson2msgpack::write_double(const double d) noexcept {
+  *buff++ = 0xcb;
+  ::memcpy(buff, &d, sizeof(d));
+  buff += sizeof(d);
+}
+
+void simdjson2msgpack::write_byte(const uint8_t b) noexcept {
+  *buff = b;
+  buff++;
+}
+
+void simdjson2msgpack::write_uint32(const uint32_t w) noexcept {
+  ::memcpy(buff, &w, sizeof(w));
+  buff += sizeof(w);
+}
+
+uint8_t *simdjson2msgpack::skip_uint32() noexcept {
+  uint8_t *ret = buff;
+  buff += sizeof(uint32_t);
+  return ret;
+}
+
+void simdjson2msgpack::write_uint32_at(const uint32_t w, uint8_t *p) noexcept {
+  ::memcpy(p, &w, sizeof(w));
+}
+
+void simdjson2msgpack::write_raw_string(
+    simdjson::ondemand::raw_json_string in) {
+  write_byte(0xdb);
+  uint8_t *location = skip_uint32();
+  std::string_view v = parser.unescape(in, buff);
+  write_uint32_at(uint32_t(v.size()), location);
+}
+
+void simdjson2msgpack::recursive_processor(simdjson::ondemand::value element) {
+  switch (element.type()) {
+  case simdjson::ondemand::json_type::array: {
+    uint32_t counter = 0;
+    write_byte(0xdd);
+    uint8_t *location = skip_uint32();
+    for (auto child : element.get_array()) {
+      counter++;
+      recursive_processor(child.value());
+    }
+    write_uint32_at(counter, location);
+  } break;
+  case simdjson::ondemand::json_type::object: {
+    uint32_t counter = 0;
+    write_byte(0xdf);
+    uint8_t *location = skip_uint32();
+    for (auto field : element.get_object()) {
+      counter++;
+      write_raw_string(field.key());
+      recursive_processor(field.value());
+    }
+    write_uint32_at(counter, location);
+  } break;
+  case simdjson::ondemand::json_type::number:
+    write_double(element.get_double());
+    break;
+  case simdjson::ondemand::json_type::string:
+    write_raw_string(element.get_raw_json_string());
+    break;
+  case simdjson::ondemand::json_type::boolean:
+    write_byte(0xc2 + element.get_bool());
+    break;
+  case simdjson::ondemand::json_type::null:
+    write_byte(0xc0);
+    break;
+  default:
+    SIMDJSON_UNREACHABLE();
+  }
+}
+
+struct simdjson_ondemand {
+  using StringType = std::string_view;
+
+  simdjson2msgpack parser{};
+
+  bool run(simdjson::padded_string &json, char *buffer,
+           std::string_view &result) {
+    result = parser.to_msgpack(json, reinterpret_cast<uint8_t *>(buffer));
+    return true;
+  }
+};
+
+BENCHMARK_TEMPLATE(json2msgpack, simdjson_ondemand)->UseManualTime();
+
+} // namespace json2msgpack
+
+#endif // SIMDJSON_EXCEPTIONS

benchmark/json2msgpack/yyjson.h

@@ -0,0 +1,122 @@
+#pragma once
+#ifdef SIMDJSON_COMPETITION_YYJSON
+
+#include "json2msgpack.h"
+
+namespace json2msgpack {
+
+struct yyjson2msgpack {
+  inline std::string_view to_msgpack(yyjson_doc *doc, uint8_t *buf);
+
+private:
+  inline void write_double(const double d) noexcept;
+  inline void write_byte(const uint8_t b) noexcept;
+  inline void write_uint32(const uint32_t w) noexcept;
+  inline void write_string(const char *s, size_t length) noexcept;
+  inline void recursive_processor(yyjson_val *obj);
+
+  uint8_t *buff{};
+};
+
+std::string_view yyjson2msgpack::to_msgpack(yyjson_doc *doc, uint8_t *buf) {
+  buff = buf;
+  yyjson_val *root = yyjson_doc_get_root(doc);
+  recursive_processor(root);
+  return std::string_view(reinterpret_cast<char *>(buf), size_t(buff - buf));
+}
+
+void yyjson2msgpack::write_string(const char *c, size_t len) noexcept {
+  write_byte(0xdb);
+  write_uint32(uint32_t(len));
+  ::memcpy(buff, c, len);
+  buff += len;
+}
+
+void yyjson2msgpack::write_double(const double d) noexcept {
+  *buff++ = 0xcb;
+  ::memcpy(buff, &d, sizeof(d));
+  buff += sizeof(d);
+}
+
+void yyjson2msgpack::write_byte(const uint8_t b) noexcept {
+  *buff = b;
+  buff++;
+}
+
+void yyjson2msgpack::write_uint32(const uint32_t w) noexcept {
+  ::memcpy(buff, &w, sizeof(w));
+  buff += sizeof(w);
+}
+
+void yyjson2msgpack::recursive_processor(yyjson_val *obj) {
+  size_t idx, max;
+  yyjson_val *val;
+  yyjson_val *key;
+  switch (yyjson_get_type(obj)) {
+  case YYJSON_TYPE_STR:
+    write_string(yyjson_get_str(obj), yyjson_get_len(obj));
+    break;
+  case YYJSON_TYPE_ARR:
+    write_byte(0xdf);
+    write_uint32(uint32_t(yyjson_arr_size(obj)));
+    yyjson_arr_foreach(obj, idx, max, val) { recursive_processor(val); }
+   break;
+  case YYJSON_TYPE_OBJ:
+    write_byte(0xdd);
+    write_uint32(uint32_t(yyjson_obj_size(obj)));
+    yyjson_obj_foreach(obj, idx, max, key, val) {
+      write_string(yyjson_get_str(key), yyjson_get_len(key));
+      recursive_processor(val);
+    }
+    break;
+  case YYJSON_TYPE_BOOL:
+    write_byte(0xc2 + yyjson_get_bool(obj));
+    break;
+  case YYJSON_TYPE_NULL:
+    write_byte(0xc0);
+    break;
+  case YYJSON_TYPE_NUM:
+    switch (yyjson_get_subtype(obj)) {
+    case YYJSON_SUBTYPE_UINT:
+      write_double(double(yyjson_get_uint(obj)));
+      break;
+    case YYJSON_SUBTYPE_SINT:
+      write_double(double(yyjson_get_sint(obj)));
+      break;
+    case YYJSON_SUBTYPE_REAL:
+      write_double(yyjson_get_real(obj));
+      break;
+    default:
+      SIMDJSON_UNREACHABLE();
+    }
+    break;
+  default:
+    SIMDJSON_UNREACHABLE();
+  }
+}
+
+struct yyjson : yyjson2msgpack {
+  bool run(simdjson::padded_string &json, char *buffer,
+           std::string_view &result) {
+    yyjson_doc *doc = yyjson_read(json.data(), json.size(), 0);
+    result = to_msgpack(doc, reinterpret_cast<uint8_t*>(buffer));
+    return true;
+  }
+};
+
+BENCHMARK_TEMPLATE(json2msgpack, yyjson)->UseManualTime();
+
+struct yyjson_insitu : yyjson2msgpack {
+  bool run(simdjson::padded_string &json, char *buffer,
+           std::string_view &result) {
+    yyjson_doc *doc =
+        yyjson_read_opts(json.data(), json.size(), YYJSON_READ_INSITU, 0, 0);
+    result = to_msgpack(doc, reinterpret_cast<uint8_t*>(buffer));
+    return true;
+  }
+};
+BENCHMARK_TEMPLATE(json2msgpack, yyjson_insitu)->UseManualTime();
+
+} // namespace json2msgpack
+
+#endif // SIMDJSON_COMPETITION_YYJSON

benchmark/json_benchmark/run_json_benchmark.h

@@ -10,8 +10,8 @@ void maybe_display_implementation() {
   static bool displayed_implementation = false;
   if(!displayed_implementation) {
     displayed_implementation = true;
-    std::cout << "simdjson::dom implementation:      " << simdjson::active_implementation->name() << std::endl;
-    std::cout << "simdjson::ondemand implementation (stage 1): " << simdjson::active_implementation->name() << std::endl;
+    std::cout << "simdjson::dom implementation:      " << simdjson::get_active_implementation()->name() << std::endl;
+    std::cout << "simdjson::ondemand implementation (stage 1): " << simdjson::get_active_implementation()->name() << std::endl;
     std::cout << "simdjson::ondemand implementation (stage 2): " << simdjson::builtin_implementation()->name() << std::endl;
   }
 }
@@ -58,10 +58,11 @@ template<typename B, typename R> static void run_json_benchmark(benchmark::State
   if (collector.has_events()) {
     state.counters["instructions"] = events.instructions();
     state.counters["cycles"]       = events.cycles();
+#if !SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
     state.counters["branch_miss"]  = events.branch_misses();
     state.counters["cache_miss"]   = events.cache_misses();
     state.counters["cache_ref"]    = events.cache_references();
-
+#endif
     state.counters["instructions_per_byte"]  = events.instructions() / double(bench.bytes_per_iteration());
     state.counters["instructions_per_cycle"] = events.instructions() / events.cycles();
     state.counters["cycles_per_byte"]        = events.cycles()       / double(bench.bytes_per_iteration());
@@ -69,9 +70,11 @@ template<typename B, typename R> static void run_json_benchmark(benchmark::State
 
     state.counters["best_instructions"] = events.best.instructions();
     state.counters["best_cycles"]       = events.best.cycles();
+#if !SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
     state.counters["best_branch_miss"]  = events.best.branch_misses();
     state.counters["best_cache_miss"]   = events.best.cache_misses();
     state.counters["best_cache_ref"]    = events.best.cache_references();
+#endif
 
     state.counters["best_instructions_per_byte"]  = events.best.instructions() / double(bench.bytes_per_iteration());
     state.counters["best_instructions_per_cycle"] = events.best.instructions() / events.best.cycles();
@@ -92,9 +95,11 @@ template<typename B, typename R> static void run_json_benchmark(benchmark::State
   if (collector.has_events()) {
     label << " instructions=" << setw(12) << uint64_t(events.best.instructions())     << setw(0);
     label << " cycles="       << setw(12) << uint64_t(events.best.cycles())           << setw(0);
+#if !SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
     label << " branch_miss="  << setw(8)  << uint64_t(events.best.branch_misses())    << setw(0);
     label << " cache_miss="   << setw(8)  << uint64_t(events.best.cache_misses())     << setw(0);
     label << " cache_ref="    << setw(10) << uint64_t(events.best.cache_references()) << setw(0);
+#endif
   }
 
   label << " items=" << setw(10) << bench.items_per_iteration() << setw(0);

benchmark/kostya/kostya.h

@@ -79,7 +79,7 @@ static const simdjson::padded_string &get_built_json_array() {
 
 struct simdjson_dom;
 
-template<typename I> simdjson_really_inline static void kostya(benchmark::State &state) {
+template<typename I> simdjson_inline static void kostya(benchmark::State &state) {
   run_json_benchmark<runner<I>, runner<simdjson_dom>>(state);
 }
 

benchmark/kostya/nlohmann_json_sax.h

@@ -42,7 +42,7 @@ struct nlohmann_json_sax {
             return true;
         }
         bool number_unsigned(number_unsigned_t val) override {  // Need this event because coordinate value can be equal to 1
-            buffer[k] = val;
+            buffer[k] = double(val);
             if (k == 2) {
                 result.emplace_back(json_benchmark::point{buffer[0],buffer[1],buffer[2]});
                 k = 0;

benchmark/kostya/rapidjson.h

@@ -13,7 +13,7 @@ struct rapidjson_base {
 
   Document doc;
 
-  simdjson_really_inline double get_double(Value &object, std::string_view key) {
+  simdjson_inline double get_double(Value &object, std::string_view key) {
     auto field = object.FindMember(key.data());
     if (field == object.MemberEnd()) { throw "Missing double field"; }
     if (!field->value.IsNumber()) { throw "Field is not double"; }
@@ -51,7 +51,7 @@ BENCHMARK_TEMPLATE(kostya, rapidjson_lossless)->UseManualTime();
 
 struct rapidjson_insitu : rapidjson_base {
   bool run(simdjson::padded_string &json, std::vector<point> &result) {
-    return rapidjson_base::run(doc.ParseInsitu<kParseValidateEncodingFlag>(json.data()), result);
+    return rapidjson_base::run(doc.ParseInsitu<kParseValidateEncodingFlag|kParseInsituFlag>(json.data()), result);
   }
 };
 BENCHMARK_TEMPLATE(kostya, rapidjson_insitu)->UseManualTime();

benchmark/kostya/sajson.h

@@ -11,8 +11,9 @@ struct sajson {
 
   size_t ast_buffer_size{0};
   size_t *ast_buffer{nullptr};
+  ~sajson() { free(ast_buffer); }
 
-  simdjson_really_inline double get_double(const ::sajson::value &obj, std::string_view key) {
+  simdjson_inline double get_double(const ::sajson::value &obj, std::string_view key) {
     using namespace sajson;
 
     auto val = obj.get_value_of_key({key.data(), key.length()});

benchmark/kostya/yyjson.h

@@ -9,7 +9,7 @@ namespace kostya {
 struct yyjson_base {
   static constexpr diff_flags DiffFlags = diff_flags::NONE;
 
-  simdjson_really_inline double get_double(yyjson_val *obj, std::string_view key) {
+  simdjson_inline double get_double(yyjson_val *obj, std::string_view key) {
     yyjson_val *val = yyjson_obj_getn(obj, key.data(), key.length());
     if (!val) { throw "missing point field!"; }
     if (yyjson_get_type(val) != YYJSON_TYPE_NUM) { throw "Number is not a type!"; }

benchmark/large_amazon_cellphones/large_amazon_cellphones.h

@@ -6,6 +6,9 @@
 
 namespace large_amazon_cellphones {
 
+const bool UNTHREADED = false;
+const bool THREADED = true;
+
 static const simdjson::padded_string &get_built_json();
 
 using namespace json_benchmark;
@@ -13,11 +16,11 @@ using namespace json_benchmark;
 struct brand {
     double cumulative_rating;
     uint64_t reviews_count;
-    simdjson_really_inline bool operator==(const brand &other) const {
+    simdjson_inline bool operator==(const brand &other) const {
         return cumulative_rating == other.cumulative_rating &&
             reviews_count == other.reviews_count;
     }
-    simdjson_really_inline bool operator!=(const brand &other) const { return !(*this == other); }
+    simdjson_inline bool operator!=(const brand &other) const { return !(*this == other); }
 };
 
 simdjson_unused static std::ostream &operator<<(std::ostream &o, const brand &b) {
@@ -81,11 +84,11 @@ static const simdjson::padded_string &get_built_json() {
   return json;
 }
 
-
+template<bool threaded>
 struct simdjson_dom;
 
-template<typename I> simdjson_really_inline static void large_amazon_cellphones(benchmark::State &state) {
-  run_json_benchmark<runner<I>, runner<simdjson_dom>>(state);
+template<typename I> simdjson_inline static void large_amazon_cellphones(benchmark::State &state) {
+  run_json_benchmark<runner<I>, runner<simdjson_dom<UNTHREADED>>>(state);
 }
 
-}   // namespace large_amazon_cellphones
+}   // namespace large_amazon_cellphones

benchmark/large_amazon_cellphones/simdjson_dom.h

@@ -9,12 +9,16 @@ namespace large_amazon_cellphones {
 
 using namespace simdjson;
 
+template<bool threaded>
 struct simdjson_dom {
   using StringType = std::string;
 
   dom::parser parser{};
 
   bool run(simdjson::padded_string &json, std::map<StringType, brand> &result) {
+#ifdef SIMDJSON_THREADS_ENABLED
+    parser.threaded = threaded;
+#endif
     auto stream = parser.parse_many(json);
     auto i = stream.begin();
     ++i;  // Skip first line
@@ -38,7 +42,10 @@ struct simdjson_dom {
 
 };
 
-BENCHMARK_TEMPLATE(large_amazon_cellphones, simdjson_dom)->UseManualTime();
+BENCHMARK_TEMPLATE(large_amazon_cellphones, simdjson_dom<UNTHREADED>)->UseManualTime();
+#ifdef SIMDJSON_THREADS_ENABLED
+BENCHMARK_TEMPLATE(large_amazon_cellphones, simdjson_dom<THREADED>)->UseManualTime();
+#endif
 
 } // namespace large_amazon_cellphones
 

benchmark/large_amazon_cellphones/simdjson_ondemand.h

@@ -8,12 +8,16 @@ namespace large_amazon_cellphones {
 
 using namespace simdjson;
 
+template<bool threaded>
 struct simdjson_ondemand {
   using StringType = std::string;
 
   ondemand::parser parser{};
 
   bool run(simdjson::padded_string &json, std::map<StringType, brand> &result) {
+#ifdef SIMDJSON_THREADS_ENABLED
+    parser.threaded = threaded;
+#endif
     ondemand::document_stream stream = parser.iterate_many(json);
     ondemand::document_stream::iterator i = stream.begin();
     ++i;  // Skip first line
@@ -58,7 +62,10 @@ struct simdjson_ondemand {
 
 };
 
-BENCHMARK_TEMPLATE(large_amazon_cellphones, simdjson_ondemand)->UseManualTime();
+BENCHMARK_TEMPLATE(large_amazon_cellphones, simdjson_ondemand<UNTHREADED>)->UseManualTime();
+#ifdef SIMDJSON_THREADS_ENABLED
+BENCHMARK_TEMPLATE(large_amazon_cellphones, simdjson_ondemand<THREADED>)->UseManualTime();
+#endif
 
 } // namespace amazon_cellphones
 

benchmark/large_random/nlohmann_json_sax.h

@@ -34,7 +34,7 @@ struct nlohmann_json_sax {
             return true;
         }
         bool number_unsigned(number_unsigned_t val) override {
-            buffer[k] = val;
+            buffer[k] = double(val);
             if (k == 2) {
                 result.emplace_back(json_benchmark::point{buffer[0],buffer[1],buffer[2]});
                 k = 0;

benchmark/large_random/rapidjson.h

@@ -13,7 +13,7 @@ struct rapidjson_base {
 
   Document doc;
 
-  simdjson_really_inline double get_double(Value &object, std::string_view key) {
+  simdjson_inline double get_double(Value &object, std::string_view key) {
     auto field = object.FindMember(key.data());
     if (field == object.MemberEnd()) { throw "Missing double field"; }
     if (!field->value.IsNumber()) { throw "Field is not double"; }
@@ -48,7 +48,7 @@ BENCHMARK_TEMPLATE(large_random, rapidjson_lossless)->UseManualTime();
 
 struct rapidjson_insitu : rapidjson_base {
   bool run(simdjson::padded_string &json, std::vector<point> &result) {
-    return rapidjson_base::run(doc.ParseInsitu<kParseValidateEncodingFlag>(json.data()), result);
+    return rapidjson_base::run(doc.ParseInsitu<kParseValidateEncodingFlag|kParseInsituFlag>(json.data()), result);
   }
 };
 BENCHMARK_TEMPLATE(large_random, rapidjson_insitu)->UseManualTime();

benchmark/large_random/sajson.h

@@ -11,8 +11,9 @@ struct sajson {
 
   size_t ast_buffer_size{0};
   size_t *ast_buffer{nullptr};
+  ~sajson() { free(ast_buffer); }
 
-  simdjson_really_inline double get_double(const ::sajson::value &obj, std::string_view key) {
+  simdjson_inline double get_double(const ::sajson::value &obj, std::string_view key) {
     using namespace sajson;
 
     auto val = obj.get_value_of_key({key.data(), key.length()});

benchmark/large_random/yyjson.h

@@ -9,7 +9,7 @@ namespace large_random {
 struct yyjson_base {
   static constexpr diff_flags DiffFlags = diff_flags::NONE;
 
-  simdjson_really_inline double get_double(yyjson_val *obj, std::string_view key) {
+  simdjson_inline double get_double(yyjson_val *obj, std::string_view key) {
     yyjson_val *val = yyjson_obj_getn(obj, key.data(), key.length());
     if (!val) { throw "missing point field!"; }
     if (yyjson_get_type(val) != YYJSON_TYPE_NUM) { throw "Number is not a type!"; }

benchmark/largerandom/iter.h

@@ -10,28 +10,28 @@ using namespace simdjson;
 
 class Iter {
 public:
-  simdjson_really_inline bool Run(const padded_string &json);
+  simdjson_inline bool Run(const padded_string &json);
 
-  simdjson_really_inline const std::vector<my_point> &Result() { return container; }
-  simdjson_really_inline size_t ItemCount() { return container.size(); }
+  simdjson_inline const std::vector<my_point> &Result() { return container; }
+  simdjson_inline size_t ItemCount() { return container.size(); }
 
 private:
   ondemand::parser parser{};
   std::vector<my_point> container{};
 
-  simdjson_really_inline double first_double(ondemand::json_iterator &iter) {
+  simdjson_inline double first_double(ondemand::json_iterator &iter) {
     if (iter.start_object().error() || iter.field_key().error() || iter.field_value()) { throw "Invalid field"; }
     return iter.consume_double();
   }
 
-  simdjson_really_inline double next_double(ondemand::json_iterator &iter) {
+  simdjson_inline double next_double(ondemand::json_iterator &iter) {
     if (!iter.has_next_field() || iter.field_key().error() || iter.field_value()) { throw "Invalid field"; }
     return iter.consume_double();
   }
 
 };
 
-simdjson_really_inline bool Iter::Run(const padded_string &json) {
+simdjson_inline bool Iter::Run(const padded_string &json) {
   container.clear();
 
   auto iter = parser.iterate_raw(json).value();

benchmark/largerandom/ondemand.h

@@ -10,16 +10,16 @@ using namespace simdjson;
 
 class OnDemand {
 public:
-  simdjson_really_inline bool Run(const padded_string &json);
-  simdjson_really_inline const std::vector<my_point> &Result() { return container; }
-  simdjson_really_inline size_t ItemCount() { return container.size(); }
+  simdjson_inline bool Run(const padded_string &json);
+  simdjson_inline const std::vector<my_point> &Result() { return container; }
+  simdjson_inline size_t ItemCount() { return container.size(); }
 
 private:
   ondemand::parser parser{};
   std::vector<my_point> container{};
 };
 
-simdjson_really_inline bool OnDemand::Run(const padded_string &json) {
+simdjson_inline bool OnDemand::Run(const padded_string &json) {
   container.clear();
 
   auto doc = parser.iterate(json);

benchmark/largerandom/sax.h

@@ -11,13 +11,13 @@ using namespace simdjson::builtin::stage2;
 
 class Sax {
 public:
-  simdjson_really_inline bool Run(const padded_string &json) noexcept;
+  simdjson_inline bool Run(const padded_string &json) noexcept;
 
-  simdjson_really_inline const std::vector<my_point> &Result() { return container; }
-  simdjson_really_inline size_t ItemCount() { return container.size(); }
+  simdjson_inline const std::vector<my_point> &Result() { return container; }
+  simdjson_inline size_t ItemCount() { return container.size(); }
 
 private:
-  simdjson_really_inline error_code RunNoExcept(const padded_string &json) noexcept;
+  simdjson_inline error_code RunNoExcept(const padded_string &json) noexcept;
   error_code Allocate(size_t new_capacity);
   std::unique_ptr<uint8_t[]> string_buf{};
   size_t capacity{};
@@ -34,21 +34,21 @@ public:
 
   explicit sax_point_reader_visitor(std::vector<my_point> &_points) : points(_points) {}
 
-  simdjson_really_inline error_code visit_object_start(json_iterator &) {
+  simdjson_inline error_code visit_object_start(json_iterator &) {
     idx = 0;
     return SUCCESS;
   }
-  simdjson_really_inline error_code visit_primitive(json_iterator &, const uint8_t *value) {
+  simdjson_inline error_code visit_primitive(json_iterator &, const uint8_t *value) {
     if(idx == GOT_SOMETHING_ELSE) { return simdjson::SUCCESS; }
     return numberparsing::parse_double(value).get(buffer[idx]);
   }
-  simdjson_really_inline error_code visit_object_end(json_iterator &)  {
+  simdjson_inline error_code visit_object_end(json_iterator &)  {
     points.emplace_back(my_point{buffer[0], buffer[1], buffer[2]});
     return SUCCESS;
   }
 
-  simdjson_really_inline error_code visit_document_start(json_iterator &) { return SUCCESS; }
-  simdjson_really_inline error_code visit_key(json_iterator &, const uint8_t * key) {
+  simdjson_inline error_code visit_document_start(json_iterator &) { return SUCCESS; }
+  simdjson_inline error_code visit_key(json_iterator &, const uint8_t * key) {
     switch(key[1]) {
       // Technically, we should check the other characters
       // in the key, but we are cheating to go as fast
@@ -67,13 +67,13 @@ public:
     }
     return SUCCESS;
   }
-  simdjson_really_inline error_code visit_array_start(json_iterator &)  { return SUCCESS; }
-  simdjson_really_inline error_code visit_array_end(json_iterator &) { return SUCCESS; }
-  simdjson_really_inline error_code visit_document_end(json_iterator &)  { return SUCCESS; }
-  simdjson_really_inline error_code visit_empty_array(json_iterator &)  { return SUCCESS; }
-  simdjson_really_inline error_code visit_empty_object(json_iterator &)  { return SUCCESS; }
-  simdjson_really_inline error_code visit_root_primitive(json_iterator &, const uint8_t *)  { return SUCCESS; }
-  simdjson_really_inline error_code increment_count(json_iterator &) { return SUCCESS; }
+  simdjson_inline error_code visit_array_start(json_iterator &)  { return SUCCESS; }
+  simdjson_inline error_code visit_array_end(json_iterator &) { return SUCCESS; }
+  simdjson_inline error_code visit_document_end(json_iterator &)  { return SUCCESS; }
+  simdjson_inline error_code visit_empty_array(json_iterator &)  { return SUCCESS; }
+  simdjson_inline error_code visit_empty_object(json_iterator &)  { return SUCCESS; }
+  simdjson_inline error_code visit_root_primitive(json_iterator &, const uint8_t *)  { return SUCCESS; }
+  simdjson_inline error_code increment_count(json_iterator &) { return SUCCESS; }
 };
 
 // NOTE: this assumes the dom_parser is already allocated

benchmark/partial_tweets/nlohmann_json.h

@@ -9,7 +9,7 @@ namespace partial_tweets {
 struct nlohmann_json {
   using StringType=std::string;
 
-  simdjson_really_inline uint64_t nullable_int(nlohmann::json value) {
+  simdjson_inline uint64_t nullable_int(nlohmann::json value) {
     if (value.is_null()) { return 0; }
     return value;
   }
@@ -37,4 +37,4 @@ BENCHMARK_TEMPLATE(partial_tweets, nlohmann_json)->UseManualTime();
 
 } // namespace partial_tweets
 
-#endif // SIMDJSON_COMPETITION_NLOHMANN_JSON
+#endif // SIMDJSON_COMPETITION_NLOHMANN_JSON

benchmark/partial_tweets/ondemand.h

@@ -17,27 +17,27 @@ public:
       displayed_implementation = true;
     }
   }
-  simdjson_really_inline bool Run(const padded_string &json);
-  simdjson_really_inline const std::vector<tweet> &Result() { return tweets; }
-  simdjson_really_inline size_t ItemCount() { return tweets.size(); }
+  simdjson_inline bool Run(const padded_string &json);
+  simdjson_inline const std::vector<tweet> &Result() { return tweets; }
+  simdjson_inline size_t ItemCount() { return tweets.size(); }
 
 private:
   ondemand::parser parser{};
   std::vector<tweet> tweets{};
 
-  simdjson_really_inline uint64_t nullable_int(ondemand::value value) {
+  simdjson_inline uint64_t nullable_int(ondemand::value value) {
     if (value.is_null()) { return 0; }
     return value;
   }
 
-  simdjson_really_inline twitter_user read_user(ondemand::object user) {
+  simdjson_inline twitter_user read_user(ondemand::object user) {
     return { user.find_field("id"), user.find_field("screen_name") };
   }
 
   static inline bool displayed_implementation = false;
 };
 
-simdjson_really_inline bool OnDemand::Run(const padded_string &json) {
+simdjson_inline bool OnDemand::Run(const padded_string &json) {
   tweets.clear();
 
   // Walk the document, parsing the tweets as we go

benchmark/partial_tweets/partial_tweets.h

@@ -39,7 +39,7 @@ struct runner : public file_runner<I> {
 
 struct simdjson_dom;
 
-template<typename I> simdjson_really_inline static void partial_tweets(benchmark::State &state) {
+template<typename I> simdjson_inline static void partial_tweets(benchmark::State &state) {
   run_json_benchmark<runner<I>, runner<simdjson_dom>>(state);
 }
 

benchmark/partial_tweets/rapidjson.h

@@ -13,27 +13,27 @@ struct rapidjson_base {
 
   Document doc{};
 
-  simdjson_really_inline std::string_view get_string_view(Value &object, std::string_view key) {
+  simdjson_inline std::string_view get_string_view(Value &object, std::string_view key) {
     // TODO use version that supports passing string length?
     auto field = object.FindMember(key.data());
     if (field == object.MemberEnd()) { throw "Missing object field"; }
     if (!field->value.IsString()) { throw "Field is not a string"; }
     return { field->value.GetString(), field->value.GetStringLength() };
   }
-  simdjson_really_inline uint64_t get_uint64(Value &object, std::string_view key) {
+  simdjson_inline uint64_t get_uint64(Value &object, std::string_view key) {
     auto field = object.FindMember(key.data());
     if (field == object.MemberEnd()) { throw "Missing object field"; }
     if (!field->value.IsUint64()) { throw "Field is not uint64"; }
     return field->value.GetUint64();
   }
-  simdjson_really_inline uint64_t get_nullable_uint64(Value &object, std::string_view key) {
+  simdjson_inline uint64_t get_nullable_uint64(Value &object, std::string_view key) {
     auto field = object.FindMember(key.data());
     if (field == object.MemberEnd()) { throw "Missing nullable uint64 field"; }
     if (field->value.IsNull()) { return 0; }
     if (!field->value.IsUint64()) { throw "Field is not nullable uint64"; }
     return field->value.GetUint64();
   }
-  simdjson_really_inline partial_tweets::twitter_user<std::string_view> get_user(Value &object, std::string_view key) {
+  simdjson_inline partial_tweets::twitter_user<std::string_view> get_user(Value &object, std::string_view key) {
     auto field = object.FindMember(key.data());
     if (field == object.MemberEnd()) { throw "Missing user field"; }
     if (!field->value.IsObject()) { throw "User field is not an object"; }
@@ -70,7 +70,7 @@ BENCHMARK_TEMPLATE(partial_tweets, rapidjson)->UseManualTime();
 
 struct rapidjson_insitu : rapidjson_base {
   bool run(simdjson::padded_string &json, std::vector<tweet<std::string_view>> &result) {
-    return rapidjson_base::run(doc.ParseInsitu<kParseValidateEncodingFlag>(json.data()), result);
+    return rapidjson_base::run(doc.ParseInsitu<kParseValidateEncodingFlag|kParseInsituFlag>(json.data()), result);
   }
 };
 BENCHMARK_TEMPLATE(partial_tweets, rapidjson_insitu)->UseManualTime();

benchmark/partial_tweets/sajson.h

@@ -11,12 +11,14 @@ struct sajson {
 
   size_t ast_buffer_size{0};
   size_t *ast_buffer{nullptr};
-  simdjson_really_inline std::string_view get_string_view(const ::sajson::value &obj, std::string_view key) {
+  ~sajson() { free(ast_buffer); }
+
+  simdjson_inline std::string_view get_string_view(const ::sajson::value &obj, std::string_view key) {
     auto val = obj.get_value_of_key({key.data(), key.length()});
     if (val.get_type() != ::sajson::TYPE_STRING) { throw "field is not a string"; }
     return { val.as_cstring(), val.get_string_length() };
   }
-  simdjson_really_inline uint64_t get_uint52(const ::sajson::value &obj, std::string_view key) {
+  simdjson_inline uint64_t get_uint52(const ::sajson::value &obj, std::string_view key) {
     auto val = obj.get_value_of_key({key.data(), key.length()});
     switch (val.get_type()) {
       case ::sajson::TYPE_INTEGER: {
@@ -28,7 +30,7 @@ struct sajson {
         throw "field not integer";
     }
   }
-  simdjson_really_inline uint64_t get_str_uint64(const ::sajson::value &obj, std::string_view key) {
+  simdjson_inline uint64_t get_str_uint64(const ::sajson::value &obj, std::string_view key) {
     // Since sajson only supports 53-bit numbers, and IDs in twitter.json can be > 53 bits, we read the corresponding id_str and parse that.
     auto val = obj.get_value_of_key({key.data(), key.length()});
     if (val.get_type() != ::sajson::TYPE_STRING) { throw "field not a string"; }
@@ -38,7 +40,7 @@ struct sajson {
     if (endptr != &str[val.get_string_length()]) { throw "field is a string, but not an integer string"; }
     return result;
   }
-  simdjson_really_inline uint64_t get_nullable_str_uint64(const ::sajson::value &obj, std::string_view key) {
+  simdjson_inline uint64_t get_nullable_str_uint64(const ::sajson::value &obj, std::string_view key) {
     auto val = obj.get_value_of_key({key.data(), key.length()});
     if (val.get_type() == ::sajson::TYPE_NULL) { return 0; }
     if (val.get_type() != ::sajson::TYPE_STRING) { throw "field not a string"; }
@@ -48,7 +50,7 @@ struct sajson {
     if (endptr != &str[val.get_string_length()]) { throw "field is a string, but not an integer string"; }
     return result;
   }
-  simdjson_really_inline partial_tweets::twitter_user<std::string_view> get_user(const ::sajson::value &obj, std::string_view key) {
+  simdjson_inline partial_tweets::twitter_user<std::string_view> get_user(const ::sajson::value &obj, std::string_view key) {
     auto user = obj.get_value_of_key({key.data(), key.length()});
     if (user.get_type() != ::sajson::TYPE_OBJECT) { throw "user is not an object"; }
     return { get_str_uint64(user, "id_str"), get_string_view(user, "screen_name") };

benchmark/partial_tweets/simdjson_dom.h

@@ -13,7 +13,7 @@ struct simdjson_dom {
 
   dom::parser parser{};
 
-  simdjson_really_inline uint64_t nullable_int(dom::element element) {
+  simdjson_inline uint64_t nullable_int(dom::element element) {
     if (element.is_null()) { return 0; }
     return element;
   }
@@ -40,4 +40,4 @@ BENCHMARK_TEMPLATE(partial_tweets, simdjson_dom)->UseManualTime();
 
 } // namespace partial_tweets
 
-#endif // SIMDJSON_EXCEPTIONS
+#endif // SIMDJSON_EXCEPTIONS

benchmark/partial_tweets/simdjson_ondemand.h

@@ -13,12 +13,12 @@ struct simdjson_ondemand {
 
   ondemand::parser parser{};
 
-  simdjson_really_inline uint64_t nullable_int(ondemand::value value) {
+  simdjson_inline uint64_t nullable_int(ondemand::value value) {
     if (value.is_null()) { return 0; }
     return value;
   }
 
-  simdjson_really_inline twitter_user<std::string_view> read_user(ondemand::object user) {
+  simdjson_inline twitter_user<std::string_view> read_user(ondemand::object user) {
     return { user.find_field("id"), user.find_field("screen_name") };
   }
 

benchmark/partial_tweets/tweet.h

@@ -32,7 +32,7 @@ struct tweet {
   uint64_t retweet_count{};
   uint64_t favorite_count{};
   template<typename OtherStringType>
-  simdjson_really_inline bool operator==(const tweet<OtherStringType> &other) const {
+  simdjson_inline bool operator==(const tweet<OtherStringType> &other) const {
     return created_at == other.created_at &&
            id == other.id &&
            result == other.result &&
@@ -42,7 +42,7 @@ struct tweet {
            favorite_count == other.favorite_count;
   }
   template<typename OtherStringType>
-  simdjson_really_inline bool operator!=(const tweet<OtherStringType> &other) const { return !(*this == other); }
+  simdjson_inline bool operator!=(const tweet<OtherStringType> &other) const { return !(*this == other); }
 };
 
 template<typename StringType>

benchmark/partial_tweets/yyjson.h

@@ -9,24 +9,24 @@ namespace partial_tweets {
 struct yyjson_base {
   using StringType=std::string_view;
 
-  simdjson_really_inline std::string_view get_string_view(yyjson_val *obj, std::string_view key) {
+  simdjson_inline std::string_view get_string_view(yyjson_val *obj, std::string_view key) {
     auto val = yyjson_obj_getn(obj, key.data(), key.length());
     if (!yyjson_is_str(val)) { throw "field is not uint64 or null!"; }
     return { yyjson_get_str(val), yyjson_get_len(val) };
   }
-  simdjson_really_inline uint64_t get_uint64(yyjson_val *obj, std::string_view key) {
+  simdjson_inline uint64_t get_uint64(yyjson_val *obj, std::string_view key) {
     auto val = yyjson_obj_getn(obj, key.data(), key.length());
     if (!yyjson_is_uint(val)) { throw "field is not uint64 or null!"; }
     return yyjson_get_uint(val);
   }
-  simdjson_really_inline uint64_t get_nullable_uint64(yyjson_val *obj, std::string_view key) {
+  simdjson_inline uint64_t get_nullable_uint64(yyjson_val *obj, std::string_view key) {
     auto val = yyjson_obj_getn(obj, key.data(), key.length());
     if (!yyjson_is_uint(val)) { }
     auto type = yyjson_get_type(val);
     if (type != YYJSON_TYPE_NUM && type != YYJSON_TYPE_NULL ) { throw "field is not uint64 or null!"; }
     return yyjson_get_uint(val);
   }
-  simdjson_really_inline partial_tweets::twitter_user<std::string_view> get_user(yyjson_val *obj, std::string_view key) {
+  simdjson_inline partial_tweets::twitter_user<std::string_view> get_user(yyjson_val *obj, std::string_view key) {
     auto user = yyjson_obj_getn(obj, key.data(), key.length());
     if (!yyjson_is_obj(user)) { throw "missing twitter user field!"; }
     return { get_uint64(user, "id"), get_string_view(user, "screen_name") };

benchmark/top_tweet/nlohmann_json_sax.h

@@ -48,7 +48,7 @@ struct nlohmann_json_sax {
         }
         bool number_unsigned(number_unsigned_t val) override {
             if (values & key_rt && !(values & found_rt)) {   // retweet_count
-                rt = val;
+                rt = int(val);
                 values &= ~(key_rt);
                 values |= (found_rt);
                 if (rt <= max_rt && rt >= result.retweet_count) {    // Check if current tweet has more retweet than previous top tweet

benchmark/top_tweet/rapidjson.h

@@ -59,7 +59,7 @@ BENCHMARK_TEMPLATE(top_tweet, rapidjson)->UseManualTime();
 
 struct rapidjson_insitu : rapidjson_base {
   bool run(simdjson::padded_string &json, int64_t max_retweet_count, top_tweet_result<StringType> &result) {
-    return rapidjson_base::run(doc.ParseInsitu<kParseValidateEncodingFlag>(json.data()), max_retweet_count, result);
+    return rapidjson_base::run(doc.ParseInsitu<kParseValidateEncodingFlag|kParseInsituFlag>(json.data()), max_retweet_count, result);
   }
 };
 BENCHMARK_TEMPLATE(top_tweet, rapidjson_insitu)->UseManualTime();

benchmark/top_tweet/sajson.h

@@ -11,6 +11,7 @@ struct sajson {
 
   size_t ast_buffer_size{0};
   size_t *ast_buffer{nullptr};
+  ~sajson() { free(ast_buffer); }
 
   bool run(simdjson::padded_string &json, int32_t max_retweet_count, top_tweet_result<StringType> &result) {
     if (!ast_buffer) {

benchmark/top_tweet/top_tweet.h

@@ -13,13 +13,13 @@ struct top_tweet_result {
   StringType screen_name{};
   StringType text{};
   template<typename OtherStringType>
-  simdjson_really_inline bool operator==(const top_tweet_result<OtherStringType> &other) const {
+  simdjson_inline bool operator==(const top_tweet_result<OtherStringType> &other) const {
     return retweet_count == other.retweet_count &&
            screen_name == other.screen_name &&
            text == other.text;
   }
   template<typename OtherStringType>
-  simdjson_really_inline bool operator!=(const top_tweet_result<OtherStringType> &other) const { return !(*this == other); }
+  simdjson_inline bool operator!=(const top_tweet_result<OtherStringType> &other) const { return !(*this == other); }
 };
 
 template<typename StringType>
@@ -60,7 +60,7 @@ struct runner : public file_runner<I> {
 
 struct simdjson_dom;
 
-template<typename I> simdjson_really_inline static void top_tweet(benchmark::State &state) {
+template<typename I> simdjson_inline static void top_tweet(benchmark::State &state) {
   json_benchmark::run_json_benchmark<runner<I>, runner<simdjson_dom>>(state);
 }
 

cmake/developer-options.cmake

@@ -175,6 +175,18 @@ if(CMAKE_C_COMPILER_ID MATCHES "Intel")
   set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -static-intel")
 endif()
 
+option(
+    SIMDJSON_AVX512_ALLOWED
+    "Enable AVX-512 instructions (only affects processors and compilers with AVX-512 support)."
+    ON
+)
+if(SIMDJSON_AVX512_ALLOWED)
+  add_compile_definitions(SIMDJSON_AVX512_ALLOWED=1)
+else()
+  add_compile_definitions(SIMDJSON_AVX512_ALLOWED=0)
+  message(STATUS "AVX-512 instructions are not allowed.")
+endif()
+
 include(CheckSymbolExists)
 check_symbol_exists(fork unistd.h HAVE_POSIX_FORK)
 check_symbol_exists(wait sys/wait.h HAVE_POSIX_WAIT)

cmake/implementation-flags.cmake

@@ -1,7 +1,7 @@
 #
 # Implementation selection
 #
-set(SIMDJSON_ALL_IMPLEMENTATIONS fallback westmere haswell arm64 ppc64)
+set(SIMDJSON_ALL_IMPLEMENTATIONS fallback westmere haswell icelake arm64 ppc64)
 
 set(
     SIMDJSON_IMPLEMENTATION ""
@@ -15,7 +15,7 @@ set(
     SIMDJSON_EXCLUDE_IMPLEMENTATION ""
     CACHE STRING "\
 Semicolon-separated list of implementations to exclude \
-(haswell/westmere/arm64/ppc64/fallback). By default, excludes any \
+(icelake/haswell/westmere/arm64/ppc64/fallback). By default, excludes any \
 implementations that are unsupported at compile time or cannot be selected at \
 runtime."
 )

dependencies/CMakeLists.txt

@@ -13,8 +13,9 @@ cmake_dependent_option(SIMDJSON_GOOGLE_BENCHMARKS "compile the Google Benchmark
 if(SIMDJSON_GOOGLE_BENCHMARKS)
   set_off(BENCHMARK_ENABLE_TESTING)
   set_off(BENCHMARK_ENABLE_INSTALL)
+  set_off(BENCHMARK_ENABLE_WERROR)
 
-  import_dependency(google_benchmarks google/benchmark 8982e1e)
+  import_dependency(google_benchmarks google/benchmark f91b6b4)
   add_dependency(google_benchmarks)
 endif()
 
@@ -69,11 +70,11 @@ int main() {}
   target_include_directories(jsmn SYSTEM PUBLIC "${jsmn_SOURCE_DIR}")
   target_compile_definitions(jsmn INTERFACE SIMDJSON_COMPETITION_JSMN)
 
-  message(STATUS "Importing json (nlohmann/json@v3.9.1)")
+  message(STATUS "Importing json (nlohmann/json@v3.10.5)")
   set(nlohmann_json_SOURCE_DIR "${dep_root}/json")
   if(NOT EXISTS "${nlohmann_json_SOURCE_DIR}")
     file(DOWNLOAD
-            "https://github.com/nlohmann/json/releases/download/v3.9.1/json.hpp"
+            "https://github.com/nlohmann/json/releases/download/v3.10.5/json.hpp"
             "${nlohmann_json_SOURCE_DIR}/nlohmann/json.hpp")
   endif()
   add_library(nlohmann_json INTERFACE)

dependencies/jsoncppdist/json/json-forwards.h

@@ -79,6 +79,150 @@ license you like.
 /// to prevent private header inclusion.
 #define JSON_IS_AMALGAMATION
 
+// //////////////////////////////////////////////////////////////////////
+// Beginning of content of file: include/json/version.h
+// //////////////////////////////////////////////////////////////////////
+
+#ifndef JSON_VERSION_H_INCLUDED
+#define JSON_VERSION_H_INCLUDED
+
+// Note: version must be updated in three places when doing a release. This
+// annoying process ensures that amalgamate, CMake, and meson all report the
+// correct version.
+// 1. /meson.build
+// 2. /include/json/version.h
+// 3. /CMakeLists.txt
+// IMPORTANT: also update the SOVERSION!!
+
+#define JSONCPP_VERSION_STRING "1.9.5"
+#define JSONCPP_VERSION_MAJOR 1
+#define JSONCPP_VERSION_MINOR 9
+#define JSONCPP_VERSION_PATCH 5
+#define JSONCPP_VERSION_QUALIFIER
+#define JSONCPP_VERSION_HEXA                                                   \
+  ((JSONCPP_VERSION_MAJOR << 24) | (JSONCPP_VERSION_MINOR << 16) |             \
+   (JSONCPP_VERSION_PATCH << 8))
+
+#ifdef JSONCPP_USING_SECURE_MEMORY
+#undef JSONCPP_USING_SECURE_MEMORY
+#endif
+#define JSONCPP_USING_SECURE_MEMORY 0
+// If non-zero, the library zeroes any memory that it has allocated before
+// it frees its memory.
+
+#endif // JSON_VERSION_H_INCLUDED
+
+// //////////////////////////////////////////////////////////////////////
+// End of content of file: include/json/version.h
+// //////////////////////////////////////////////////////////////////////
+
+
+
+
+
+
+// //////////////////////////////////////////////////////////////////////
+// Beginning of content of file: include/json/allocator.h
+// //////////////////////////////////////////////////////////////////////
+
+// Copyright 2007-2010 Baptiste Lepilleur and The JsonCpp Authors
+// Distributed under MIT license, or public domain if desired and
+// recognized in your jurisdiction.
+// See file LICENSE for detail or copy at http://jsoncpp.sourceforge.net/LICENSE
+
+#ifndef JSON_ALLOCATOR_H_INCLUDED
+#define JSON_ALLOCATOR_H_INCLUDED
+
+#include <cstring>
+#include <memory>
+
+#pragma pack(push, 8)
+
+namespace Json {
+template <typename T> class SecureAllocator {
+public:
+  // Type definitions
+  using value_type = T;
+  using pointer = T*;
+  using const_pointer = const T*;
+  using reference = T&;
+  using const_reference = const T&;
+  using size_type = std::size_t;
+  using difference_type = std::ptrdiff_t;
+
+  /**
+   * Allocate memory for N items using the standard allocator.
+   */
+  pointer allocate(size_type n) {
+    // allocate using "global operator new"
+    return static_cast<pointer>(::operator new(n * sizeof(T)));
+  }
+
+  /**
+   * Release memory which was allocated for N items at pointer P.
+   *
+   * The memory block is filled with zeroes before being released.
+   */
+  void deallocate(pointer p, size_type n) {
+    // memset_s is used because memset may be optimized away by the compiler
+    memset_s(p, n * sizeof(T), 0, n * sizeof(T));
+    // free using "global operator delete"
+    ::operator delete(p);
+  }
+
+  /**
+   * Construct an item in-place at pointer P.
+   */
+  template <typename... Args> void construct(pointer p, Args&&... args) {
+    // construct using "placement new" and "perfect forwarding"
+    ::new (static_cast<void*>(p)) T(std::forward<Args>(args)...);
+  }
+
+  size_type max_size() const { return size_t(-1) / sizeof(T); }
+
+  pointer address(reference x) const { return std::addressof(x); }
+
+  const_pointer address(const_reference x) const { return std::addressof(x); }
+
+  /**
+   * Destroy an item in-place at pointer P.
+   */
+  void destroy(pointer p) {
+    // destroy using "explicit destructor"
+    p->~T();
+  }
+
+  // Boilerplate
+  SecureAllocator() {}
+  template <typename U> SecureAllocator(const SecureAllocator<U>&) {}
+  template <typename U> struct rebind { using other = SecureAllocator<U>; };
+};
+
+template <typename T, typename U>
+bool operator==(const SecureAllocator<T>&, const SecureAllocator<U>&) {
+  return true;
+}
+
+template <typename T, typename U>
+bool operator!=(const SecureAllocator<T>&, const SecureAllocator<U>&) {
+  return false;
+}
+
+} // namespace Json
+
+#pragma pack(pop)
+
+#endif // JSON_ALLOCATOR_H_INCLUDED
+
+// //////////////////////////////////////////////////////////////////////
+// End of content of file: include/json/allocator.h
+// //////////////////////////////////////////////////////////////////////
+
+
+
+
+
+
 // //////////////////////////////////////////////////////////////////////
 // Beginning of content of file: include/json/config.h
 // //////////////////////////////////////////////////////////////////////
@@ -99,47 +243,38 @@ license you like.
 #include <string>
 #include <type_traits>
 
-/// If defined, indicates that json library is embedded in CppTL library.
-//# define JSON_IN_CPPTL 1
-
-/// If defined, indicates that json may leverage CppTL library
-//#  define JSON_USE_CPPTL 1
-/// If defined, indicates that cpptl vector based map should be used instead of
-/// std::map
-/// as Value container.
-//#  define JSON_USE_CPPTL_SMALLMAP 1
-
 // If non-zero, the library uses exceptions to report bad input instead of C
 // assertion macros. The default is to use exceptions.
 #ifndef JSON_USE_EXCEPTION
 #define JSON_USE_EXCEPTION 1
 #endif
 
+// Temporary, tracked for removal with issue #982.
+#ifndef JSON_USE_NULLREF
+#define JSON_USE_NULLREF 1
+#endif
+
 /// If defined, indicates that the source file is amalgamated
 /// to prevent private header inclusion.
 /// Remarks: it is automatically defined in the generated amalgamated header.
 // #define JSON_IS_AMALGAMATION
 
-#ifdef JSON_IN_CPPTL
-#include <cpptl/config.h>
-#ifndef JSON_USE_CPPTL
-#define JSON_USE_CPPTL 1
-#endif
-#endif
-
-#ifdef JSON_IN_CPPTL
-#define JSON_API CPPTL_API
-#elif defined(JSON_DLL_BUILD)
+// Export macros for DLL visibility
+#if defined(JSON_DLL_BUILD)
 #if defined(_MSC_VER) || defined(__MINGW32__)
 #define JSON_API __declspec(dllexport)
 #define JSONCPP_DISABLE_DLL_INTERFACE_WARNING
+#elif defined(__GNUC__) || defined(__clang__)
+#define JSON_API __attribute__((visibility("default")))
 #endif // if defined(_MSC_VER)
+
 #elif defined(JSON_DLL)
 #if defined(_MSC_VER) || defined(__MINGW32__)
 #define JSON_API __declspec(dllimport)
 #define JSONCPP_DISABLE_DLL_INTERFACE_WARNING
 #endif // if defined(_MSC_VER)
-#endif // ifdef JSON_IN_CPPTL
+#endif // ifdef JSON_DLL_BUILD
+
 #if !defined(JSON_API)
 #define JSON_API
 #endif
@@ -152,8 +287,8 @@ license you like.
 #if defined(_MSC_VER) && _MSC_VER < 1900
 // As recommended at
 // https://stackoverflow.com/questions/2915672/snprintf-and-visual-studio-2010
-extern JSON_API int
-msvc_pre1900_c99_snprintf(char* outBuf, size_t size, const char* format, ...);
+extern JSON_API int msvc_pre1900_c99_snprintf(char* outBuf, size_t size,
+                                              const char* format, ...);
 #define jsoncpp_snprintf msvc_pre1900_c99_snprintf
 #else
 #define jsoncpp_snprintf std::snprintf
@@ -164,69 +299,30 @@ msvc_pre1900_c99_snprintf(char* outBuf, size_t size, const char* format, ...);
 // Storages, and 64 bits integer support is disabled.
 // #define JSON_NO_INT64 1
 
-#if defined(_MSC_VER) // MSVC
-#define JSONCPP_DEPRECATED(message) __declspec(deprecated(message))
-#endif // defined(_MSC_VER)
-
 // JSONCPP_OVERRIDE is maintained for backwards compatibility of external tools.
 // C++11 should be used directly in JSONCPP.
 #define JSONCPP_OVERRIDE override
 
-#if __cplusplus >= 201103L
-#define JSONCPP_NOEXCEPT noexcept
-#define JSONCPP_OP_EXPLICIT explicit
-#elif defined(_MSC_VER) && _MSC_VER < 1900
-#define JSONCPP_NOEXCEPT throw()
-#define JSONCPP_OP_EXPLICIT explicit
-#elif defined(_MSC_VER) && _MSC_VER >= 1900
-#define JSONCPP_NOEXCEPT noexcept
-#define JSONCPP_OP_EXPLICIT explicit
-#else
-#define JSONCPP_NOEXCEPT throw()
-#define JSONCPP_OP_EXPLICIT
-#endif
-
-#ifndef JSON_HAS_RVALUE_REFERENCES
-
-#if defined(_MSC_VER)
-#define JSON_HAS_RVALUE_REFERENCES 1
-#endif // MSVC >= 2013
-
-#ifdef __clang__
-#if __has_feature(cxx_rvalue_references)
-#define JSON_HAS_RVALUE_REFERENCES 1
-#endif // has_feature
-
-#elif defined __GNUC__ // not clang (gcc comes later since clang emulates gcc)
-#if defined(__GXX_EXPERIMENTAL_CXX0X__) || (__cplusplus >= 201103L)
-#define JSON_HAS_RVALUE_REFERENCES 1
-#endif // GXX_EXPERIMENTAL
-
-#endif // __clang__ || __GNUC__
-
-#endif // not defined JSON_HAS_RVALUE_REFERENCES
-
-#ifndef JSON_HAS_RVALUE_REFERENCES
-#define JSON_HAS_RVALUE_REFERENCES 0
-#endif
-
 #ifdef __clang__
 #if __has_extension(attribute_deprecated_with_message)
 #define JSONCPP_DEPRECATED(message) __attribute__((deprecated(message)))
 #endif
-#elif defined __GNUC__ // not clang (gcc comes later since clang emulates gcc)
+#elif defined(__GNUC__) // not clang (gcc comes later since clang emulates gcc)
 #if (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 5))
 #define JSONCPP_DEPRECATED(message) __attribute__((deprecated(message)))
 #elif (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 1))
 #define JSONCPP_DEPRECATED(message) __attribute__((__deprecated__))
-#endif // GNUC version
-#endif // __clang__ || __GNUC__
+#endif                  // GNUC version
+#elif defined(_MSC_VER) // MSVC (after clang because clang on Windows emulates
+                        // MSVC)
+#define JSONCPP_DEPRECATED(message) __declspec(deprecated(message))
+#endif // __clang__ || __GNUC__ || _MSC_VER
 
 #if !defined(JSONCPP_DEPRECATED)
 #define JSONCPP_DEPRECATED(message)
 #endif // if !defined(JSONCPP_DEPRECATED)
 
-#if __GNUC__ >= 6
+#if defined(__clang__) || (defined(__GNUC__) && (__GNUC__ >= 6))
 #define JSON_USE_INT64_DOUBLE_CONVERSION 1
 #endif
 
@@ -238,37 +334,37 @@ msvc_pre1900_c99_snprintf(char* outBuf, size_t size, const char* format, ...);
 #endif // if !defined(JSON_IS_AMALGAMATION)
 
 namespace Json {
-typedef int Int;
-typedef unsigned int UInt;
+using Int = int;
+using UInt = unsigned int;
 #if defined(JSON_NO_INT64)
-typedef int LargestInt;
-typedef unsigned int LargestUInt;
+using LargestInt = int;
+using LargestUInt = unsigned int;
 #undef JSON_HAS_INT64
 #else                 // if defined(JSON_NO_INT64)
 // For Microsoft Visual use specific types as long long is not supported
 #if defined(_MSC_VER) // Microsoft Visual Studio
-typedef __int64 Int64;
-typedef unsigned __int64 UInt64;
+using Int64 = __int64;
+using UInt64 = unsigned __int64;
 #else                 // if defined(_MSC_VER) // Other platforms, use long long
-typedef int64_t Int64;
-typedef uint64_t UInt64;
+using Int64 = int64_t;
+using UInt64 = uint64_t;
 #endif                // if defined(_MSC_VER)
-typedef Int64 LargestInt;
-typedef UInt64 LargestUInt;
+using LargestInt = Int64;
+using LargestUInt = UInt64;
 #define JSON_HAS_INT64
 #endif // if defined(JSON_NO_INT64)
 
 template <typename T>
-using Allocator = typename std::conditional<JSONCPP_USING_SECURE_MEMORY,
-                                            SecureAllocator<T>,
-                                            std::allocator<T>>::type;
+using Allocator =
+    typename std::conditional<JSONCPP_USING_SECURE_MEMORY, SecureAllocator<T>,
+                              std::allocator<T>>::type;
 using String = std::basic_string<char, std::char_traits<char>, Allocator<char>>;
-using IStringStream = std::basic_istringstream<String::value_type,
-                                               String::traits_type,
-                                               String::allocator_type>;
-using OStringStream = std::basic_ostringstream<String::value_type,
-                                               String::traits_type,
-                                               String::allocator_type>;
+using IStringStream =
+    std::basic_istringstream<String::value_type, String::traits_type,
+                             String::allocator_type>;
+using OStringStream =
+    std::basic_ostringstream<String::value_type, String::traits_type,
+                             String::allocator_type>;
 using IStream = std::istream;
 using OStream = std::ostream;
 } // namespace Json
@@ -310,17 +406,23 @@ using JSONCPP_OSTREAM = Json::OStream;
 namespace Json {
 
 // writer.h
+class StreamWriter;
+class StreamWriterBuilder;
+class Writer;
 class FastWriter;
 class StyledWriter;
+class StyledStreamWriter;
 
 // reader.h
 class Reader;
+class CharReader;
+class CharReaderBuilder;
 
-// features.h
+// json_features.h
 class Features;
 
 // value.h
-typedef unsigned int ArrayIndex;
+using ArrayIndex = unsigned int;
 class StaticString;
 class Path;
 class PathArgument;

doc/basics.md

@@ -3,38 +3,46 @@ The Basics
 
 An overview of what you need to know to use simdjson, with examples.
 
-* [Requirements](#requirements)
-* [Including simdjson](#including-simdjson)
-* [Using simdjson with package managers](#using-simdjson-with-package-managers)
-* [Using simdjson as a CMake dependency](#using-simdjson-as-a-cmake-dependency)
-* [Versions](#versions)
-* [The Basics: Loading and Parsing JSON Documents](#the-basics-loading-and-parsing-json-documents)
-* [Documents are Iterators](#documents-are-iterators)
-* [C++11 Support and string_view](#c11-support-and-string_view)
-* [Using the Parsed JSON](#using-the-parsed-json)
-* [Minifying JSON strings without parsing](#minifying-json-strings-without-parsing)
-* [UTF-8 validation (alone)](#utf-8-validation-alone)
-* [JSON Pointer](#json-pointer)
-* [Error Handling](#error-handling)
-  * [Error Handling Example without Exceptions](#error-handling-example-without-exceptions)
-  * [Disabling Exceptions](#disabling-exceptions)
-  * [Exceptions](#exceptions)
-  * [Current location in document](#current-location-in-documnet)
-* [Rewinding](#rewinding)
-* [Direct Access to the Raw String](#direct-access-to-the-raw-string)
-* [Newline-Delimited JSON (ndjson) and JSON lines](#newline-delimited-json-ndjson-and-json-lines)
-* [Parsing Numbers Inside Strings](#parsing-numbers-inside-strings)
-* [Dynamic Number Types](#dynamic-number-types)
-* [Thread Safety](#thread-safety)
-* [Standard Compliance](#standard-compliance)
+- [The Basics](#the-basics)
+  - [Requirements](#requirements)
+  - [Including simdjson](#including-simdjson)
+  - [Using simdjson with package managers](#using-simdjson-with-package-managers)
+  - [Using simdjson as a CMake dependency](#using-simdjson-as-a-cmake-dependency)
+  - [Versions](#versions)
+  - [The Basics: Loading and Parsing JSON Documents](#the-basics-loading-and-parsing-json-documents)
+  - [Documents are Iterators](#documents-are-iterators)
+    - [Parser, Document and JSON Scope](#parser-document-and-json-scope)
+  - [C++11 Support and string_view](#c11-support-and-string_view)
+  - [Using the Parsed JSON](#using-the-parsed-json)
+    - [Using the Parsed JSON: Additional examples](#using-the-parsed-json-additional-examples)
+  - [Minifying JSON strings without parsing](#minifying-json-strings-without-parsing)
+  - [UTF-8 validation (alone)](#utf-8-validation-alone)
+  - [JSON Pointer](#json-pointer)
+  - [Error Handling](#error-handling)
+    - [Error Handling Examples without Exceptions](#error-handling-examples-without-exceptions)
+    - [Disabling Exceptions](#disabling-exceptions)
+    - [Exceptions](#exceptions)
+    - [Current location in document](#current-location-in-document)
+  - [Rewinding](#rewinding)
+  - [Direct Access to the Raw String](#direct-access-to-the-raw-string)
+  - [Newline-Delimited JSON (ndjson) and JSON lines](#newline-delimited-json-ndjson-and-json-lines)
+  - [Parsing Numbers Inside Strings](#parsing-numbers-inside-strings)
+  - [Dynamic Number Types](#dynamic-number-types)
+  - [Raw Strings](#raw-strings)
+  - [Thread Safety](#thread-safety)
+  - [Standard Compliance](#standard-compliance)
+  - [Backwards Compatibility](#backwards-compatibility)
 
 
 Requirements
 ------------------
 
-- A recent compiler (LLVM clang6 or better, GNU GCC 7.4 or better, Xcode 11 or better) on a 64-bit (PPC, ARM or x64 Intel/AMD) POSIX systems such as macOS, freeBSD or Linux. We require that the compiler supports the C++11 standard or better.
+- A recent compiler (LLVM clang 6 or better, GNU GCC 7.4 or better, Xcode 11 or better) on a 64-bit (PPC, ARM or x64 Intel/AMD) POSIX systems such as macOS, freeBSD or Linux. We require that the compiler supports the C++11 standard or better.
 - Visual Studio 2017 or better under 64-bit Windows. Users should target a 64-bit build (x64) instead of a 32-bit build (x86). We support the LLVM clang compiler under Visual Studio (clangcl) as well as as the regular Visual Studio compiler. We also support MinGW 64-bit under Windows.
 
+
+Support for AVX-512 require a processor with AVX512-VBMI2 support (Ice Lake or better) under a 64-bit system and a recent compiler (LLVM clang 6 or better, GCC 8 or better, Visual Studio 2019 or better).
+
 Including simdjson
 ------------------
 
@@ -54,6 +62,7 @@ c++ myproject.cpp simdjson.cpp
 
 Note:
 - Users on macOS and other platforms where default compilers do not provide C++11 compliant by default should request it with the appropriate flag (e.g., `c++ -std=c++17 myproject.cpp simdjson.cpp`).
+- The library relies on [runtime CPU detection](implementation-selection.md): avoid specifying an architecture at compile time (e.g., `-march-native`).
 
 Using simdjson with package managers
 ------------------
@@ -127,19 +136,23 @@ The Basics: Loading and Parsing JSON Documents
 ----------------------------------------------
 
 The simdjson library allows you to navigate and validate JSON documents ([RFC 8259](https://www.tbray.org/ongoing/When/201x/2017/12/14/rfc8259.html)).
-As required by the standard, your JSON document should be Unicode (UTF-8) strings.
+As required by the standard, your JSON document should be in a Unicode (UTF-8) string. The whole
+string, from the beginning to the end, needs to be valid: we do not attempt to tolerate bad
+inputs before or after a document.
 
-
-The simdjson library offers a tree-like [API](https://en.wikipedia.org/wiki/API), which you can access by creating a `ondemand::parser` and calling the `iterate()` method:
+The simdjson library offers a tree-like [API](https://en.wikipedia.org/wiki/API), which you can
+access by creating a `ondemand::parser` and calling the `iterate()` method. The iterate method
+quickly indexes the input string and may detect some errors. The following example illustrates
+how to get started with an input JSON file (`"twitter.json"`):
 
 ```c++
 ondemand::parser parser;
-auto json = padded_string::load("twitter.json");
+auto json = padded_string::load("twitter.json"); // load JSON file 'twitter.json'.
 ondemand::document doc = parser.iterate(json); // position a pointer at the beginning of the JSON data
 ```
 
-Or by creating a padded string---for efficiency reasons, simdjson requires a string with a few
- bytes (`simdjson::SIMDJSON_PADDING`) at the end---and calling `iterate()`:
+You can also create a padded string---for efficiency reasons, simdjson requires a string
+with a few bytes (`simdjson::SIMDJSON_PADDING`) at the end---and calling `iterate()`:
 
 ```c++
 ondemand::parser parser;
@@ -196,7 +209,8 @@ allocations during parsing when using simdjson. [See our performance notes for d
 C++11 Support and string_view
 -------------
 
-The simdjson library builds on compilers supporting the [C++11 standard](https://en.wikipedia.org/wiki/C%2B%2B11). It is also a strict requirement: we have no plan to support older C++ compilers.
+The simdjson library builds on compilers supporting the [C++11 standard](https://en.wikipedia.org/wiki/C%2B%2B11).
+It is also a strict requirement: we have no plan to support older C++ compilers.
 
 We represent parsed Unicode (UTF-8) strings in simdjson using the `std::string_view` class. It avoids
 the need to copy the data, as would be necessary with the `std::string` class. It also
@@ -214,7 +228,6 @@ is often best viewed as a temporary string value that is tied to the document yo
 At the cost of some memory allocation, you may convert your `std::string_view` instances for long-term storage into `std::string` instances:
 `std::string mycopy(view)` (C++17) or  `std::string mycopy(view.begin(), view.end())` (prior to C++17).
 
-
 The `std::string_view` class has become standard as part of C++17 but it is not always available
 on compilers which only supports C++11. When we detect that `string_view` is natively
 available, we define the macro `SIMDJSON_HAS_STRING_VIEW`.
@@ -232,24 +245,45 @@ transcode the UTF-8 strings produced by the simdjson library to other formats. S
 Using the Parsed JSON
 ---------------------
 
+We recommend that you first compile and run your code in Debug mode: under Visual Studio,
+it means having the `_DEBUG` macro defined, and, for other compilers, it means leaving
+the `__OPTIMIZE__` macro undefined. The simdjson code will set `SIMDJSON_DEVELOPMENT_CHECKS=1`.
+Alternatively, you can set the macro `SIMDJSON_DEVELOPMENT_CHECKS` to 1 prior to including
+the `simdjson.h` header to enable these additional checks: just make sure you remove the
+definition once your code has been tested. When `SIMDJSON_DEVELOPMENT_CHECKS` is set to 1, the
+simdjson library runs additional (expensive) tests on your code to help ensure that you are
+using the library in a safe manner. Once your code has been tested, you can then run it in
+Release mode: under Visual Studio, it means having the `_DEBUG` macro undefined, and, for other
+compilers, it means setting `__OPTIMIZE__` to a positive integer. You can also forcefully
+disable these checks by setting `SIMDJSON_DEVELOPMENT_CHECKS` to 0. Once your code is tested, we
+further encourage you to define `NDEBUG` in your Release builds to disable additional runtime
+testing and get the best performance.
+
 Once you have a document (`simdjson::ondemand::document`), you can navigate it with
-idiomatic C++ iterators, operators and casts. Besides the documents instances and
+idiomatic C++ iterators, operators and casts. Besides the document instances and
 native types (`double`, `uint64_t`, `int64_t`, `bool`), we also access
 Unicode (UTF-8) strings (`std::string_view`), objects (`simdjson::ondemand::object`)
 and arrays (`simdjson::ondemand::array`).
-We also have a generic type (`simdjson::ondemand::value`) which represent a potential
-array or object, or scalar type (`double`, `uint64_t`, `int64_t`, `bool`, `null`, string) inside an array or an object. Both generic types (`simdjson::ondemand::document` and `simdjson::ondemand::value`) have a `type()` method returning
-a `json_type` value describing the value (`json_type::array`, `json_type::object`, `json_type::number`, `json_type::string`, `json_type::boolean`, `json_type::null`).
+We also have a generic ephemeral type (`simdjson::ondemand::value`) which represents a potential
+array or object, or scalar type (`double`, `uint64_t`, `int64_t`, `bool`, `null`, string) inside
+an array or an object. Both generic types (`simdjson::ondemand::document` and
+`simdjson::ondemand::value`) have a `type()` method returning a `json_type` value describing the
+value (`json_type::array`, `json_type::object`, `json_type::number`, `json_type::string`,
+`json_type::boolean`, `json_type::null`). A generic value (`simdjson::ondemand::value`)
+is only valid temporarily, as soon as you access other values, other keys in objects, etc.
+it becomes invalid: you should therefore consume the value immediately by converting it to a
+scalar type, an array or an object.
 
 Advanced users who need to determine the number types (integer or float) dynamically,
 should review our section [dynamic number types](#dynamic-number-types). Indeed,
 we have an additional `ondemand::number` type which may represent either integers
 or floating-point values, depending on how the numbers are formatted.
 floating-point values followed by an integer.
-While you are accessing the document, the `document` instance should remain in scope:
-it is your "iterator" which keeps track of where you are in the JSON document.
-By design, there is one and only one `document` instance per JSON document.
 
+We invite you to keep the following rules in mind:
+1. While you are accessing the document, the `document` instance should remain in scope: it is your "iterator" which keeps track of where you are in the JSON document. By design, there is one and only one `document` instance per JSON document.
+2. Because On Demand is really just an iterator, you must fully consume the current object or array before accessing a sibling object or array.
+3. Values can only be consumed once, you should get the values and store them if you plan to need them multiple times. You are expected to access the keys of an object just once. You are expected to go through the values of an array just once.
 
 The following specific instructions indicate how to use the JSON when exceptions are enabled, but simdjson has full, idiomatic
 support for users who avoid exceptions. See [the simdjson error handling documentation](basics.md#error-handling) for more.
@@ -259,16 +293,26 @@ support for users who avoid exceptions. See [the simdjson error handling documen
   However, it is not fully validated. On Demand only fully validates the values you use and the
   structure leading to it.
 * **Extracting Values:** You can cast a JSON element to a native type:
-  `double(element)` or `double x = json_element`. This works for `std::string_view`, double, uint64_t, int64_t, bool,
-  ondemand::object and ondemand::array. At this point, the number, string or boolean will be parsed,
-  or the initial `[` or `{` will be verified. An exception is thrown if the cast is not possible.
+  `double(element)`. This works for `std::string_view`, double, uint64_t, int64_t, bool,
+  ondemand::object and ondemand::array. We also have explicit methods such as `get_string()`, `get_double()`,
+  `get_uint64()`, `get_int64()`, `get_bool()`, `get_object()` and `get_array()`. After a cast or an explicit method,
+  the number, string or boolean will be parsed, or the initial `[` or `{` will be verified. An exception is thrown if
+  the cast is not possible.
 
   > IMPORTANT NOTE: values can only be parsed once. Since documents are *iterators*, once you have
-  > parsed a value (such as by casting to double), you cannot get at it again.
+  > parsed a value (such as by casting to double), you cannot get at it again. It is an error to call
+  > `get_string()` twice on an object (or to cast an object twice to `std::string_view`).
 * **Field Access:** To get the value of the "foo" field in an object, use `object["foo"]`. This will
   scan through the object looking for the field with the matching string, doing a character-by-character
   comparison. For efficiency reason, you should avoid looking up the same field repeatedly: e.g., do
-  not do `object["foo"]` followed by `object["foo"]` with the same `object` instance.
+  not do `object["foo"]` followed by `object["foo"]` with the same `object` instance. If you consume an
+  object twice: `std::string_view(object["foo"]` followed by `std::string_view(object["foo"]`, your code
+  is in error. Furthermore, you can only consume one field at a time, on the same object. The
+  value instance you get from  `content["bids"]` becomes invalid when you call `content["asks"]`.
+  If you have retrieved `content["bids"].get_array()` and you later call
+  `content["asks"].get_array()`, then the first array should no longer be accessed: it would be
+  unsafe to do so. You can detect such mistakes by first compiling and running the code in
+  Debug mode: an OUT_OF_ORDER_ITERATION error is generated.
 
   > NOTE: JSON allows you to escape characters in keys. E.g., the key `"date"` may be written as
   > `"\u0064\u0061\u0074\u0065"`. By default, simdjson does *not* unescape keys when matching by default.
@@ -437,7 +481,7 @@ support for users who avoid exceptions. See [the simdjson error handling documen
   ```
 * **Tree Walking and JSON Element Types:** Sometimes you don't necessarily have a document
   with a known type, and are trying to generically inspect or walk over JSON elements. To do that, you can use iterators and the `type()` method. You can also represent arbitrary JSON values with
-  `ondemand::value` instances: it can represent anything except a scalar document (lone number, string, null or Boolean). You can check for scalar documents with the method `scalar()`.
+  `ondemand::value` instances: it can represent anything except a scalar document (lone number, string, null or Boolean). You can check for scalar documents with the method `scalar()`. You may also access [raw strings](#raw-strings).
   For example, the following is a quick and dirty recursive function that verbosely prints the JSON document as JSON. This example also illustrates lifecycle requirements: the `document` instance holds the iterator. The document must remain in scope while you are accessing instances of `value`, `object` and `array`.
   ```c++
   void recursive_print_json(ondemand::value element) {
@@ -683,30 +727,23 @@ auto cars_json = R"( [
 ] )"_padded;
 
 ondemand::parser parser;
-ondemand::document cars;
 std::vector<double> measured;
-parser.iterate(cars_json).get(cars);
+ondemand::document cars = parser.iterate(cars_json);
 std::vector<car_type> content;
 for (int i = 0; i < 3; i++) {
-    ondemand::object obj;
     std::string json_pointer = "/" + std::to_string(i);
     // Each successive at_pointer call invalidates
     // previously parsed values, strings, objects and array.
-    cars.at_pointer(json_pointer).get(obj);
+    ondemand::object obj(cars.at_pointer(json_pointer).get_object());
     // We materialize the object.
-    std::string_view make;
-    ASSERT_SUCCESS(obj["make"].get(make));
-    std::string_view model;
-    ASSERT_SUCCESS(obj["model"].get(model));
-    uint64_t year;
-    ASSERT_SUCCESS(obj["year"].get(year));
+    std::string_view make = obj["make"];
+    std::string_view model = obj["model"];
+    uint64_t year(obj["year"]);
     // We materialize the array.
-    ondemand::array arr;
-    ASSERT_SUCCESS(obj["tire_pressure"].get(arr));
+    ondemand::array arr(obj["tire_pressure"].get_array());
     std::vector<double> values;
     for(auto x : arr) {
-        double value_double;
-        ASSERT_SUCCESS(x.get(value_double));
+        double value_double(x.get_double());
         values.push_back(value_double);
     }
     content.emplace_back(make, model, year, std::move(values));
@@ -741,12 +778,16 @@ The entire simdjson API is usable with and without exceptions. All simdjson APIs
 pair. You can retrieve the value with .get() without generating an exception, like so:
 
 ```c++
-ondemand::element doc;
+ondemand::document doc;
 auto error = parser.iterate(json).get(doc);
 if (error) { cerr << error << endl; exit(1); }
 ```
 
-When you use the code this way, it is your responsibility to check for error before using the
+When there is no error, the error code simdjson::SUCCESS is returned: it evaluates as false as a Boolean.
+We have several error codes to indicate errors, they all evaluate to true as a Boolean: your software should not generally not depend on exact
+error codes. We may change the error codes in future releases and the exact error codes could vary depending on your system.
+
+When you use the code without exceptions, it is your responsibility to check for error before using the
 result: if there is an error, the result value will not be valid and using it will caused undefined behavior. Most compilers should be able to help you if you activate the right
 set of warnings: they can identify variables that are written to but never otherwise accessed.
 
@@ -868,6 +909,9 @@ int main(void) {
 }
 ```
 
+The `at` method can only be called once on an array. It cannot be used
+to iterate through the values of an array.
+
 ### Error Handling Examples without Exceptions
 
 This is how the example in "Using the Parsed JSON" could be written using only error code checking (without exceptions):
@@ -1119,7 +1163,7 @@ Direct Access to the Raw String
 
 The simdjson library makes explicit assumptions about types. For examples, numbers
 must be integers (up to 64-bit integers) or binary64 floating-point numbers. Some users
-have different needs. For example, some users might want to support big infloating-point number followed by an integer.tegers.
+have different needs. For example, some users might want to support big integers.
 The library makes this possible by providing a `raw_json_token` method which returns
 a `std::string_view` instance containing the value as a string which you may then
 parse as you see fit.
@@ -1153,11 +1197,19 @@ The `raw_json_token()` should be fast and free of allocation.
 Newline-Delimited JSON (ndjson) and JSON lines
 ----------------------------------------------
 
-The simdjson library also supports multithreaded JSON streaming through a large file containing many
-smaller JSON documents in either [ndjson](http://ndjson.org) or [JSON lines](http://jsonlines.org)
-format. If your JSON documents all contain arrays or objects, we even support direct file
-concatenation without whitespace. The concatenated file has no size restrictions (including larger
-than 4GB), though each individual document must be no larger than 4 GB.
+When processing large inputs (e.g., in the context of data engineering), engineers commonly
+serialize data into streams of multiple JSON documents. That is, instead of one large
+(e.g., 2 GB) JSON document containing multiple records, it is often preferable to
+write out multiple records as independent JSON documents, to be read one-by-one.
+
+The simdjson library also supports multithreaded JSON streaming through a large file
+containing many smaller JSON documents in either [ndjson](http://ndjson.org)
+or [JSON lines](http://jsonlines.org) format. If your JSON documents all contain arrays
+or objects, we even support direct file concatenation without whitespace. However, if there
+is content between your JSON documents, it should be exclusively ASCII white-space characters.
+
+The concatenated file has no size restrictions (including larger than 4GB), though each
+individual document must be no larger than 4 GB.
 
 Here is an example:
 
@@ -1366,10 +1418,6 @@ You must check the type before accessing the value: it is an error to call `get_
 
 The `get_number()` function is designed with performance in mind. When calling `get_number()`, you scan the number string only once, determining efficiently the type and storing it in an efficient manner.
 
-If you only need to compute `v.get_number().get_number_type()` on
-a `document` or `value` instance, you should call directly the faster method
-`v.get_number_type()` which does not generate an
-intermediate `number` instance.
 
 Consider the following example:
 ```C++
@@ -1382,9 +1430,7 @@ Consider the following example:
       std::cout << "negative: " << val.is_negative() << " ";
       std::cout << "is_integer: " << val.is_integer() << " ";
       ondemand::number num = val.get_number();
-      // direct computation without materializing the number:
-      ondemand::number_type dt = val.get_number_type();
-      if(t != dt) { throw std::runtime_error("bug"); }
+      ondemand::number_type t = num.get_number_type();
       switch(t) {
         case ondemand::number_type::signed_integer:
           std::cout  << "integer: " << int64_t(num) << " ";
@@ -1413,6 +1459,42 @@ It will output:
 9999999999999999999 negative: 0 is_integer: 1 large 64-bit integer: 9999999999999999999 large 64-bit integer: 9999999999999999999
 ```
 
+Raw Strings
+-----------
+
+It is sometimes useful to have access to a raw (unescaped) string: we make available a
+minimalist `raw_json_string` data type which contains a pointer inside the string in the
+original document, right after the quote. It is accessible via `get_raw_json_string()` on a
+string instance and returned by the `key()` method on an object's field instance. It is always
+optional: replacing `get_raw_json_string()` with `get_string()` and `key()` by
+`unescaped_key()` returns an `string_view` instance of the unescaped string.
+
+You can quickly compare a `raw_json_string` instance with a target string. You may also
+unescape  the `raw_json_string` on your own string buffer: `parser.unescape(mystr, ptr)`
+advances the provided pointer `ptr` and returns a string_view instance on the newly serialized
+string upon success, otherwise it returns an error. When unescaping to your own string buffer,
+you should ensure that you have sufficient memory space: the total size of the strings plus
+`simdjson::SIMDJSON_PADDING` bytes. The following example illustrates how we can unescape
+JSON string to a user-provided buffer:
+
+```C++
+    auto json = R"( {"name": "Jack The Ripper \u0033"} )"_padded;
+    // We create a buffer large enough to store all strings we need:
+    std::unique_ptr<uint8_t[]> buffer(new uint8_t[json.size() + simdjson::SIMDJSON_PADDING]);
+    uint8_t * ptr = buffer.get();
+    ondemand::parser parser;
+    ondemand::document doc = parser.iterate(json);
+    // We store our strings as 'string_view' instances in a vector:
+    std::vector<std::string_view> mystrings;
+    for (auto key_value : doc.get_object()) {
+      std::string_view keysv = parser.unescape(key_value.key(), ptr);// writes 'name'
+      mystrings.push_back(keysv);
+      std::string_view valuesv = parser.unescape(key_value.value().get_raw_json_string(), ptr);
+      // writes 'Jack The Ripper 3', escaping the \u0033
+      mystrings.push_back(valuesv);
+    }
+```
+
 Thread Safety
 -------------
 
@@ -1438,8 +1520,8 @@ The simdjson library is fully compliant with  the [RFC 8259](https://www.tbray.o
 - A single string or a single number is considered to be a valid JSON document.
 - We fully validate the numbers according to the JSON specification. For example,  the string `01` is not valid JSON document since the specification states that *leading zeros are not allowed*.
 - The specification allows implementations to set limits on the range and precision of numbers accepted.  We support 64-bit floating-point numbers as well as integer values.
-  - We parse integers and floating-point numbers afloating-point number followed by an integer.s separate types which allows us to support all signed (two's complement) 64-bit integersfloating-point number followed by an integer., like a Java `long` or a C/C++ `long long` and all 64-bit unsigned integers. When we cannot represent exactly an integer as a signed or unsigned 64-bit value, we reject the JSON document.
-  - We support the full range of 64-bit floating-point numbers (binary64). The values range from `std::numeric_limits<double>::lowest()`  to `std::numefloating-point number followed by an integer.ric_limits<double>::max()`, so from -1.7976e308 all the way to 1.7975e308. Extreme values (less or equal to -1e308, greater or equal to 1e308) are rejected: we refuse to parse the input document. Numbers are parsed with a perfect accuracy (ULP 0): the nearest floating-point value is chosen, rounding to even when needed. If you serialized your floating-point numbers with 17floating-point value  followed by an integer. significant digits in a standard compliant manner, the simdjson library is guaranfloating-point number followed by an integer.teed to recover the same numbers, exactly.
+  - We parse integers and floating-point numbers as separate types which allows us to support all signed (two's complement) 64-bit integers, like a Java `long` or a C/C++ `long long` and all 64-bit unsigned integers. When we cannot represent exactly an integer as a signed or unsigned 64-bit value, we reject the JSON document.
+  - We support the full range of 64-bit floating-point numbers (binary64). The values range from `std::numeric_limits<double>::lowest()`  to `std::numeric_limits<double>::max()`, so from -1.7976e308 all the way to 1.7975e308. Extreme values (less or equal to -1e308, greater or equal to 1e308) are rejected: we refuse to parse the input document. Numbers are parsed with a perfect accuracy (ULP 0): the nearest floating-point value is chosen, rounding to even when needed. If you serialized your floating-point numbers with 17 significant digits in a standard compliant manner, the simdjson library is guaranteed to recover the same numbers, exactly.
 - The specification states that JSON text exchanged between systems that are not part of a closed ecosystem MUST be encoded using UTF-8. The simdjson library does full UTF-8 validation as part of the parsing. The specification states that implementations MUST NOT add a byte order mark: the simdjson library rejects documents starting with a  byte order mark.
 - The simdjson library validates string content for unescaped characters. Unescaped line breaks and tabs in strings are not allowed.
 - The simdjson library accepts objects with repeated keys: all of the name/value pairs, including duplicates, are reported. We do not enforce key uniqueness.

doc/dom.md

@@ -3,8 +3,8 @@ The Document-Object-Model (DOM) front-end
 
 An overview of what you need to know to use simdjson, with examples.
 
-* [DOM vs On Demand](#dom-vs-ondemand)
-* [The Basics: Loading and Parsing JSON Documents](#the-basics-loading-and-parsing-json-documents)
+* [DOM vs On Demand](#dom-vs-on-demand)
+* [The Basics: Loading and Parsing JSON Documents](#the-basics-loading-and-parsing-json-documents-using-the-dom-front-end)
 * [Using the Parsed JSON](#using-the-parsed-json)
 * [C++17 Support](#c17-support)
 * [JSON Pointer](#json-pointer)
@@ -81,7 +81,7 @@ Once you have an element, you can navigate it with idiomatic C++ iterators, oper
   know the type of the value, you can cast it right there, too! `for (double value : array) { ... }`
 * **Object Iteration:** You can iterate through an object's fields, too: `for (auto [key, value] : object)`
 * **Array Index:** To get at an array value by index, use the at() method: `array.at(0)` gets the
-  first element.
+  first element. The at() method has linear-time complexity so it should not be used to iterate over the values of an array.
   > Note that array[0] does not compile, because implementing [] gives the impression indexing is a
   > O(1) operation, which it is not presently in simdjson. Instead, you should iterate over the elements
   > using a for-loop, as in our examples.
@@ -249,7 +249,11 @@ auto error = parser.parse(json).get(doc);
 if (error) { cerr << error << endl; exit(1); }
 ```
 
-When you use the code this way, it is your responsibility to check for error before using the
+When there is no error, the error code simdjson::SUCCESS is returned: it evaluates as false as a Boolean.
+We have several error codes to indicate errors, they all evaluate to true as a Boolean: your software should not generally not depend on exact
+error codes. We may change the error codes in future releases and the exact error codes could vary depending on your system.
+
+When you use the code without exceptions, it is your responsibility to check for error before using the
 result: if there is an error, the result value will not be valid and using it will caused undefined
 behavior.
 
@@ -318,6 +322,9 @@ int main(void) {
 }
 ```
 
+The `at()` method has linear-time complexity: it should not be used to iterate
+over the content of an array.
+
 ### Error Handling Example
 
 This is how the example in "Using the Parsed JSON" could be written using only error code checking:

doc/implementation-selection.md

@@ -17,6 +17,7 @@ different version of the JSON parser for different CPU architectures, often with
 algorithms to take better advantage of a given CPU!
 
 The current implementations are:
+* icelake: AVX-512F, AVX-512VBMI, etc.
 * haswell: AVX2 (2013 Intel Haswell or later)
 * westmere: SSE4.2 (2010 Westmere or later).
 * arm64: 64-bit ARMv8-A NEON
@@ -28,7 +29,7 @@ compiles *all* the implementations into the executable. On Intel, it will includ
 (haswell, westmere and fallback), on ARM it will include 2 (arm64 and fallback), and on PPC it will include 2 (ppc64 and fallback).
 
 If you know more about where you're going to run and want to save the space, you can disable any of
-these implementations at compile time with `-DSIMDJSON_IMPLEMENTATION_X=0` (where X is HASWELL,
+these implementations at compile time with `-DSIMDJSON_IMPLEMENTATION_X=0` (where X is ICELAKE, HASWELL,
 WESTMERE, ARM64, PPC64 and FALLBACK).
 
 The simdjson library automatically sets header flags for each implementation as it compiles; there
@@ -50,9 +51,9 @@ Inspecting the Detected Implementation
 You can check what implementation is running with `active_implementation`:
 
 ```c++
-cout << "simdjson v" << STRINGIFY(SIMDJSON_VERSION) << endl;
-cout << "Detected the best implementation for your machine: " << simdjson::active_implementation->name();
-cout << "(" << simdjson::active_implementation->description() << ")" << endl;
+cout << "simdjson v" << SIMDJSON_STRINGIFY(SIMDJSON_VERSION) << endl;
+cout << "Detected the best implementation for your machine: " << simdjson::get_active_implementation()->name();
+cout << "(" << simdjson::get_active_implementation()->description() << ")" << endl;
 ```
 
 Implementation detection will happen in this case when you first call `name()`.
@@ -63,7 +64,7 @@ Querying Available Implementations
 You can list all available implementations, regardless of which one was selected:
 
 ```c++
-for (auto implementation : simdjson::available_implementations) {
+for (auto implementation : simdjson::get_available_implementations()) {
   cout << implementation->name() << ": " << implementation->description() << endl;
 }
 ```
@@ -71,10 +72,10 @@ for (auto implementation : simdjson::available_implementations) {
 And look them up by name:
 
 ```c++
-cout << simdjson::available_implementations["fallback"]->description() << endl;
+cout << simdjson::get_available_implementations()["fallback"]->description() << endl;
 ```
 Though the fallback implementation should always be available, others might be missing. When
-an implementation is not available, the bracket call `simdjson::available_implementations[name]`
+an implementation is not available, the bracket call `simdjson::get_available_implementations()[name]`
 will return the null pointer.
 
 The available implementations have been compiled but may not necessarily be run safely on your system
@@ -90,18 +91,18 @@ can select the CPU architecture yourself:
 
 ```c++
 // Use the fallback implementation, even though my machine is fast enough for anything
-simdjson::active_implementation = simdjson::available_implementations["fallback"];
+simdjson::get_active_implementation() = simdjson::get_available_implementations()["fallback"];
 ```
 
 You are responsible for ensuring that the requirements of the selected implementation match your current system.
-Furthermore, you should check that the implementation is available before setting it to `simdjson::active_implementation`
+Furthermore, you should check that the implementation is available before setting it to `simdjson::get_active_implementation()`
 by comparing it with the null pointer.
 
 ```c++
-auto my_implementation = simdjson::available_implementations["haswell"];
+auto my_implementation = simdjson::get_available_implementations()["haswell"];
 if(! my_implementation) { exit(1); }
 if(! my_implementation->supported_by_runtime_system()) { exit(1); }
-simdjson::active_implementation = my_implementation;
+simdjson::get_active_implementation() = my_implementation;
 ```
 
 Checking that an Implementation can Run on your System
@@ -110,12 +111,12 @@ Checking that an Implementation can Run on your System
 You should call `supported_by_runtime_system()` to compare the processor's features with the need of the implementation.
 
 ```c++
-for (auto implementation : simdjson::available_implementations) {
+for (auto implementation : simdjson::get_available_implementations()) {
   if(implementation->supported_by_runtime_system()) {
     cout << implementation->name() << ": " << implementation->description() << endl;
   }
 }
 ```
 
-The call to `supported_by_runtime_system()` maybe relatively expensive. Do not call  `supported_by_runtime_system()` each
-time you parse a JSON input (for example). It is meant to be called a handful of times at most in the life of a program.
+The call to `supported_by_runtime_system()` may be relatively expensive. Do not call  `supported_by_runtime_system()` each
+time you parse a JSON input (for example). It is meant to be called a handful of times at most in the life of a program.

doc/iterate_many.md

@@ -1,7 +1,9 @@
 iterate_many
 ==========
 
-An interface providing features to work with files or streams containing multiple small JSON documents. Given an input such as
+When serializing large databases, it is often better to write out many independent JSON
+documents, instead of one large monolithic document containing many records. The simdjson
+library provides high-speed access to files or streams containing multiple small JSON documents separated by ASCII white-space characters. Given an input such as
 ```JSON
 {"text":"a"}
 {"text":"b"}
@@ -114,7 +116,9 @@ Whitespace Characters:
 - **Linefeed**
 - **Carriage return**
 - **Horizontal tab**
-- **Nothing**
+
+If your documents are all objects or arrays, then you may even have nothing between them.
+E.g., `[1,2]{"32":1}` is recognized as two documents.
 
 Some official formats **(non-exhaustive list)**:
 - [Newline-Delimited JSON (NDJSON)](http://ndjson.org/)

doc/ondemand_design.md

@@ -709,42 +709,6 @@ in production systems:
     }
 ```
 
-
-### Long-Running Processes and Memory Capacity
-
-The On Demand approach also automatically expands its memory capacity when larger documents are parsed. However, for longer processes where very large files are processed (such as server loops), this capacity is not resized down. Similarly to the DOM-based approach (see [here](https://github.com/simdjson/simdjson/blob/master/doc/dom.md#server-loops-long-running-processes-and-memory-capacity)]), On Demand also lets you adjust the maximal capacity that the parser can process:
-
-* You can set an upper bound (*max_capacity*) when construction the parser:
-```C++
-    ondemand::parser parser(1000*1000);  // Never grows past documents > 1 MB
-    auto doc = parser.iterate(json);
-    for (web_request request : listen()) {
-      padded_string json;
-      padded_string json = padded_string::load(request.body);
-      auto error = parser.iterate(json);
-      // If the document was above our limit, emit 413 = payload too large
-      if (error == CAPACITY) { request.respond(413); continue; }
-      // ...
-    }
-```
-
-The capacity will grow as the parser encounters larger documents up to 1 MB.
-
-* You can also allocate a *fixed capacity* that will never grow:
-```C++
-    ondemand::parser parser(1000*1000);
-    parser.allocate(1000*1000)  // Fix the capacity to 1 MB
-    auto doc = parser.iterate(json);
-    for (web_request request : listen()) {
-      padded_string json;
-      padded_string json = padded_string::load(request.body);
-      auto error = parser.iterate(json);
-      // If the document was above our limit, emit 413 = payload too large
-      if (error == CAPACITY) { request.respond(413); continue; }
-      // ...
-    }
-```
-You can also manually set the maximal capacity using the method `set_max_capacity()`.
 ### Benefits of the On Demand Approach
 
 We expect that the On Demand approach has many of the performance benefits of the schema-based approach, while providing a flexibility that is similar to that of the DOM-based approach.
@@ -764,7 +728,6 @@ There are currently additional technical limitations which we expect to resolve
 
 * The simdjson library offers runtime dispatching which allows you to compile one binary and have it run at full speed on different processors, taking advantage of the specific features of the processor. The On Demand API has limited runtime dispatch support. Under x64 systems, to fully benefit from the On Demand API, we recommend that you compile your code for a specific processor. E.g., if your processor supports AVX2 instructions, you should compile your binary executable with AVX2 instruction support (by using your compiler's commands). If you are sufficiently technically proficient, you can implement runtime dispatching within your application, by compiling your On Demand code for different processors.
 * There is an initial phase which scans the entire document quickly, irrespective of the size of the document. We plan to break this phase into distinct steps for large files in a future release as we have done with other components of our API (e.g., `parse_many`).
-* The On Demand API does not support JSON Pointer. This capability is currently limited to our core API.
 
 ### Applicability of the On Demand Approach
 

doc/performance.md

@@ -56,6 +56,43 @@ or simply
 ```
 
 
+Server Loops: Long-Running Processes and Memory Capacity
+---------------------------------
+
+The On Demand approach also automatically expands its memory capacity when larger documents are parsed. However, for longer processes where very large files are processed (such as server loops), this capacity is not resized down. On Demand also lets you adjust the maximal capacity that the parser can process:
+
+* You can set an upper bound (*max_capacity*) when construction the parser:
+```C++
+    ondemand::parser parser(1000*1000);  // Never grows past documents > 1 MB
+    auto doc = parser.iterate(json);
+    for (web_request request : listen()) {
+      padded_string json;
+      padded_string json = padded_string::load(request.body);
+      auto error = parser.iterate(json);
+      // If the document was above our limit, emit 413 = payload too large
+      if (error == CAPACITY) { request.respond(413); continue; }
+      // ...
+    }
+```
+
+The capacity will grow as the parser encounters larger documents up to 1 MB.
+
+* You can also allocate a *fixed capacity* that will never grow:
+```C++
+    ondemand::parser parser(1000*1000);
+    parser.allocate(1000*1000)  // Fix the capacity to 1 MB
+    auto doc = parser.iterate(json);
+    for (web_request request : listen()) {
+      padded_string json;
+      padded_string json = padded_string::load(request.body);
+      auto error = parser.iterate(json);
+      // If the document was above our limit, emit 413 = payload too large
+      if (error == CAPACITY) { request.respond(413); continue; }
+      // ...
+    }
+```
+You can also manually set the maximal capacity using the method `set_max_capacity()`.
+
 Large files and huge page support
 ---------------------------------
 
@@ -123,6 +160,8 @@ You should not expect the simdjson library to cause *downclocking* of your recen
 - [Whenever 512-bit AVX-512 instructions are used](https://lemire.me/blog/2018/09/07/avx-512-when-and-how-to-use-these-new-instructions/).
 - Whenever heavy 256-bit or wider instructions are used. Heavy instructions are those involving floating point operations or integer multiplications (since these execute on the floating point unit).
 
-The simdjson library does not currently support AVX-512 instructions and it does not make use of heavy 256-bit instructions. We do use vectorized multiplications, but only using 128-bit registers. Thus there should be no downclocking due to simdjson on recent processors.
+The simdjson library does not make use of heavy 256-bit instructions. We do use vectorized multiplications, but only using 128-bit registers. Thus there should be no downclocking due to simdjson on recent processors, except when AVX-512 is allowed and
+detected. However, we only allow AVX-512 on recent processors (Ice Lake/Tiger Lake or better) where [little to no frequency throttling is expected](https://travisdowns.github.io/blog/2020/08/19/icl-avx512-freq.html). If you can still concerned, you can easily disable AVX-512 with the CMake option `SIMDJSON_AVX512_ALLOWED` set to `OFF` (e.g., `cmake -D SIMDJSON_AVX512_ALLOWED=OFF -B build && cmake --build build`) or by setting
+the macro `SIMDJSON_AVX512_ALLOWED` to `0` in C++ prior to importing the headers.
 
 You may still be worried about which SIMD instruction set is used by simdjson.  Thankfully,  [you can always determine and change which architecture-specific implementation is used](implementation-selection.md) by simdjson. Thus even if your CPU supports AVX2, you do not need to use AVX2. You are in control.