Releasing 0.7.0.

fix: small typo error (#1328 )
more thread testing (#1324 )
2026-06-08 17:27:07 +00:00 · 2020-12-04 18:56:15 -05:00 · 2020-12-03 13:16:45 -05:00 · 2020-12-02 11:33:36 -05:00 · 2020-12-01 19:09:55 -05:00 · 2020-12-01 15:59:20 -05:00
219 changed files with 14179 additions and 3438 deletions
@@ -4,7 +4,13 @@ configuration: Release
 image: Visual Studio 2019
 platform: x64

+cache:
+  - C:\dependencies -> dependencies\CMakeLists.txt
+
 environment:
+  # Forward slash is used because this is used in CMake as is
+  simdjson_DEPENDENCY_CACHE_DIR: C:/dependencies
+
  matrix:
    - job_name: VS2019
      CMAKE_ARGS:  -A %Platform%
@@ -18,6 +24,10 @@ environment:
      platform: Win32
      CMAKE_ARGS:  -A %Platform% -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_ENABLE_THREADS=ON # This should be the default. Testing anyway.
      CTEST_ARGS: -E "checkperf|ondemand_basictests"
+    - job_name: VS2019 (Win32, No Exceptions)
+      platform: Win32
+      CMAKE_ARGS:  -A %Platform% -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_ENABLE_THREADS=ON -DSIMDJSON_EXCEPTIONS=OFF
+      CTEST_ARGS: -E "checkperf|ondemand_basictests"
    - job_name: VS2015
      image: Visual Studio 2015
      CMAKE_ARGS:  -A %Platform% -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_ENABLE_THREADS=OFF
@@ -61,6 +61,19 @@ executors:

 # Reusable test commands (and initializer for clang 6)
 commands:
+  dependency_restore:
+    steps:
+      - restore_cache:
+          keys:
+            - cmake-cache-{{ checksum "dependencies/CMakeLists.txt" }}
+
+  dependency_cache:
+    steps:
+      - save_cache:
+          key: cmake-cache-{{ checksum "dependencies/CMakeLists.txt" }}
+          paths:
+            - dependencies/.cache
+
  install_cmake:
    steps:
      - run: apt-get update -qq
@@ -74,7 +87,9 @@ commands:
  cmake_build_cache:
    steps:
      - cmake_prep
+      - dependency_restore
      - run: cmake $CMAKE_FLAGS -DCMAKE_INSTALL_PREFIX:PATH=destination -B build . 
+      - dependency_cache # dependencies are produced in the configure step

  cmake_build:
    steps:
@@ -166,7 +181,16 @@ jobs:
    executor: clang10
    environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -E checkperf }
    steps: [ cmake_test ]
-
+  threadsanitize-gcc10:
+    description: Build and run tests on GCC 10 and AVX 2 with a cmake sanitize build
+    executor: gcc10
+    environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE_THREADS=ON, BUILD_FLAGS: "", CTEST_FLAGS: --output-on-failure -E checkperf }
+    steps: [ cmake_test ]
+  threadsanitize-clang10:
+    description: Build and run tests on clang 10 and AVX 2 with a cmake sanitize build
+    executor: clang10
+    environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE_THREADS=ON, CTEST_FLAGS: --output-on-failure -E checkperf }
+    steps: [ cmake_test ]
  # dynamic
  dynamic-gcc10:
    description: Build and run tests on GCC 10 and AVX 2 with a cmake dynamic build
@@ -245,6 +269,8 @@ workflows:
      # full single-implementation tests
      - sanitize-gcc10
      - sanitize-clang10
+      - threadsanitize-gcc10
+      - threadsanitize-clang10
      - dynamic-gcc10
      - dynamic-clang10
      - unthreaded-gcc10
@@ -6,6 +6,11 @@ task:

  env:
    ASSUME_ALWAYS_YES: YES
+    simdjson_DEPENDENCY_CACHE_DIR: $HOME/.dep_cache
+  dep_cache:
+    folder: $HOME/.dep_cache
+    reupload_on_changes: false
+    fingerprint_script: cat dependencies/CMakeLists.txt
  setup_script:
    - pkg update -f
    - pkg install bash
@@ -1,7 +1,7 @@
 *
 !.git
 !Makefile
-!amalgamate.sh
+!amalgamate.py
 !benchmark
 !dependencies
 !include
@@ -12,4 +12,4 @@
 !src
 !style
 !tests
-!tools
+!tools
@@ -11,8 +11,8 @@ steps:
    CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=ON
    CTEST_FLAGS: -j4 --output-on-failure -E checkperf 
  commands:
-    - apt-get update -qq
-    - apt-get install -y g++ cmake gcc
+    - scripts/addcmakeppa.sh "$(env -i sh -c '. /etc/os-release; echo $VERSION_CODENAME')"
+    - apt-get install -y g++ cmake gcc git
    - mkdir build
    - cd build
    - cmake $CMAKE_FLAGS ..
@@ -32,8 +32,8 @@ steps:
    CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=ON
    CTEST_FLAGS: -j4 --output-on-failure -E checkperf 
  commands:
-    - apt-get update -qq
-    - apt-get install -y clang++-6.0 cmake
+    - scripts/addcmakeppa.sh "$(env -i sh -c '. /etc/os-release; echo $VERSION_CODENAME')"
+    - apt-get install -y clang++-6.0 cmake git
    - mkdir build
    - cd build
    - cmake $CMAKE_FLAGS ..
@@ -53,8 +53,9 @@ steps:
    CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback
    CTEST_FLAGS: -j4 --output-on-failure -E checkperf
  commands:
+    - echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list
    - apt-get update -qq
-    - apt-get install -y cmake
+    - apt-get -t buster-backports install -y cmake
    - mkdir build
    - cd build
    - cmake $CMAKE_FLAGS ..
@@ -102,8 +103,9 @@ steps:
    CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF
    CTEST_FLAGS: -j4 --output-on-failure -E checkperf
  commands:
+    - echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list
    - apt-get update -qq
-    - apt-get install -y cmake
+    - apt-get -t buster-backports install -y cmake
    - mkdir build
    - cd build
    - cmake $CMAKE_FLAGS ..
@@ -143,8 +145,9 @@ steps:
    CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback
    CTEST_FLAGS: -j4 --output-on-failure -E checkperf
  commands:
+    - echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list
    - apt-get update -qq
-    - apt-get install -y cmake
+    - apt-get -t buster-backports install -y cmake
    - mkdir build
    - cd build
    - cmake $CMAKE_FLAGS ..
@@ -180,6 +183,27 @@ steps:
    - ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -LE "acceptance|per_implementation" # Everything we haven't run yet, run now.
 ---
 kind: pipeline
+name: cpp20-clang11-libcpp
+platform: { os: linux, arch: amd64 }
+steps:
+- name: Build and Test
+  image: pauldreik/llvm-11
+  user: root
+  environment:
+    CC: clang-11
+    CXX: clang++-11
+    CMAKE_FLAGS: -GNinja
+    BUILD_FLAGS:
+    CTEST_FLAGS: -j4 --output-on-failure -E checkperf
+    CXXFLAGS: -std=c++20 -stdlib=libc++
+  commands:
+    - mkdir build
+    - cd build
+    - cmake $CMAKE_FLAGS ..
+    - cmake --build . $BUILD_FLAGS
+    - ctest $CTEST_FLAGS
+---
+kind: pipeline
 name: arm64-gcc8
 platform: { os: linux, arch: arm64 }
 steps:
@@ -192,8 +216,9 @@ steps:
    CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_IMPLEMENTATION=arm64;fallback
    CTEST_FLAGS: -j4 --output-on-failure -E checkperf
  commands:
+    - echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list
    - apt-get update -qq
-    - apt-get install -y cmake
+    - apt-get -t buster-backports install -y cmake
    - mkdir build
    - cd build
    - cmake $CMAKE_FLAGS ..
@@ -208,7 +233,7 @@ name: arm64-clang6
 platform: { os: linux, arch: arm64 }
 steps:
 - name: Build and Test
-  image: ubuntu:18.04
+  image: debian:buster-backports
  environment:
    CC: clang-6.0
    CXX: clang++-6.0
@@ -216,8 +241,9 @@ steps:
    BUILD_FLAGS: -- -j
    CTEST_FLAGS: -j4 --output-on-failure  -E checkperf
  commands:
-    - apt-get update -qq
-    - apt-get install -y clang cmake git
+    - apt-get -qq update
+    - apt-get -t buster-backports install -y cmake
+    - apt-get install -y clang-6.0 git
    - mkdir build
    - cd build
    - cmake $CMAKE_FLAGS ..
@@ -237,8 +263,9 @@ steps:
    CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF
    CTEST_FLAGS: -j4 --output-on-failure -E checkperf
  commands:
+    - echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list
    - apt-get update -qq
-    - apt-get install -y cmake
+    - apt-get -t buster-backports install -y cmake
    - mkdir build
    - cd build
    - cmake $CMAKE_FLAGS ..
@@ -250,7 +277,7 @@ name: arm64-dynamic-clang6
 platform: { os: linux, arch: arm64 }
 steps:
 - name: Build and Test
-  image: ubuntu:18.04
+  image: debian:buster-backports
  environment:
    CC: clang-6.0
    CXX: clang++-6.0
@@ -258,8 +285,9 @@ steps:
    BUILD_FLAGS: -- -j
    CTEST_FLAGS: -j4 --output-on-failure -E checkperf
  commands:
-    - apt-get update -qq
-    - apt-get install -y clang cmake git
+    - apt-get -qq update
+    - apt-get -t buster-backports install -y cmake
+    - apt-get install -y clang-6.0 git
    - mkdir build
    - cd build
    - cmake $CMAKE_FLAGS ..
@@ -279,8 +307,10 @@ steps:
    CC: gcc
    CXX: g++
  commands:
+    - echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list
    - apt-get update -qq
-    - apt-get install -y cmake libstdc++6
+    - apt-get -t buster-backports install -y cmake
+    - apt-get install -y libstdc++6
    - mkdir build
    - cd build
    - cmake $CMAKE_FLAGS ..
@@ -295,7 +325,7 @@ name: arm64-sanitize-clang6
 platform: { os: linux, arch: arm64 }
 steps:
 - name: Build and Test
-  image: ubuntu:18.04
+  image: debian:buster-backports
  environment:
    CC: clang-6.0
    CXX: clang++-6.0
@@ -303,8 +333,9 @@ steps:
    BUILD_FLAGS: -- -j
    CTEST_FLAGS: -j4 --output-on-failure -E checkperf
  commands:
-    - apt-get update -qq
-    - apt-get install -y clang cmake git
+    - apt-get -qq update
+    - apt-get -t buster-backports install -y cmake
+    - apt-get install -y clang-6.0 git
    - mkdir build
    - cd build
    - cmake $CMAKE_FLAGS ..
@@ -329,8 +360,6 @@ steps:
    CTEST_FLAGS: -j4 --output-on-failure -E checkperf
    CXXFLAGS: -stdlib=libc++
  commands:
-    - apt-get update -qq
-    - apt-get install -y cmake
    - mkdir build
    - cd build
    - cmake $CMAKE_FLAGS ..
@@ -352,8 +381,27 @@ steps:
    CTEST_FLAGS: -j4 --output-on-failure -E checkperf
    CXXFLAGS: -stdlib=libc++
  commands:
-    - apt-get update -qq
-    - apt-get install -y cmake
+    - mkdir build
+    - cd build
+    - cmake $CMAKE_FLAGS ..
+    - cmake --build . $BUILD_FLAGS
+    - ctest $CTEST_FLAGS
+---
+kind: pipeline
+name: libcpp-clang7
+platform: { os: linux, arch: amd64 }
+steps:
+- name: Build and Test
+  image: conanio/clang7
+  user: root
+  environment:
+    CC: clang-7
+    CXX: clang++-7
+    BUILD_FLAGS: -- -j
+    CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=ON
+    CTEST_FLAGS: -j4 --output-on-failure -E checkperf
+    CXXFLAGS: -stdlib=libc++
+  commands:
    - mkdir build
    - cd build
    - cmake $CMAKE_FLAGS ..
@@ -373,8 +421,9 @@ steps:
    CMAKE_FLAGS: -DSIMDJSON_EXCEPTIONS=OFF
    CTEST_FLAGS: -j4 --output-on-failure -E checkperf
  commands:
+    - echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list
    - apt-get update -qq
-    - apt-get install -y cmake
+    - apt-get -t buster-backports install -y cmake
    - mkdir build
    - cd build
    - cmake $CMAKE_FLAGS ..
@@ -393,7 +442,7 @@ steps:
    DEBIAN_FRONTEND: noninteractive
    ASAN_OPTIONS: detect_leaks=0
  commands:
-    - apt-get update -qq
+    - apt-get -qq update
    - apt-get install -q -y clang cmake git wget zip ninja-build
    - wget --quiet https://dl.bintray.com/pauldreik/simdjson-fuzz-corpus/corpus/corpus.tar
    - tar xf corpus.tar && rm corpus.tar
@@ -18,7 +18,7 @@ Before submitting an issue, please ensure that you have read the documentation:
 **Describe the bug**
 A clear and concise description of what the bug is.

-Note that a compiler warning is not a bug. 
+Note that a compiler warning is not a bug.

 **To Reproduce**
 Steps to reproduce the behaviour: provide a code sample if possible.
@@ -32,10 +32,10 @@ Note that a stack trace from your own program is not enough.
 - Compiler [e.g. Apple clang version 11.0.3 (clang-1103.0.32.59) x86_64-apple-darwin19.4.0]
 - Version [e.g. 22]

-We support up-to-date 64-bit ARM and x64 FreeBSD, macOS, Windows and Linux systems. Please ensure that your configuration is supported before labelling the issue as a bug. In particular, we do not support legacy 32-bit systems. 
+We support up-to-date 64-bit ARM and x64 FreeBSD, macOS, Windows and Linux systems. Please ensure that your configuration is supported before labelling the issue as a bug. In particular, we do not support legacy 32-bit systems.

 **Indicate whether you are willing or able to provide a bug fix as a pull request**

-If you plan to contribute to simdjson, please read our 
+If you plan to contribute to simdjson, please read our
 * CONTRIBUTING guide: https://github.com/simdjson/simdjson/blob/master/CONTRIBUTING.md and our
 * HACKING guide: https://github.com/simdjson/simdjson/blob/master/HACKING.md
@@ -32,6 +32,6 @@ A clear and concise description of any alternative solutions or features you've
 Add any other context or screenshots about the feature request here.

 ** Are you willing to contribute code or documentation toward this new feature? **
-If you plan to contribute to simdjson, please read our 
+If you plan to contribute to simdjson, please read our
 * CONTRIBUTING guide: https://github.com/simdjson/simdjson/blob/master/CONTRIBUTING.md and our
 * HACKING guide: https://github.com/simdjson/simdjson/blob/master/HACKING.md
@@ -27,6 +27,6 @@ Is your issue:
 4. A documentation issue? Can you suggest an improvement?


-If you plan to contribute to simdjson, please read our 
+If you plan to contribute to simdjson, please read our
 * CONTRIBUTING guide: https://github.com/simdjson/simdjson/blob/master/CONTRIBUTING.md and our
 * HACKING guide: https://github.com/simdjson/simdjson/blob/master/HACKING.md
@@ -0,0 +1,8 @@
+
+
+Our tests check whether you have introduced trailing white space. If such a test fails, please check the "artifacts button" above, which if you click it gives a link to a downloadable file to help you identify the issue. You can also run scripts/remove_trailing_whitespace.sh locally if you have a bash shell and the sed command available on your system.
+
+If you plan to contribute to simdjson, please read our
+
+CONTRIBUTING guide: https://github.com/simdjson/simdjson/blob/master/CONTRIBUTING.md and our
+HACKING guide: https://github.com/simdjson/simdjson/blob/master/HACKING.md
@@ -1,12 +1,22 @@
 name: Alpine Linux
-'on':
-  - push
-  - pull_request
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
 jobs:
  ubuntu-build:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
+      - uses: actions/cache@v2
+        with:
+          path: dependencies/.cache
+          key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
      - name: start docker
        run: |
          docker run -w /src -dit --name alpine -v $PWD:/src alpine:latest
@@ -24,4 +34,4 @@ jobs:
          ./alpine.sh cmake --build build_for_alpine
      - name: test
        run: |
-          ./alpine.sh bash -c "cd build_for_alpine && ctest"
+          ./alpine.sh bash -c "cd build_for_alpine && ctest  -E checkperf"
@@ -0,0 +1,37 @@
+name: Detect trailing whitespace
+
+on:
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  whitespace:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - name: Remove whitespace and check the diff
+        run: |
+          set -eu
+          scripts/remove_trailing_whitespace.sh
+          git diff >whitespace.patch
+          cat whitespace.patch
+          if [ $(wc -c <whitespace.patch) -ne 0 ] ; then
+             echo " !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! "
+             echo "You have trailing whitespace, please download the artifact"
+             echo "and apply with git apply <whitespace.patch or"
+             echo "run scripts/remove_trailing_whitespace.sh locally."
+             echo " !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! "
+             exit 1
+          else
+             echo "no trailing whitespace found, good!"
+          fi
+      - name: Archive whitespace patch
+        uses: actions/upload-artifact@v2
+        if: always()
+        with:
+          name: whitespace-patch
+          path: |
+            whitespace.patch
+          if-no-files-found: ignore
+
@@ -5,6 +5,8 @@ on:
    branches:
      - master
  pull_request:
+    branches:
+      - master
  schedule:
    - cron: 23 */8 * * *

@@ -12,13 +14,16 @@ jobs:
  build:    
    runs-on: ubuntu-latest
    env:
-      # fuzzers that use the default implementation
-      defaultimplfuzzers: atpointer dump dump_raw_tape element minify  parser print_json
-      # fuzzers that loop over the implementations themselves
-      implfuzzers: implementations minifyimpl utf8
+      # fuzzers that change behaviour with SIMDJSON_FORCE_IMPLEMENTATION
+      defaultimplfuzzers: atpointer dump dump_raw_tape element minify parser print_json
+      # fuzzers that loop over the implementations themselves, or don't need to switch.
+      implfuzzers: implementations minifyimpl ndjson ondemand padded utf8
      implementations: haswell westmere fallback
      UBSAN_OPTIONS: halt_on_error=1
      MAXLEN: -max_len=4000
+      CLANGVERSION: 11
+      # which optimization level to use for the sanitizer build (see build_fuzzer.variants.sh)
+      OPTLEVEL: -O3

    steps:
    - name: Install packages necessary for building
@@ -27,10 +32,15 @@ jobs:
        sudo apt-get install --quiet ninja-build valgrind zip unzip
        wget https://apt.llvm.org/llvm.sh
        chmod +x llvm.sh
-        sudo ./llvm.sh 10
+        sudo ./llvm.sh $CLANGVERSION

    - uses: actions/checkout@v1

+    - uses: actions/cache@v2
+      with:
+        path: dependencies/.cache
+        key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
+
    - name: Create and prepare the initial seed corpus
      run: |
        fuzz/build_corpus.sh
@@ -51,7 +61,7 @@ jobs:
        clang++ --version

    - name: Build all the variants
-      run: fuzz/build_fuzzer_variants.sh
+      run: CLANGSUFFIX=-$CLANGVERSION fuzz/build_fuzzer_variants.sh

    - name: Explore fast (release build, default implementation)
      run: |
@@ -71,7 +81,7 @@ jobs:
          others=$(find out -type d -not -name $fuzzer -not -name out -not -name cmin)
          for implementation in $implementations; do
            export SIMDJSON_FORCE_IMPLEMENTATION=$implementation
-            build-sanitizers/fuzz/fuzz_$fuzzer out/$fuzzer $others seedcorpus -max_total_time=20 $MAXLEN
+            build-sanitizers$OPTLEVEL/fuzz/fuzz_$fuzzer out/$fuzzer $others seedcorpus -max_total_time=20 $MAXLEN
          done
          echo now have $(ls out/$fuzzer |wc -l) files in corpus        
        done
@@ -82,7 +92,7 @@ jobs:
        for fuzzer in $implfuzzers; do
          # get input from everyone else (corpus cross pollination)
          others=$(find out -type d -not -name $fuzzer -not -name out -not -name cmin)
-          build-sanitizers/fuzz/fuzz_$fuzzer out/$fuzzer $others seedcorpus -max_total_time=20 $MAXLEN
+          build-sanitizers$OPTLEVEL/fuzz/fuzz_$fuzzer out/$fuzzer $others seedcorpus -max_total_time=20 $MAXLEN
          echo now have $(ls out/$fuzzer |wc -l) files in corpus
        done

@@ -149,4 +159,3 @@ jobs:
          leak-*
          timeout-*
        if-no-files-found: ignore
-
@@ -1,6 +1,12 @@
 name: MinGW32-CI

-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master

 # Important: scoop will either install 32-bit GCC or 64-bit GCC, not both.

@@ -26,6 +32,12 @@ jobs:
          path: |
            C:\ProgramData\scoop
          key: scoop32 # static key: should be good forever
+
+      - uses: actions/cache@v2
+        with:
+          path: dependencies/.cache
+          key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
+
      - name: Setup Windows # This should almost never run if the cache works.
        if: steps.cache.outputs.cache-hit != 'true'
        shell: powershell
@@ -1,6 +1,12 @@
 name: MinGW64-CI

-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master

 # Important: scoop will either install 32-bit GCC or 64-bit GCC, not both.

@@ -26,6 +32,12 @@ jobs:
          path: |
            C:\ProgramData\scoop
          key: scoop64 # static key: should be good forever
+
+      - uses: actions/cache@v2
+        with:
+          path: dependencies/.cache
+          key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
+
      - name: Setup Windows # This should almost never run if the cache works.
        if: steps.cache.outputs.cache-hit != 'true'
        shell: powershell
@@ -1,6 +1,12 @@
 name: MSYS2-CI

-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master

 jobs:
  windows-mingw:
@@ -30,6 +36,10 @@ jobs:

    steps:
      - uses: actions/checkout@v2
+      - uses: actions/cache@v2
+        with:
+          path: dependencies/.cache
+          key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
      - uses: msys2/setup-msys2@v2
        with:
          update: true
@@ -1,16 +1,22 @@
 name: Performance check on Ubuntu 18.04 CI (GCC 7)

-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master

 jobs:
  ubuntu-build:
    runs-on: ubuntu-18.04
    steps:
      - uses: actions/checkout@v2
-      - name: Setup cmake
-        uses: jwlawson/actions-setup-cmake@v1.0
+      - uses: actions/cache@v2
        with:
-          cmake-version: '3.9.x'
+          path: dependencies/.cache
+          key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
      - name: Use cmake
        run: |
          mkdir build &&
@@ -18,4 +24,3 @@ jobs:
          cmake  -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_BUILD_STATIC=ON -DCMAKE_INSTALL_PREFIX:PATH=destination ..  &&
          cmake --build . --target checkperf  &&
          ctest --output-on-failure -R checkperf  ubuntu18-checkperf.yml
-
@@ -0,0 +1,27 @@
+name: Ubuntu 18.04 CI (GCC 7) with Thread Sanitizer
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  ubuntu-build:
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/cache@v2
+        with:
+          path: dependencies/.cache
+          key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
+      - name: Use cmake
+        run: |
+          mkdir build &&
+          cd build &&
+          cmake  -DSIMDJSON_SANITIZE_THREADS=ON ..  &&
+          cmake --build . --target document_stream_tests --target parse_many_test  &&
+          ctest --output-on-failure  -R parse_many_test  &&
+          ctest --output-on-failure  -R document_stream_tests
@@ -1,16 +1,22 @@
 name: Ubuntu 18.04 CI (GCC 7)

-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master

 jobs:
  ubuntu-build:
    runs-on: ubuntu-18.04
    steps:
      - uses: actions/checkout@v2
-      - name: Setup cmake
-        uses: jwlawson/actions-setup-cmake@v1.0
+      - uses: actions/cache@v2
        with:
-          cmake-version: '3.9.x'
+          path: dependencies/.cache
+          key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
      - name: Use cmake
        run: |
          mkdir build &&
@@ -1,16 +1,22 @@
 name: Performance check on Ubuntu 20.04 CI (GCC 9)

-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master

 jobs:
  ubuntu-build:
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@v2
-      - name: Setup cmake
-        uses: jwlawson/actions-setup-cmake@v1.0
+      - uses: actions/cache@v2
        with:
-          cmake-version: '3.9.x'
+          path: dependencies/.cache
+          key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
      - name: Use cmake
        run: |
          mkdir build &&
@@ -0,0 +1,27 @@
+name: Ubuntu 20.04 CI (GCC 9) with Thread Sanitizer
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  ubuntu-build:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/cache@v2
+        with:
+          path: dependencies/.cache
+          key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
+      - name: Use cmake
+        run: |
+          mkdir build &&
+          cd build &&
+          cmake  -DSIMDJSON_SANITIZE_THREADS=ON ..  &&
+          cmake --build . --target document_stream_tests --target parse_many_test  &&
+          ctest --output-on-failure  -R parse_many_test  &&
+          ctest --output-on-failure  -R document_stream_tests
@@ -1,16 +1,22 @@
 name: Ubuntu 20.04 CI (GCC 9)

-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master

 jobs:
  ubuntu-build:
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@v2
-      - name: Setup cmake
-        uses: jwlawson/actions-setup-cmake@v1.0
+      - uses: actions/cache@v2
        with:
-          cmake-version: '3.9.x'
+          path: dependencies/.cache
+          key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
      - name: Use cmake
        run: |
          mkdir build &&
@@ -1,6 +1,12 @@
 name: VS16-CI

-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master

 jobs:
  ci:
@@ -8,6 +14,10 @@ jobs:
    runs-on: windows-latest
    steps:
    - uses: actions/checkout@v2
+    - uses: actions/cache@v2
+      with:
+        path: dependencies/.cache
+        key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
    - name: 'Run CMake with VS16'
      uses: lukka/run-cmake@v2
      with:
@@ -1,6 +1,12 @@
 name: VS16-CLANG-CI

-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master

 jobs:
  ci:
@@ -8,6 +14,10 @@ jobs:
    runs-on: windows-latest
    steps:
    - uses: actions/checkout@v2
+    - uses: actions/cache@v2
+      with:
+        path: dependencies/.cache
+        key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
    - name: 'Run CMake with VS16'
      uses: lukka/run-cmake@v2
      with:
@@ -22,4 +32,4 @@ jobs:
        
    - name: 'Run CTest'
      run: ctest -C Release -E checkperf  --output-on-failure 
-      working-directory: "${{ github.workspace }}/../../_temp/windows"
+      working-directory: "${{ github.workspace }}/../../_temp/windows"
@@ -1,6 +1,12 @@
 name: VS16-Ninja-CI

-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master

 jobs:
  ci:
@@ -8,6 +14,10 @@ jobs:
    runs-on: windows-latest
    steps:
    - uses: actions/checkout@v2
+    - uses: actions/cache@v2
+      with:
+        path: dependencies/.cache
+        key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
    - name: 'Run CMake with VS16'
      uses: lukka/run-cmake@v2
      with:
@@ -96,3 +96,4 @@ objs

 # Generated docs
 /doc/api
+*.orig
@@ -1,42 +0,0 @@
-[submodule "scalarvssimd/rapidjson"]
-	path = dependencies/rapidjson
-	url = https://github.com/Tencent/rapidjson.git
-[submodule "dependencies/sajson"]
-	path = dependencies/sajson
-	url = https://github.com/chadaustin/sajson.git
-[submodule "dependencies/json11"]
-	path = dependencies/json11
-	url = https://github.com/dropbox/json11.git
-[submodule "dependencies/fastjson"]
-	path = dependencies/fastjson
-	url = https://github.com/mikeando/fastjson.git
-[submodule "dependencies/gason"]
-	path = dependencies/gason
-	url = https://github.com/vivkin/gason.git
-[submodule "dependencies/ujson4c"]
-	path = dependencies/ujson4c
-	url = https://github.com/esnme/ujson4c.git
-[submodule "dependencies/jsmn"]
-	path = dependencies/jsmn
-	url = https://github.com/zserge/jsmn.git
-[submodule "dependencies/cJSON"]
-	path = dependencies/cJSON
-	url = https://github.com/DaveGamble/cJSON.git
-[submodule "dependencies/jsoncpp"]
-	path = dependencies/jsoncpp
-	url = https://github.com/open-source-parsers/jsoncpp.git
-[submodule "dependencies/json"]
-	path = dependencies/json
-	url = https://github.com/nlohmann/json.git
-[submodule "dependencies/benchmark"]
-	path = dependencies/benchmark
-	url = https://github.com/google/benchmark.git
-[submodule "dependencies/cxxopts"]
-	path = dependencies/cxxopts
-	url = https://github.com/jarro2783/cxxopts
-[submodule "dependencies/boost.json"]
-	path = dependencies/boost.json
-	url = https://github.com/CPPAlliance/json.git
-[submodule "dependencies/yyjson"]
-	path = dependencies/yyjson
-	url = https://github.com/ibireme/yyjson.git
@@ -1,30 +1,188 @@
 language: cpp
-sudo: false
-addons:
-  apt:
-    sources:
-      - ubuntu-toolchain-r-test
-    packages:
-      - gcc-7
-      - g++-7
-      - clang-format
-      - python
-branches:
-  only:
-    - master

-script: 
-   - export CXX=g++-7
-   - export CC=gcc-7
-   - make
-   - make test 
-   - make everything
-   - make amalgamate
-   - make clean
-   - make SANITIZEGOLD=1 test
-   - make clean
-   - ARCHFLAGS="-march=nehalem" make 
-   - ARCHFLAGS="-march=nehalem" make test
-   - ARCHFLAGS="-march=nehalem" make everything
-   - ./style/run-clang-format.py -r include/ benchmark/ src/ tests/
+dist: bionic

+arch:
+  - ppc64le
+
+cache:
+  directories:
+    - $HOME/.dep_cache
+
+env:
+  global:
+    - simdjson_DEPENDENCY_CACHE_DIR=$HOME/.dep_cache
+
+matrix:
+  include:
+    - os: linux
+      addons:
+        apt:
+          sources:
+            - ubuntu-toolchain-r-test
+          packages:
+            - g++-8
+      env:
+        - COMPILER="CC=gcc-8 && CXX=g++-8"
+      compiler: gcc-8
+
+    - os: linux
+      addons:
+        apt:
+          sources:
+            - ubuntu-toolchain-r-test
+          packages:
+            - g++-9
+      env:
+        - COMPILER="CC=gcc-9 && CXX=g++-9"
+      compiler: gcc-9
+
+    - os: linux
+      addons:
+        apt:
+          sources:
+            - ubuntu-toolchain-r-test
+          packages:
+            - g++-10
+      env:
+        - COMPILER="CC=gcc-10 && CXX=g++-10"
+      compiler: gcc-10
+
+    - os: linux
+      addons:
+        apt:
+          sources:
+            - ubuntu-toolchain-r-test
+          packages:
+            - g++-10
+      env:
+        - COMPILER="CC=gcc-10 && CXX=g++-10"
+        - SANITIZE="on"
+      compiler: gcc-10-sanitize
+
+    - os: linux
+      addons:
+        apt:
+          sources:
+            - ubuntu-toolchain-r-test
+          packages:
+            - g++-10
+      env:
+        - COMPILER="CC=gcc-10 && CXX=g++-10"
+        - STATIC="on"
+      compiler: gcc-10-static
+
+    - os: linux
+      addons:
+        apt:
+          sources:
+            - llvm-toolchain-bionic-6.0
+          packages:
+            - clang-6.0
+      env:
+        - COMPILER="CC=clang-6.0 && CXX=clang++-6.0"
+      compiler: clang-6
+
+    - os: linux
+      addons:
+        apt:
+          sources:
+            - llvm-toolchain-bionic-7
+          packages:
+            - clang-7
+      env:
+        - COMPILER="CC=clang-7 && CXX=clang++-7"
+      compiler: clang-7
+
+    - os: linux
+      addons:
+        apt:
+          sources:
+            - llvm-toolchain-bionic-8
+          packages:
+            - clang-8
+      env:
+        - COMPILER="CC=clang-8 && CXX=clang++-8"
+      compiler: clang-8
+
+    - os: linux
+      addons:
+        apt:
+          sources:
+            - llvm-toolchain-bionic-9
+          packages:
+            - clang-9
+      env:
+        - COMPILER="CC=clang-9 && CXX=clang++-9"
+      compiler: clang-9
+
+    - os: linux
+      addons:
+        apt:
+          packages:
+            - clang-10
+          sources:
+            - ubuntu-toolchain-r-test
+            - sourceline: 'deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-10 main'
+              key_url: 'https://apt.llvm.org/llvm-snapshot.gpg.key'
+      env:
+        - COMPILER="CC=clang-10 && CXX=clang++-10"
+      compiler: clang-10
+
+    - os: linux
+      addons:
+        apt:
+          packages:
+            - clang-10
+          sources:
+            - ubuntu-toolchain-r-test
+            - sourceline: 'deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-10 main'
+              key_url: 'https://apt.llvm.org/llvm-snapshot.gpg.key'
+      env:
+        - COMPILER="CC=clang-10 && CXX=clang++-10"
+        - STATIC="on"
+      compiler: clang-10-static
+
+    - os: linux
+      addons:
+        apt:
+          packages:
+            - clang-10
+          sources:
+            - ubuntu-toolchain-r-test
+            - sourceline: 'deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-10 main'
+              key_url: 'https://apt.llvm.org/llvm-snapshot.gpg.key'
+      env:
+        - COMPILER="CC=clang-10 && CXX=clang++-10"
+        - SANITIZE="on"
+      compiler: clang-10-sanitize
+
+before_install:
+  - eval "${COMPILER}"
+
+install:
+  - wget -q -O - "https://raw.githubusercontent.com/simdjson/debian-ppa/master/key.gpg" | sudo apt-key add -
+  - sudo apt-add-repository "deb https://raw.githubusercontent.com/simdjson/debian-ppa/master simdjson main"
+  - sudo apt-get -qq update
+  - sudo apt-get purge cmake cmake-data
+  - sudo apt-get -t simdjson -y install cmake
+  - export CMAKE_CXX_FLAGS="-maltivec -mcpu=power9 -mtune=power9"
+  - export CMAKE_C_FLAGS="${CMAKE_CXX_FLAGS}"
+  - export CMAKE_FLAGS="-DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS} -DSIMDJSON_IMPLEMENTATION=ppc64;fallback";
+  - if [[ "${SANITIZE}" == "on" ]]; then
+      export CMAKE_FLAGS="${CMAKE_FLAGS} -DSIMDJSON_SANITIZE=ON";
+      export ASAN_OPTIONS="detect_leaks=0";
+    fi
+  - if [[ "${STATIC}" == "on" ]]; then
+      export CMAKE_FLAGS="${CMAKE_FLAGS} -DSIMDJSON_BUILD_STATIC=ON";
+    fi
+  - export CTEST_FLAGS="-j4 --output-on-failure -E checkperf"
+
+script:
+  - mkdir build
+  - cd build
+  - cmake $CMAKE_FLAGS ..
+  - cmake --build . -- -j2
+  - SIMDJSON_FORCE_IMPLEMENTATION=ppc64 ctest $CTEST_FLAGS -L per_implementation
+  - SIMDJSON_FORCE_IMPLEMENTATION=fallback ctest $CTEST_FLAGS -L per_implementation
+  - ctest $CTEST_FLAGS -LE "acceptance|per_implementation"
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.9) # CMP0069 NEW
+cmake_minimum_required(VERSION 3.13)

 project(simdjson
  DESCRIPTION "Parsing gigabytes of JSON per second"
@@ -6,11 +6,11 @@ project(simdjson
 )

 set(PROJECT_VERSION_MAJOR 0)
-set(PROJECT_VERSION_MINOR 6)
-set(PROJECT_VERSION_PATCH 1)
-set(SIMDJSON_SEMANTIC_VERSION "0.6.1" CACHE STRING "simdjson semantic version")
-set(SIMDJSON_LIB_VERSION "4.0.0" CACHE STRING "simdjson library version")
-set(SIMDJSON_LIB_SOVERSION "4" CACHE STRING "simdjson library soversion")
+set(PROJECT_VERSION_MINOR 7)
+set(PROJECT_VERSION_PATCH 0)
+set(SIMDJSON_SEMANTIC_VERSION "0.7.0" CACHE STRING "simdjson semantic version")
+set(SIMDJSON_LIB_VERSION "5.0.0" CACHE STRING "simdjson library version")
+set(SIMDJSON_LIB_SOVERSION "5" CACHE STRING "simdjson library soversion")
 set(SIMDJSON_GITHUB_REPOSITORY https://github.com/simdjson/simdjson)

 include(GNUInstallDirs)
@@ -42,6 +42,7 @@ We have few hard rules, but we have some:
 - Calls to `abort()` are forbidden in the core library. This follows from the [Writing R Extensions](https://cran.r-project.org/doc/manuals/R-exts.html) manual which states that "Under no circumstances should your compiled code ever call abort or exit".
 - All source code files (.h, .cpp) must be ASCII.
 - All C macros introduced in public headers need to be prefixed with either `SIMDJSON_` or `simdjson_`.
+- We avoid trailing white space characters within lines. That is, your lines of code should not terminate with unnecessary spaces. Generally, please avoid making unnecessary changes to white-space characters when contributing code.

 Tools, tests and benchmarks are not held to these same strict rules.

@@ -35,5 +35,6 @@ Matjaž Ostroveršnik
 Nong Li
 Furkan Taşkale
 Brendan Knapp
+Danila Kutenin
 # if you have contributed to the project and your name does not 
 # appear in this list, please let us know!
@@ -38,7 +38,7 @@ PROJECT_NAME           = simdjson
 # could be handy for archiving the generated documentation or if some version
 # control system is used.

-PROJECT_NUMBER         = "0.6.1"
+PROJECT_NUMBER         = "0.7.0"

 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a
@@ -48,7 +48,7 @@ simdjson's source structure, from the top level, looks like this:
  implementations).
  * simdjson.cpp: A "main source" that includes all implementation files from src/. This is
    equivalent to the distributed simdjson.cpp.
-  * arm64/|fallback/|haswell/|westmere/: Architecture-specific implementations. All functions are
+  * arm64/|fallback/|haswell/|ppc64/|westmere/: Architecture-specific implementations. All functions are
    Each architecture defines its own namespace, e.g. simdjson::haswell.
  * generic/: Generic implementations of the simdjson parser. These files may be included and
    compiled multiple times, from whichever architectures use them. They assume they are already
@@ -67,7 +67,7 @@ Other important files and directories:
 * **.circleci:** Definitions for Circle CI.
 * **.github/workflows:** Definitions for GitHub Actions (CI).
 * **singleheader:** Contains generated `simdjson.h` and `simdjson.cpp` that we release. The files `singleheader/simdjson.h` and `singleheader/simdjson.cpp` should never be edited by hand.
-* **singleheader/amalgamate.sh:** Generates `singleheader/simdjson.h` and `singleheader/simdjson.cpp` for release (bash script). 
+* **singleheader/amalgamate.py:** Generates `singleheader/simdjson.h` and `singleheader/simdjson.cpp` for release (python script).
 * **benchmark:** This is where we do benchmarking. Benchmarking is core to every change we make; the
  cardinal rule is don't regress performance without knowing exactly why, and what you're trading
  for it. Many of our benchmarks are microbenchmarks. We are effectively doing controlled scientific experiments for the purpose of understanding what affects our performance. So we simplify as much as possible. We try to avoid irrelevant factors such as page faults, interrupts, unnnecessary system calls. We recommend checking the performance as follows:
@@ -169,12 +169,13 @@ systematically regenerated on releases. To ensure you have the latest code, you
 mkdir build
 cd build
 cmake ..
+cmake --build . # needed, because currently dependencies do not work fully for the amalgamate target
 cmake --build . --target amalgamate
 ```

-You need to have a working bash on your system.
+You need to have python3 installed on your system.

-The amalgamator script is `amalgamate.sh` generates singleheader/simdjson.h by
+The amalgamator script `amalgamate.py` generates singleheader/simdjson.h by
 reading through include/simdjson.h, copy/pasting each header file into the amalgamated file at the
 point it gets included (but only once per header). singleheader/simdjson.cpp is generated from
 src/simdjson.cpp the same way, except files under generic/ may be included and copy/pasted multiple
@@ -251,7 +252,7 @@ We assume you have a common 64-bit Windows PC with at least Visual Studio 2019.
 - Install [CMake](https://cmake.org/download/). When you install it, make sure to ask that `cmake` be made available from the command line. Please choose a recent version of cmake.
 - Create a subdirectory within simdjson, such as `build`.
 - Using a shell, go to this newly created directory. You can start a shell directly from GitHub Desktop (Repository > Open in Command Prompt).
- Type `cmake  ..` in the shell while in the `build` repository. 
+- Type `cmake  ..` in the shell while in the `build` repository.
 - This last command (`cmake ...`) created a Visual Studio solution file in the newly created directory (e.g., `simdjson.sln`). Open this file in Visual Studio. You should now be able to build the project and run the tests. For example, in the `Solution Explorer` window (available from the `View` menu), right-click `ALL_BUILD` and select `Build`. To test the code, still in the `Solution Explorer` window, select `RUN_TESTS` and select `Build`.


@@ -3,7 +3,7 @@
 [![Ubuntu 20.04 CI](https://github.com/simdjson/simdjson/workflows/Ubuntu%2020.04%20CI%20(GCC%209)/badge.svg)](https://simdjson.org/plots.html)
 ![VS16-CI](https://github.com/simdjson/simdjson/workflows/VS16-CI/badge.svg)
 ![MinGW64-CI](https://github.com/simdjson/simdjson/workflows/MinGW64-CI/badge.svg)
-[![][license img]][license]  [![Doxygen Documentation](https://img.shields.io/badge/docs-doxygen-green.svg)](https://simdjson.org/api/0.6.0/index.html)
+[![][license img]][license]  [![Doxygen Documentation](https://img.shields.io/badge/docs-doxygen-green.svg)](https://simdjson.org/api/0.7.0/index.html)

 simdjson : Parsing gigabytes of JSON per second
 ===============================================
@@ -40,10 +40,10 @@ Table of Contents
 Quick Start
 -----------

-          
+
 The simdjson library is easily consumable with a single .h and .cpp file.

-0. Prerequisites: `g++` (version 7 or better) or `clang++` (version 6 or better), and a 64-bit system with a command-line shell (e.g., Linux, macOS, freeBSD). We also support programming environnements like Visual Studio and Xcode, but different steps are needed.
+0. Prerequisites: `g++` (version 7 or better) or `clang++` (version 6 or better), and a 64-bit system with a command-line shell (e.g., Linux, macOS, freeBSD). We also support programming environments like Visual Studio and Xcode, but different steps are needed.
 1. Pull [simdjson.h](singleheader/simdjson.h) and [simdjson.cpp](singleheader/simdjson.cpp) into a directory, along with the sample file [twitter.json](jsonexamples/twitter.json).
   ```
   wget https://raw.githubusercontent.com/simdjson/simdjson/master/singleheader/simdjson.h https://raw.githubusercontent.com/simdjson/simdjson/master/singleheader/simdjson.cpp https://raw.githubusercontent.com/simdjson/simdjson/master/jsonexamples/twitter.json
@@ -73,7 +73,7 @@ Usage documentation is available:
 * [Performance](doc/performance.md) shows some more advanced scenarios and how to tune for them.
 * [Implementation Selection](doc/implementation-selection.md) describes runtime CPU detection and
  how you can work with it.
-* [API](https://simdjson.org/api/0.6.0/annotated.html) contains the automatically generated API documentation.
+* [API](https://simdjson.org/api/0.7.0/annotated.html) contains the automatically generated API documentation.

 Performance results
 -------------------
@@ -168,7 +168,7 @@ instructions, reducing branch misprediction, and reducing data dependency to tak
 CPU's multiple execution cores.

 Some people [enjoy reading our paper](https://arxiv.org/abs/1902.08318): A description of the design
-and implementation of simdjson is in our research article: 
+and implementation of simdjson is in our research article:
 - Geoff Langdale, Daniel Lemire, [Parsing Gigabytes of JSON per Second](https://arxiv.org/abs/1902.08318), VLDB Journal 28 (6), 2019.

 We have an in-depth paper focused on the UTF-8 validation:
@@ -4,7 +4,7 @@ link_libraries(simdjson-windows-headers test-data)

 if (TARGET benchmark::benchmark)
  add_executable(bench_sax bench_sax.cpp)
-  target_link_libraries(bench_sax simdjson-internal-flags simdjson-include-source benchmark::benchmark)
+  target_link_libraries(bench_sax PRIVATE simdjson-internal-flags simdjson-include-source benchmark::benchmark)
 endif (TARGET benchmark::benchmark)

 link_libraries(simdjson simdjson-flags)
@@ -24,19 +24,19 @@ target_compile_definitions(parse_nostringparsing PRIVATE SIMDJSON_SKIPSTRINGPARS

 if (TARGET competition-all)
  add_executable(distinctuseridcompetition distinctuseridcompetition.cpp)
-  target_link_libraries(distinctuseridcompetition competition-core)
+  target_link_libraries(distinctuseridcompetition PRIVATE competition-core)

  add_executable(minifiercompetition minifiercompetition.cpp)
-  target_link_libraries(minifiercompetition competition-core)
+  target_link_libraries(minifiercompetition PRIVATE competition-core)

  add_executable(parseandstatcompetition parseandstatcompetition.cpp)
-  target_link_libraries(parseandstatcompetition competition-core)
+  target_link_libraries(parseandstatcompetition PRIVATE competition-core)

  add_executable(parsingcompetition parsingcompetition.cpp)
-  target_link_libraries(parsingcompetition competition-core)
+  target_link_libraries(parsingcompetition PRIVATE competition-core)

  add_executable(allparsingcompetition parsingcompetition.cpp)
-  target_link_libraries(allparsingcompetition competition-all)
+  target_link_libraries(allparsingcompetition PRIVATE competition-all)
  target_compile_definitions(allparsingcompetition PRIVATE ALLPARSER)
 endif()

@@ -8,8 +8,8 @@ using namespace std;

 const padded_string EMPTY_ARRAY("[]", 2);

-const char *TWITTER_JSON = SIMDJSON_BENCHMARK_DATA_DIR "twitter.json";
-const char *NUMBERS_JSON = SIMDJSON_BENCHMARK_DATA_DIR "numbers.json";
+static const char *TWITTER_JSON = SIMDJSON_BENCHMARK_DATA_DIR "twitter.json";
+static const char *NUMBERS_JSON = SIMDJSON_BENCHMARK_DATA_DIR "numbers.json";

 static void recover_one_string(State& state) {
  dom::parser parser;
@@ -87,8 +87,8 @@ static void serialize_big_string_to_string(State& state) {
  std::vector<char> content;
  content.push_back('\"');
  for(size_t i = 0 ; i < 100000; i ++) {
-    content.push_back('0' + char(i%10)); // we add what looks like a long list of digits 
-  } 
+    content.push_back('0' + char(i%10)); // we add what looks like a long list of digits
+  }
  content.push_back('\"');
  dom::element doc;
  simdjson::error_code error;
@@ -139,7 +139,7 @@ static void serialize_twitter_to_string(State& state) {
  }
  // we validate the result
  {
-    auto serial = simdjson::to_string(doc); 
+    auto serial = simdjson::to_string(doc);
    dom::element doc2; // we parse the stringify output
    if ((error = parser.parse(serial).get(doc2))) { throw std::runtime_error("serialization error"); }
    auto serial2 = simdjson::to_string(doc2); // we stringify again
@@ -211,7 +211,7 @@ static void numbers_scan(State& state) {
    }
    benchmark::DoNotOptimize(container.data());
    benchmark::ClobberMemory();
-  }  
+  }
 }
 BENCHMARK(numbers_scan);

@@ -236,7 +236,7 @@ static void numbers_size_scan(State& state) {
    if(pos != container.size()) { cerr << "bad count" << endl; }
    benchmark::DoNotOptimize(container.data());
    benchmark::ClobberMemory();
-  }  
+  }
 }
 BENCHMARK(numbers_size_scan);

@@ -315,7 +315,7 @@ static void numbers_load_scan(State& state) {
    }
    benchmark::DoNotOptimize(container.data());
    benchmark::ClobberMemory();
-  }  
+  }
 }
 BENCHMARK(numbers_load_scan);

@@ -341,7 +341,7 @@ static void numbers_load_size_scan(State& state) {
    if(pos != container.size()) { cerr << "bad count" << endl; }
    benchmark::DoNotOptimize(container.data());
    benchmark::ClobberMemory();
-  }  
+  }
 }
 BENCHMARK(numbers_load_size_scan);

@@ -360,7 +360,7 @@ static void numbers_exceptions_scan(State& state) {
    }
    benchmark::DoNotOptimize(container.data());
    benchmark::ClobberMemory();
-  }  
+  }
 }
 BENCHMARK(numbers_exceptions_scan);

@@ -378,7 +378,7 @@ static void numbers_exceptions_size_scan(State& state) {
    if(pos != container.size()) { cerr << "bad count" << endl; }
    benchmark::DoNotOptimize(container.data());
    benchmark::ClobberMemory();
-  }  
+  }
 }
 BENCHMARK(numbers_exceptions_size_scan);

@@ -437,7 +437,7 @@ static void numbers_exceptions_load_scan(State& state) {
    }
    benchmark::DoNotOptimize(container.data());
    benchmark::ClobberMemory();
-  }  
+  }
 }
 BENCHMARK(numbers_exceptions_load_scan);

@@ -456,7 +456,7 @@ static void numbers_exceptions_load_size_scan(State& state) {
    if(pos != container.size()) { cerr << "bad count" << endl; }
    benchmark::DoNotOptimize(container.data());
    benchmark::ClobberMemory();
-  }  
+  }
 }
 BENCHMARK(numbers_exceptions_load_size_scan);

@@ -472,6 +472,7 @@ static void twitter_count(State& state) {
 }
 BENCHMARK(twitter_count);

+#ifndef SIMDJSON_DISABLE_DEPRECATED_API
 SIMDJSON_PUSH_DISABLE_WARNINGS
 SIMDJSON_DISABLE_DEPRECATED_WARNING
 static void iterator_twitter_count(State& state) {
@@ -491,6 +492,7 @@ static void iterator_twitter_count(State& state) {
 }
 BENCHMARK(iterator_twitter_count);
 SIMDJSON_POP_DISABLE_WARNINGS
+#endif // SIMDJSON_DISABLE_DEPRECATED_API

 static void twitter_default_profile(State& state) {
  // Count unique users with a default profile.
@@ -575,8 +577,11 @@ static void error_code_twitter_default_profile(State& state) noexcept {
 }
 BENCHMARK(error_code_twitter_default_profile);

+#ifndef SIMDJSON_DISABLE_DEPRECATED_API
+
 SIMDJSON_PUSH_DISABLE_WARNINGS
 SIMDJSON_DISABLE_DEPRECATED_WARNING
+
 static void iterator_twitter_default_profile(State& state) {
  // Count unique users with a default profile.
  padded_string json;
@@ -615,8 +620,10 @@ static void iterator_twitter_default_profile(State& state) {
    if (default_users.size() != 86) { return; }
  }
 }
+
 SIMDJSON_POP_DISABLE_WARNINGS
 BENCHMARK(iterator_twitter_default_profile);
+#endif // SIMDJSON_DISABLE_DEPRECATED_API

 static void error_code_twitter_image_sizes(State& state) noexcept {
  // Count unique image sizes
@@ -648,6 +655,8 @@ static void error_code_twitter_image_sizes(State& state) noexcept {
 }
 BENCHMARK(error_code_twitter_image_sizes);

+#ifndef SIMDJSON_DISABLE_DEPRECATED_API
+
 SIMDJSON_PUSH_DISABLE_WARNINGS
 SIMDJSON_DISABLE_DEPRECATED_WARNING
 static void iterator_twitter_image_sizes(State& state) {
@@ -702,7 +711,7 @@ static void iterator_twitter_image_sizes(State& state) {
            if (!iter.up()) { return; } // back to entities
          }
          if (!iter.up()) { return; } // back to status
-        } 
+        }
      } while (iter.next()); // next status
    }

@@ -711,6 +720,9 @@ static void iterator_twitter_image_sizes(State& state) {
 }
 BENCHMARK(iterator_twitter_image_sizes);

+#endif // SIMDJSON_DISABLE_DEPRECATED_API
+
+#ifndef SIMDJSON_DISABLE_DEPRECATED_API
 static void print_json(State& state) noexcept {
  // Prints the number of results in twitter.json
  dom::parser parser;
@@ -727,6 +739,7 @@ static void print_json(State& state) noexcept {
  }
 }
 BENCHMARK(print_json);
+#endif // SIMDJSON_DISABLE_DEPRECATED_API
 SIMDJSON_POP_DISABLE_WARNINGS

-BENCHMARK_MAIN();
+BENCHMARK_MAIN();
@@ -111,6 +111,7 @@ BENCHMARK(parse_gsoc)->Repetitions(10)->ComputeStatistics("max", [](const std::v



+#ifndef SIMDJSON_DISABLE_DEPRECATED_API
 SIMDJSON_PUSH_DISABLE_WARNINGS
 SIMDJSON_DISABLE_DEPRECATED_WARNING
 static void json_parse(State& state) {
@@ -123,6 +124,8 @@ static void json_parse(State& state) {
 }
 SIMDJSON_POP_DISABLE_WARNINGS
 BENCHMARK(json_parse);
+#endif // SIMDJSON_DISABLE_DEPRECATED_API
+
 static void parser_parse_error_code(State& state) {
  dom::parser parser;
  if (parser.allocate(EMPTY_ARRAY.length())) { return; }
@@ -151,6 +154,7 @@ BENCHMARK(parser_parse_exception);

 #endif // SIMDJSON_EXCEPTIONS

+#ifndef SIMDJSON_DISABLE_DEPRECATED_API
 SIMDJSON_PUSH_DISABLE_WARNINGS
 SIMDJSON_DISABLE_DEPRECATED_WARNING
 static void build_parsed_json(State& state) {
@@ -162,6 +166,8 @@ static void build_parsed_json(State& state) {
 SIMDJSON_POP_DISABLE_WARNINGS

 BENCHMARK(build_parsed_json);
+#endif
+
 static void document_parse_error_code(State& state) {
  for (simdjson_unused auto _ : state) {
    dom::parser parser;
@@ -63,8 +63,8 @@ void print_usage(ostream& out) {
  out << "-s STAGE   - Stop after the given stage." << endl;
  out << "             -s stage1 - Stop after find_structural_bits." << endl;
  out << "             -s all    - Run all stages." << endl;
-  out << "-a ARCH    - Use the parser with the designated architecture (HASWELL, WESTMERE" << endl;
-  out << "             or ARM64). By default, detects best supported architecture." << endl;
+  out << "-a ARCH    - Use the parser with the designated architecture (HASWELL, WESTMERE," << endl;
+  out << "             PPC64 or ARM64). By default, detects best supported architecture." << endl;
 }

 void exit_usage(string message) {
@@ -99,11 +99,11 @@ struct option_struct {
      case 'a': {
          auto impl = simdjson::available_implementations[optarg];
          if(impl && impl->supported_by_runtime_system()) {
-            simdjson::active_implementation = impl; 
-          } else { 
+            simdjson::active_implementation = impl;
+          } else {
            std::cerr << "implementation " << optarg << " not found or not supported " << std::endl;
          }
-        } 
+        }
        break;
      case 's':
        if (!strcmp(optarg, "stage1")) {
@@ -228,7 +228,7 @@ struct progress_bar {
 /**
 * The speed at which we can allocate memory is strictly system specific.
 * It depends on the OS and the runtime library. It is subject to various
- * system-specific knobs. It is not something that we can reasonably 
+ * system-specific knobs. It is not something that we can reasonably
 * benchmark with crude timings.
 * If someone wants to optimize how simdjson allocate memory, then it will
 * almost surely require a distinct benchmarking tool. What is meant by
@@ -7,7 +7,7 @@

 # Clone the repository if it's not there
 find_package(Git QUIET)
-if (SIMDJSON_IS_UNDER_GIT AND SIMDJSON_GIT AND Git_FOUND AND (GIT_VERSION_STRING VERSION_GREATER  "2.1.4") AND (NOT CMAKE_GENERATOR MATCHES Ninja) ) # We use "-C" which requires a recent git
+if (Git_FOUND AND (GIT_VERSION_STRING VERSION_GREATER  "2.1.4") AND (NOT CMAKE_GENERATOR MATCHES Ninja) ) # We use "-C" which requires a recent git
  message(STATUS "Git is available and it is recent. We are enabling checkperf targets.")
  # sync_git_repository(myrepo ...) creates two targets:
  # myrepo - if the repo does not exist, creates and syncs it against the origin branch
@@ -95,7 +95,7 @@ if (SIMDJSON_IS_UNDER_GIT AND SIMDJSON_GIT AND Git_FOUND AND (GIT_VERSION_STRING
 else()
  if (CMAKE_GENERATOR MATCHES Ninja)
   message(STATUS "We disable the checkperf targets under Ninja.")
-  else() 
+  else()
   message(STATUS "Either git is unavailable or else it is too old. We are disabling checkperf targets.")
  endif()
 endif ()
@@ -20,7 +20,7 @@ void remove_duplicates(std::vector<int64_t> &v) {

 namespace distinct_user_id {
 template<typename T> static void DistinctUserID(benchmark::State &state);
-} // namespace 
+} // namespace

 //
 // Implementation
@@ -12,12 +12,12 @@ using namespace simdjson::builtin;

 class OnDemand {
 public:
-  OnDemand() { 
+  OnDemand() {
    if(!displayed_implementation) {
-      std::cout << "On Demand implementation: " << builtin_implementation()->name() << std::endl; 
+      std::cout << "On Demand implementation: " << builtin_implementation()->name() << std::endl;
      displayed_implementation = true;
    }
-  } 
+  }
  simdjson_really_inline bool Run(const padded_string &json);
  simdjson_really_inline const std::vector<int64_t> &Result() { return ids; }
  simdjson_really_inline size_t ItemCount() { return ids.size(); }
@@ -39,20 +39,20 @@ simdjson_really_inline bool OnDemand::Run(const padded_string &json) {
      //
      // You might think that you do not need the braces, but
      // you do, otherwise you will get the wrong answer. That is
-      // because you can only have one active object or array 
-      // at a time. 
+      // because you can only have one active object or array
+      // at a time.
      {
        ondemand::object user = tweet["user"];
-        int64_t id = user["id"]; 
+        int64_t id = user["id"];
        ids.push_back(id);
      }
-      // Not all tweets have a "retweeted_status", but when they do 
+      // Not all tweets have a "retweeted_status", but when they do
      // we want to go and find the user within.
      auto retweet = tweet["retweeted_status"];
      if(!retweet.error()) {
          ondemand::object retweet_content = retweet;
          ondemand::object reuser = retweet_content["user"];
-          int64_t rid = reuser["id"]; 
+          int64_t rid = reuser["id"];
          ids.push_back(rid);
      }
  }
@@ -29,7 +29,7 @@ int main(int argc, char *argv[]) {
              << std::endl;
  }
  simdjson::padded_string p;
-  bench(filename, p); 
+  bench(filename, p);
  double meanval = 0;
  double maxval = 0;
  double minval = 10000;
@@ -45,7 +45,7 @@ static std::string build_json_array(size_t N) {
  myss << R"(    "info": "some info")" << endl;
  myss << R"(})" << endl;
  string answer = myss.str();
-  cout << "Creating a source file spanning " << (answer.size() + 512) / 1024 << " KB " << endl;  
+  cout << "Creating a source file spanning " << (answer.size() + 512) / 1024 << " KB " << endl;
  return answer;
 }

@@ -29,7 +29,7 @@ static std::string build_json_array(size_t N) {
  myss << std::endl;
  myss << "]" << std::endl;
  std::string answer = myss.str();
-  std::cout << "Creating a source file spanning " << (answer.size() + 512) / 1024 << " KB " << std::endl;  
+  std::cout << "Creating a source file spanning " << (answer.size() + 512) / 1024 << " KB " << std::endl;
  return answer;
 }

@@ -52,7 +52,7 @@ public:
  simdjson_really_inline error_code visit_key(json_iterator &, const uint8_t * key) {
    switch(key[1]) {
      // Technically, we should check the other characters
-      // in the key, but we are cheating to go as fast 
+      // in the key, but we are cheating to go as fast
      // as possible.
      case 'x':
        idx = GOT_X;
@@ -62,11 +62,11 @@ public:
        break;
      case 'z':
        idx = GOT_Z;
-        break; 
+        break;
      default:
-        idx = GOT_SOMETHING_ELSE; 
+        idx = GOT_SOMETHING_ELSE;
    }
-    return SUCCESS; 
+    return SUCCESS;
  }
  simdjson_really_inline error_code visit_array_start(json_iterator &)  { return SUCCESS; }
  simdjson_really_inline error_code visit_array_end(json_iterator &) { return SUCCESS; }
@@ -10,7 +10,7 @@
 #else // no __has_include
 // Please insure that linux headers have been installed.
 #include <asm/unistd.h>       // for __NR_perf_event_open
-#endif 
+#endif
 #include <linux/perf_event.h> // for perf event constants
 #include <sys/ioctl.h>        // for ioctl
 #include <unistd.h>           // for syscall
@@ -122,7 +122,7 @@ int main(int argc, char *argv[]) {
  BEST_TIME_NOCHECK(
      "despacing with std::minify", simdjson_stringme(p),, repeat, volume, !just_data);

-      
+
  memcpy(buffer, p.data(), p.size());
  size_t outlength;
  uint8_t *cbuffer = (uint8_t *)buffer;
@@ -27,26 +27,30 @@ SIMDJSON_PUSH_DISABLE_ALL_WARNINGS
 #include <nlohmann/json.hpp>
 using json = nlohmann::json;

+#ifdef HAS_BOOST_JSON
 #include <boost/json/parser.hpp>
 #include <boost/json/monotonic_resource.hpp>
+#endif

 #ifdef ALLPARSER

-#include "fastjson.cpp"
-#include "fastjson_dom.cpp"
-#include "gason.cpp"
+#include "fastjson/core.h"
+#include "fastjson/dom.h"
+#include "fastjson/fastjson.h"
+
+#include "gason.h"
+
+#include "json11.hpp"

-#include "json11.cpp"
-extern "C" {
-#include "cJSON.c"
 #include "cJSON.h"
-#include "jsmn.c"
+
 #include "jsmn.h"
+
 #include "ujdecode.h"
-#include "ultrajsondec.c"
+extern "C" {
+#include "ultrajson.h"
 }

-#include "jsoncpp.cpp"
 #include "json/json.h"

 #endif
@@ -175,7 +179,7 @@ bool bench(const char *filename, bool verbose, bool just_data,
            std::memcpy(buffer, p.data(), p.size()) &&
                (buffer[p.size()] = '\0'),
            repeat, volume, !just_data);
-
+#ifdef HAS_BOOST_JSON
  {
    const boost::json::string_view sv(p.data(), p.size());
    boost::json::parser p;
@@ -191,15 +195,16 @@ bool bench(const char *filename, bool verbose, bool just_data,

    BEST_TIME("Boost.json", execute(sv), false, , repeat, volume, !just_data);
  }
+#endif
  {
-    
+
    auto execute = [&p]() -> bool {
      yyjson_doc *doc = yyjson_read(p.data(), p.size(), 0);
      bool is_ok = doc != nullptr;
      yyjson_doc_free(doc);
      return is_ok;
    };
-    
+
    BEST_TIME("yyjson", execute(), true, , repeat, volume, !just_data);
  }
 #ifndef ALLPARSER
@@ -37,7 +37,7 @@ simdjson_really_inline bool Iter::Run(const padded_string &json) {

  // Walk the document, parsing the tweets as we go

-  // { "statuses": 
+  // { "statuses":
  auto iter = parser.iterate_raw(json).value();
  if (!iter.start_object()   || !iter.find_field_raw("statuses")) { return false; }
  // { "statuses": [
@@ -12,12 +12,12 @@ using namespace simdjson::builtin;

 class OnDemand {
 public:
-  OnDemand() { 
+  OnDemand() {
    if(!displayed_implementation) {
-      std::cout << "On Demand implementation: " << builtin_implementation()->name() << std::endl; 
+      std::cout << "On Demand implementation: " << builtin_implementation()->name() << std::endl;
      displayed_implementation = true;
    }
-  } 
+  }
  simdjson_really_inline bool Run(const padded_string &json);
  simdjson_really_inline const std::vector<tweet> &Result() { return tweets; }
  simdjson_really_inline size_t ItemCount() { return tweets.size(); }
@@ -1,6 +1,6 @@

 if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
-  message (STATUS "The simdjson repository appears to be standalone.")  
+  message (STATUS "The simdjson repository appears to be standalone.")
  option(SIMDJSON_JUST_LIBRARY "Build just the library, omit tests, tools and benchmarks" OFF)
  message (STATUS "By default, we attempt to build everything.")
 else()
@@ -9,14 +9,6 @@ else()
  message (STATUS "By default, we just build the library.")
 endif()

-if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/.git)
-  set(SIMDJSON_IS_UNDER_GIT ON CACHE BOOL  "Whether cmake is under git control")
-  message( STATUS "The simdjson repository appears to be under git." )
-else()
-  set(SIMDJSON_IS_UNDER_GIT OFF CACHE BOOL  "Whether cmake is under git control")
-  message( STATUS "The simdjson repository does not appear to be under git." )
-endif()
-
 #
 # Flags used by exes and by the simdjson library (project-wide flags)
 #
@@ -35,6 +27,15 @@ if(SIMDJSON_SANITIZE)
  endif()
 endif()

+if(SIMDJSON_SANITIZE_THREADS)
+  target_compile_options(simdjson-flags INTERFACE -fsanitize=thread -fsanitize=undefined -fno-sanitize-recover=all)
+  target_link_libraries(simdjson-flags INTERFACE -fsanitize=thread -fsanitize=undefined -fno-sanitize-recover=all)
+
+  # Ubuntu bug for GCC 5.0+ (safe for all versions)
+  if (CMAKE_COMPILER_IS_GNUCC)
+    target_link_libraries(simdjson-flags INTERFACE -fuse-ld=gold)
+  endif()
+endif()

 if (NOT CMAKE_BUILD_TYPE)
  message(STATUS "No build type selected, default to Release")
@@ -50,12 +51,6 @@ else()
  option(SIMDJSON_BUILD_STATIC "Build a static library" OFF) # turning it on disables the production of a dynamic library
  option(SIMDJSON_USE_LIBCPP "Use the libc++ library" OFF)
 endif()
-option(SIMDJSON_COMPETITION "Compile competitive benchmarks" ON)
-
-option(SIMDJSON_GOOGLE_BENCHMARKS "compile the Google Benchmark benchmarks" ON)
-if(SIMDJSON_COMPETITION)
-  message(STATUS "Using SIMDJSON_GOOGLE_BENCHMARKS")
-endif()

 set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/tools/cmake")

@@ -85,11 +80,13 @@ else()
  target_compile_options(simdjson-internal-flags INTERFACE /WX /W3 /sdl /w34714) # https://docs.microsoft.com/en-us/cpp/error-messages/compiler-warnings/compiler-warning-level-4-c4714?view=vs-2019
 endif()
 if(SIMDJSON_VISUAL_STUDIO_BUILD_WITH_DEBUG_INFO_FOR_PROFILING)
-  target_link_options(simdjson-flags  INTERFACE    /DEBUG ) 
+  target_link_options(simdjson-flags  INTERFACE    /DEBUG )
  target_compile_options(simdjson-flags INTERFACE  /Zi)
 endif()
 else()
-  target_compile_options(simdjson-internal-flags INTERFACE -fPIC)
+  if(NOT WIN32)
+    target_compile_options(simdjson-internal-flags INTERFACE -fPIC)
+  endif()
  target_compile_options(simdjson-internal-flags INTERFACE -Werror -Wall -Wextra -Weffc++)
  target_compile_options(simdjson-internal-flags INTERFACE -Wsign-compare -Wshadow -Wwrite-strings -Wpointer-arith -Winit-self -Wconversion -Wno-sign-conversion)
 endif()
@@ -101,7 +98,7 @@ endif()
 #
 # Implementation selection
 #
-set(SIMDJSON_ALL_IMPLEMENTATIONS "fallback;westmere;haswell;arm64")
+set(SIMDJSON_ALL_IMPLEMENTATIONS "fallback;westmere;haswell;arm64;ppc64")

 set(SIMDJSON_IMPLEMENTATION "" CACHE STRING "Semicolon-separated list of implementations to include (${SIMDJSON_ALL_IMPLEMENTATIONS}). If this is not set, any implementations that are supported at compile time and may be selected at runtime will be included.")
 foreach(implementation ${SIMDJSON_IMPLEMENTATION})
@@ -110,7 +107,7 @@ foreach(implementation ${SIMDJSON_IMPLEMENTATION})
  endif()
 endforeach(implementation)

-set(SIMDJSON_EXCLUDE_IMPLEMENTATION "" CACHE STRING "Semicolon-separated list of implementations to exclude (haswell/westmere/arm64/fallback). By default, excludes any implementations that are unsupported at compile time or cannot be selected at runtime.")
+set(SIMDJSON_EXCLUDE_IMPLEMENTATION "" CACHE STRING "Semicolon-separated list of implementations to exclude (haswell/westmere/arm64/ppc64/fallback). By default, excludes any implementations that are unsupported at compile time or cannot be selected at runtime.")
 foreach(implementation ${SIMDJSON_EXCLUDE_IMPLEMENTATION})
  if(NOT (implementation IN_LIST SIMDJSON_ALL_IMPLEMENTATIONS))
    message(ERROR "Implementation ${implementation} not supported by simdjson. Possible implementations: ${SIMDJSON_ALL_IMPLEMENTATIONS}")
@@ -161,6 +158,11 @@ if(NOT SIMDJSON_IMPLEMENTATION_ARM64)
  message(DEPRECATION "SIMDJSON_IMPLEMENTATION_ARM64 is deprecated. Use SIMDJSON_IMPLEMENTATION=-arm64 instead.")
  target_compile_definitions(simdjson-flags INTERFACE SIMDJSON_IMPLEMENTATION_ARM64=0)
 endif()
+option(SIMDJSON_IMPLEMENTATION_PPC64 "Include the arm64 implementation" ON)
+if(NOT SIMDJSON_IMPLEMENTATION_PPC64)
+  message(DEPRECATION "SIMDJSON_IMPLEMENTATION_PPC64 is deprecated. Use SIMDJSON_IMPLEMENTATION=-ppc64 instead.")
+  target_compile_definitions(simdjson-flags INTERFACE SIMDJSON_IMPLEMENTATION_PPC64=0)
+endif()
 option(SIMDJSON_IMPLEMENTATION_FALLBACK "Include the fallback implementation" ON)
 if(NOT SIMDJSON_IMPLEMENTATION_FALLBACK)
  message(DEPRECATION "SIMDJSON_IMPLEMENTATION_FALLBACK is deprecated. Use SIMDJSON_IMPLEMENTATION=-fallback instead.")
@@ -178,12 +180,20 @@ endif(SIMDJSON_ONDEMAND_SAFETY_RAILS)

 option(SIMDJSON_BASH "Allow usage of bash within CMake" ON)

-option(SIMDJSON_GIT "Allow usage of git within CMake" ON)
-
 option(SIMDJSON_EXCEPTIONS "Enable simdjson's exception-throwing interface" ON)
 if(NOT SIMDJSON_EXCEPTIONS)
  message(STATUS "simdjson exception interface turned off. Code that does not check error codes will not compile.")
  target_compile_definitions(simdjson-flags INTERFACE SIMDJSON_EXCEPTIONS=0)
+  if(MSVC)
+    # CMake currently /EHsc as a default flag in CMAKE_CXX_FLAGS on MSVC. Replacing this with a more general abstraction is a WIP (see https://gitlab.kitware.com/cmake/cmake/-/issues/20610)
+    # /EHs enables standard C++ stack unwinding when catching exceptions (non-structured exception handling)
+    # /EHc used in conjection with /EHs indicates that extern "C" functions never throw (terminate-on-throw)
+    # Here, we disable both with the - argument negation operator
+    string(REPLACE "/EHsc" "/EHs-c-" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
+
+    # Because we cannot change the flag above on an invidual target (yet), the definition below must similarly be added globally
+    add_definitions(-D_HAS_EXCEPTIONS=0)
+  endif()
 endif()

 option(SIMDJSON_ENABLE_THREADS "Link with thread support" ON)
@@ -202,6 +212,11 @@ if (SIMDJSON_VERBOSE_LOGGING)
  target_compile_definitions(simdjson-flags INTERFACE SIMDJSON_VERBOSE_LOGGING=1)
 endif()

+option(SIMDJSON_DISABLE_DEPRECATED_API "Disables deprecated APIs" Off)
+if (SIMDJSON_DISABLE_DEPRECATED_API)
+    target_compile_definitions(simdjson-flags INTERFACE SIMDJSON_DISABLE_DEPRECATED_API=1)
+endif()
+
 if(SIMDJSON_USE_LIBCPP)
  target_link_libraries(simdjson-flags INTERFACE -stdlib=libc++ -lc++abi)
  # instead of the above line, we could have used
@@ -0,0 +1 @@
+.cache/
@@ -1,98 +1,110 @@
-# Initializes a git submodule if it hasn't been initialized before
+include(import.cmake)

+option(SIMDJSON_COMPETITION "Compile competitive benchmarks" ON)
+option(SIMDJSON_GOOGLE_BENCHMARKS "compile the Google Benchmark benchmarks" ON)

-find_package(Git QUIET) # We want the library to build even if git is missing
-if ((Git_FOUND) AND SIMDJSON_GIT AND (SIMDJSON_IS_UNDER_GIT))
-  message(STATUS "Git is available.")
-  # Does NOT attempt to update or otherwise modify git submodules that are already initialized.
-  function(initialize_submodule DIRECTORY)
-    if(NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${DIRECTORY}/.git)
-      message(STATUS "${CMAKE_CURRENT_SOURCE_DIR}/${DIRECTORY}/.git does not exist. Initializing ${DIRECTORY} submodule ...")
-      execute_process(COMMAND ${GIT_EXECUTABLE} submodule update --init ${CMAKE_CURRENT_SOURCE_DIR}/${DIRECTORY}
-                      WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-                      RESULT_VARIABLE GIT_EXIT_CODE)
-      if(NOT GIT_EXIT_CODE EQUAL "0")
-        message(FATAL_ERROR "${GIT_EXECUTABLE} submodule update --init dependencies/${DIRECTORY} failed with exit code ${GIT_EXIT_CODE}, please checkout submodules")
-      endif()
-    endif()
-  endfunction(initialize_submodule)
+if(SIMDJSON_GOOGLE_BENCHMARKS)
+  set_off(BENCHMARK_ENABLE_TESTING)
+  set_off(BENCHMARK_ENABLE_INSTALL)

-  if (SIMDJSON_GOOGLE_BENCHMARKS)
-    message (STATUS "'SIMDJSON_GOOGLE_BENCHMARKS' is requested, configuring..." )
-    option(BENCHMARK_ENABLE_TESTING OFF)
-    set(BENCHMARK_ENABLE_TESTING OFF)
-    option(BENCHMARK_ENABLE_INSTALL OFF)
-    set(BENCHMARK_ENABLE_INSTALL OFF)
-    initialize_submodule(benchmark)
-    add_subdirectory(benchmark)
-  endif()
-
-  if (SIMDJSON_COMPETITION)
-    initialize_submodule(cJSON)
-    add_library(competition-cJSON INTERFACE)
-    target_include_directories(competition-cJSON INTERFACE cJSON)
-
-    initialize_submodule(fastjson)
-    add_library(competition-fastjson INTERFACE)
-    target_include_directories(competition-fastjson INTERFACE fastjson/src fastjson/include)
-
-    initialize_submodule(gason)
-    add_library(competition-gason INTERFACE)
-    target_include_directories(competition-gason INTERFACE gason/src)
-
-    initialize_submodule(jsmn)
-    add_library(competition-jsmn INTERFACE)
-    target_include_directories(competition-jsmn INTERFACE jsmn)
-
-    initialize_submodule(json)
-    add_library(competition-json INTERFACE)
-    target_include_directories(competition-json INTERFACE json/single_include)
-
-    initialize_submodule(json11)
-    add_library(competition-json11 INTERFACE)
-    target_include_directories(competition-json11 INTERFACE json11)
-
-    add_library(competition-jsoncppdist INTERFACE)
-    target_include_directories(competition-jsoncppdist INTERFACE jsoncppdist)
-
-    initialize_submodule(rapidjson)
-    add_library(competition-rapidjson INTERFACE)
-    target_include_directories(competition-rapidjson INTERFACE rapidjson/include)
-
-    initialize_submodule(sajson)
-    add_library(competition-sajson INTERFACE)
-    target_include_directories(competition-sajson INTERFACE sajson/include)
-
-    initialize_submodule(ujson4c)
-    add_library(competition-ujson4c ujson4c/src/ujdecode.c)
-    target_include_directories(competition-ujson4c PUBLIC ujson4c/3rdparty ujson4c/src)
-
-    initialize_submodule(boost.json)
-    add_library(boostjson boost.json/src/src.cpp)
-    target_compile_definitions(boostjson PUBLIC BOOST_JSON_STANDALONE)
-    target_include_directories(boostjson PUBLIC boost.json/include)
-
-    initialize_submodule(yyjson)
-    add_library(yyjson yyjson/src/yyjson.c)
-    target_include_directories(yyjson PUBLIC yyjson/src)
-
-    add_library(competition-core INTERFACE)
-    target_link_libraries(competition-core INTERFACE competition-json competition-rapidjson competition-sajson competition-cJSON competition-jsmn boostjson yyjson)
-
-    add_library(competition-all INTERFACE)
-    target_link_libraries(competition-all INTERFACE competition-core competition-jsoncppdist competition-json11 competition-fastjson competition-gason competition-ujson4c)
-  endif()
-
-  initialize_submodule(cxxopts)
-  message(STATUS "We acquired cxxopts and we are adding it as a library and target.")
-  add_library(cxxopts INTERFACE)
-  target_include_directories(cxxopts INTERFACE cxxopts/include)
-else()
-  message(STATUS "Git is unavailable.")
-  if(SIMDJSON_COMPETITION)
-    message (STATUS "'SIMDJSON_COMPETITION' is requested, but we cannot download the remote repositories." )
-  endif()
-  if(SIMDJSON_GOOGLE_BENCHMARKS)
-    message (STATUS "'SIMDJSON_GOOGLE_BENCHMARKS' is requested, but we cannot download the remote repositories." )
-  endif()
+  import_dependency(google_benchmarks google/benchmark 8982e1e)
+  add_dependency(google_benchmarks)
 endif()
+
+# This prevents variables declared with set() from unnecessarily escaping and
+# should not be called more than once
+function(competition_scope_)
+  # boost json in standalone mode requires C++17 string_view
+  include(CheckCXXSourceCompiles)
+  check_cxx_source_compiles("#include <string_view>\n#if __cpp_lib_string_view < 201606\n#error no string view support\n#endif\nint main(){}" USE_BOOST_JSON)
+  if(USE_BOOST_JSON)
+  import_dependency(boostjson boostorg/json ee8d72d8502b409b5561200299cad30ccdb91415)
+  add_library(boostjson STATIC "${boostjson_SOURCE_DIR}/src/src.cpp")
+  target_compile_definitions(boostjson PUBLIC BOOST_JSON_STANDALONE)
+  target_include_directories(boostjson SYSTEM PUBLIC
+          "${boostjson_SOURCE_DIR}/include")
+  endif()
+
+  import_dependency(cjson DaveGamble/cJSON c69134d)
+  add_library(cjson STATIC "${cjson_SOURCE_DIR}/cJSON.c")
+  target_include_directories(cjson SYSTEM PUBLIC "${cjson_SOURCE_DIR}")
+
+  import_dependency(fastjson mikeando/fastjson 485f994)
+  add_library(fastjson STATIC
+          "${fastjson_SOURCE_DIR}/src/fastjson.cpp"
+          "${fastjson_SOURCE_DIR}/src/fastjson2.cpp"
+          "${fastjson_SOURCE_DIR}/src/fastjson_dom.cpp")
+  target_include_directories(fastjson SYSTEM PUBLIC
+          "${fastjson_SOURCE_DIR}/include")
+
+  import_dependency(gason vivkin/gason 7aee524)
+  add_library(gason STATIC "${gason_SOURCE_DIR}/src/gason.cpp")
+  target_include_directories(gason SYSTEM PUBLIC "${gason_SOURCE_DIR}/src")
+
+  import_dependency(jsmn zserge/jsmn 18e9fe4)
+  add_library(jsmn STATIC "${jsmn_SOURCE_DIR}/jsmn.c")
+  target_include_directories(jsmn SYSTEM PUBLIC "${jsmn_SOURCE_DIR}")
+
+  message(STATUS "Importing json (nlohmann/json@v3.9.1)")
+  set(nlohmann_json_SOURCE_DIR "${dep_root}/json")
+  if(NOT EXISTS "${nlohmann_json_SOURCE_DIR}")
+    file(DOWNLOAD
+            "https://github.com/nlohmann/json/releases/download/v3.9.1/json.hpp"
+            "${nlohmann_json_SOURCE_DIR}/nlohmann/json.hpp")
+  endif()
+  add_library(nlohmann_json INTERFACE)
+  target_include_directories(nlohmann_json SYSTEM INTERFACE "${nlohmann_json_SOURCE_DIR}")
+
+  import_dependency(json11 dropbox/json11 ec4e452)
+  add_library(json11 STATIC "${json11_SOURCE_DIR}/json11.cpp")
+  target_include_directories(json11 SYSTEM PUBLIC "${json11_SOURCE_DIR}")
+
+  set(jsoncpp_SOURCE_DIR "${simdjson_SOURCE_DIR}/dependencies/jsoncppdist")
+  add_library(jsoncpp STATIC "${jsoncpp_SOURCE_DIR}/jsoncpp.cpp")
+  target_include_directories(jsoncpp SYSTEM PUBLIC "${jsoncpp_SOURCE_DIR}")
+
+  import_dependency(rapidjson Tencent/rapidjson b32cd94)
+  add_library(rapidjson INTERFACE)
+  target_compile_definitions(rapidjson INTERFACE RAPIDJSON_HAS_STDSTRING)
+  target_include_directories(rapidjson SYSTEM INTERFACE
+          "${rapidjson_SOURCE_DIR}/include")
+
+  import_dependency(sajson chadaustin/sajson 2dcfd35)
+  add_library(sajson INTERFACE)
+  target_compile_definitions(sajson INTERFACE SAJSON_UNSORTED_OBJECT_KEYS)
+  target_include_directories(sajson SYSTEM INTERFACE
+          "${sajson_SOURCE_DIR}/include")
+
+  import_dependency(ujson4c esnme/ujson4c e14f3fd)
+  add_library(ujson4c STATIC
+          "${ujson4c_SOURCE_DIR}/src/ujdecode.c"
+          "${ujson4c_SOURCE_DIR}/3rdparty/ultrajsondec.c")
+  target_include_directories(ujson4c SYSTEM PUBLIC
+          "${ujson4c_SOURCE_DIR}/src"
+          "${ujson4c_SOURCE_DIR}/3rdparty")
+
+  import_dependency(yyjson ibireme/yyjson aa33ec5)
+  add_library(yyjson STATIC "${yyjson_SOURCE_DIR}/src/yyjson.c")
+  target_include_directories(yyjson SYSTEM PUBLIC "${yyjson_SOURCE_DIR}/src")
+
+  add_library(competition-core INTERFACE)
+  target_link_libraries(competition-core INTERFACE nlohmann_json rapidjson sajson cjson jsmn yyjson)
+  if(USE_BOOST_JSON)
+     target_compile_definitions(boostjson INTERFACE HAS_BOOST_JSON)
+     target_link_libraries(competition-core INTERFACE boostjson)
+  endif()
+
+  add_library(competition-all INTERFACE)
+  target_link_libraries(competition-all INTERFACE competition-core jsoncpp json11 fastjson gason ujson4c)
+endfunction()
+
+if(SIMDJSON_COMPETITION)
+  competition_scope_()
+endif()
+
+set_off(CXXOPTS_BUILD_EXAMPLES)
+set_off(CXXOPTS_BUILD_TESTS)
+set_off(CXXOPTS_ENABLE_INSTALL)
+
+import_dependency(cxxopts jarro2783/cxxopts 794c975)
+add_dependency(cxxopts)
@@ -0,0 +1,48 @@
+set(dep_root "${simdjson_SOURCE_DIR}/dependencies/.cache")
+if(DEFINED ENV{simdjson_DEPENDENCY_CACHE_DIR})
+  set(dep_root "$ENV{simdjson_DEPENDENCY_CACHE_DIR}")
+endif()
+
+function(import_dependency NAME GITHUB_REPO COMMIT)
+  message(STATUS "Importing ${NAME} (${GITHUB_REPO}@${COMMIT})")
+  set(target "${dep_root}/${NAME}")
+
+  # If the folder exists in the cache, then we assume that everything is as
+  # should be and do nothing
+  if(EXISTS "${target}")
+    set("${NAME}_SOURCE_DIR" "${target}" PARENT_SCOPE)
+    return()
+  endif()
+
+  set(zip_url "https://github.com/${GITHUB_REPO}/archive/${COMMIT}.zip")
+  set(archive "${dep_root}/archive.zip")
+  set(dest "${dep_root}/_extract")
+
+  file(DOWNLOAD "${zip_url}" "${archive}")
+  file(MAKE_DIRECTORY "${dest}")
+  execute_process(
+          WORKING_DIRECTORY "${dest}"
+          COMMAND "${CMAKE_COMMAND}" -E tar xf "${archive}")
+  file(REMOVE "${archive}")
+
+  # GitHub archives only ever have one folder component at the root, so this
+  # will always match that single folder
+  file(GLOB dir LIST_DIRECTORIES YES "${dest}/*")
+
+  file(RENAME "${dir}" "${target}")
+
+  set("${NAME}_SOURCE_DIR" "${target}" PARENT_SCOPE)
+endfunction()
+
+# Delegates to the dependency
+macro(add_dependency NAME)
+  if(NOT DEFINED "${NAME}_SOURCE_DIR")
+    message(FATAL_ERROR "Missing ${NAME}_SOURCE_DIR variable")
+  endif()
+
+  add_subdirectory("${${NAME}_SOURCE_DIR}" "${PROJECT_BINARY_DIR}/_deps/${NAME}")
+endmacro()
+
+function(set_off NAME)
+  set("${NAME}" OFF CACHE INTERNAL "")
+endfunction()
@@ -7,28 +7,28 @@
 // //////////////////////////////////////////////////////////////////////

 /*
-The JsonCpp library's source code, including accompanying documentation, 
+The JsonCpp library's source code, including accompanying documentation,
 tests and demonstration applications, are licensed under the following
 conditions...

-Baptiste Lepilleur and The JsonCpp Authors explicitly disclaim copyright in all 
-jurisdictions which recognize such a disclaimer. In such jurisdictions, 
+Baptiste Lepilleur and The JsonCpp Authors explicitly disclaim copyright in all
+jurisdictions which recognize such a disclaimer. In such jurisdictions,
 this software is released into the Public Domain.

 In jurisdictions which do not recognize Public Domain property (e.g. Germany as of
 2010), this software is Copyright (c) 2007-2010 by Baptiste Lepilleur and
 The JsonCpp Authors, and is released under the terms of the MIT License (see below).

-In jurisdictions which recognize Public Domain property, the user of this 
-software may choose to accept it either as 1) Public Domain, 2) under the 
-conditions of the MIT License (see below), or 3) under the terms of dual 
+In jurisdictions which recognize Public Domain property, the user of this
+software may choose to accept it either as 1) Public Domain, 2) under the
+conditions of the MIT License (see below), or 3) under the terms of dual
 Public Domain/MIT License conditions described here, as they choose.

 The MIT License is about as close to Public Domain as a license can get, and is
 described in clear, concise terms at:

   http://en.wikipedia.org/wiki/MIT_License
-   
+
 The full text of the MIT License follows:

 ========================================================================
@@ -6,28 +6,28 @@
 // //////////////////////////////////////////////////////////////////////

 /*
-The JsonCpp library's source code, including accompanying documentation, 
+The JsonCpp library's source code, including accompanying documentation,
 tests and demonstration applications, are licensed under the following
 conditions...

-Baptiste Lepilleur and The JsonCpp Authors explicitly disclaim copyright in all 
-jurisdictions which recognize such a disclaimer. In such jurisdictions, 
+Baptiste Lepilleur and The JsonCpp Authors explicitly disclaim copyright in all
+jurisdictions which recognize such a disclaimer. In such jurisdictions,
 this software is released into the Public Domain.

 In jurisdictions which do not recognize Public Domain property (e.g. Germany as of
 2010), this software is Copyright (c) 2007-2010 by Baptiste Lepilleur and
 The JsonCpp Authors, and is released under the terms of the MIT License (see below).

-In jurisdictions which recognize Public Domain property, the user of this 
-software may choose to accept it either as 1) Public Domain, 2) under the 
-conditions of the MIT License (see below), or 3) under the terms of dual 
+In jurisdictions which recognize Public Domain property, the user of this
+software may choose to accept it either as 1) Public Domain, 2) under the
+conditions of the MIT License (see below), or 3) under the terms of dual
 Public Domain/MIT License conditions described here, as they choose.

 The MIT License is about as close to Public Domain as a license can get, and is
 described in clear, concise terms at:

   http://en.wikipedia.org/wiki/MIT_License
-   
+
 The full text of the MIT License follows:

 ========================================================================
@@ -6,28 +6,28 @@
 // //////////////////////////////////////////////////////////////////////

 /*
-The JsonCpp library's source code, including accompanying documentation, 
+The JsonCpp library's source code, including accompanying documentation,
 tests and demonstration applications, are licensed under the following
 conditions...

-Baptiste Lepilleur and The JsonCpp Authors explicitly disclaim copyright in all 
-jurisdictions which recognize such a disclaimer. In such jurisdictions, 
+Baptiste Lepilleur and The JsonCpp Authors explicitly disclaim copyright in all
+jurisdictions which recognize such a disclaimer. In such jurisdictions,
 this software is released into the Public Domain.

 In jurisdictions which do not recognize Public Domain property (e.g. Germany as of
 2010), this software is Copyright (c) 2007-2010 by Baptiste Lepilleur and
 The JsonCpp Authors, and is released under the terms of the MIT License (see below).

-In jurisdictions which recognize Public Domain property, the user of this 
-software may choose to accept it either as 1) Public Domain, 2) under the 
-conditions of the MIT License (see below), or 3) under the terms of dual 
+In jurisdictions which recognize Public Domain property, the user of this
+software may choose to accept it either as 1) Public Domain, 2) under the
+conditions of the MIT License (see below), or 3) under the terms of dual
 Public Domain/MIT License conditions described here, as they choose.

 The MIT License is about as close to Public Domain as a license can get, and is
 described in clear, concise terms at:

   http://en.wikipedia.org/wiki/MIT_License
-   
+
 The full text of the MIT License follows:

 ========================================================================
@@ -25,7 +25,7 @@ An overview of what you need to know to use simdjson, with examples.
 Requirements
 ------------------

- A recent compiler (LLVM clang6 or better, GNU GCC 7.4 or better) on a 64-bit (ARM or x64 Intel/AMD) POSIX systems such as macOS, freeBSD or Linux. We require that the compiler supports the C++11 standard or better.
+- A recent compiler (LLVM clang6 or better, GNU GCC 7.4 or better) on a 64-bit (PPC, ARM or x64 Intel/AMD) POSIX systems such as macOS, freeBSD or Linux. We require that the compiler supports the C++11 standard or better.
 - Visual Studio 2017 or better under 64-bit Windows. Users should target a 64-bit build (x64) instead of a 32-bit build (x86). We support the LLVM clang compiler under Visual Studio (clangcl) as well as as the regular Visual Studio compiler. We also support MinGW 64-bit under Windows.

 Including simdjson
@@ -75,7 +75,7 @@ set(SIMDJSON_BUILD_STATIC ON CACHE INTERNAL "")
 FetchContent_MakeAvailable(simdjson)
 ```

-You should replace `GIT_TAG  v0.5.0` by the version you need. If you omit `GIT_TAG  v0.5.0`, you will work from the main branch of simdjson: we recommend that if you are working on production code, 
+You should replace `GIT_TAG  v0.5.0` by the version you need. If you omit `GIT_TAG  v0.5.0`, you will work from the main branch of simdjson: we recommend that if you are working on production code,

 Elsewhere in your project, you can  declare dependencies on simdjson with lines such as these:

@@ -112,6 +112,9 @@ dom::element doc = parser.parse("[1,2,3]"_padded); // parse a string, the _padde
 ```

 The parsed document resulting from the `parser.load` and `parser.parse` calls depends on the `parser` instance. Thus the `parser` instance must remain in scope. Furthermore, you must have at most one parsed document in play per `parser` instance.
+You cannot copy a `parser` instance, you may only move it.
+
+If you need to keep a document around long term, you can keep or move the parser instance. Note that moving a parser instance, or keeping one in a movable data structure like vector or map, can cause any outstanding `element`, `object` or `array` instances to be invalidated. If you need to store a parser in a movable data structure, you should use a `std::unique_ptr` to avoid this invalidation(e.g., `std::unique_ptr<dom::parser> parser(new dom::parser{})`).

 During the`load` or `parse` calls, neither the input file nor the input string are ever modified. After calling `load` or `parse`, the source (either a file or a string) can be safely discarded. All of the JSON data is stored in the `parser` instance.  The parsed document is also immutable in simdjson: you do not modify it by accessing it.

@@ -237,7 +240,7 @@ available, we define the macro `SIMDJSON_HAS_STRING_VIEW`.

 When we detect that it is unavailable,
 we use [string-view-lite](https://github.com/martinmoene/string-view-lite) as a
-substitute. In such cases, we use the type alias `using string_view = nonstd::string_view;` to  
+substitute. In such cases, we use the type alias `using string_view = nonstd::string_view;` to
 offer the same API, irrespective of the compiler and standard library. The macro
 `SIMDJSON_HAS_STRING_VIEW` will be *undefined* to indicate that we emulate `string_view`.

@@ -282,7 +285,7 @@ In some cases, you may have valid JSON strings that you do not wish to parse but
  // It does not have to be null-terminated.
  const char * some_string = "[ 1, 2, 3, 4] ";
  size_t length = std::strlen(some_string);
-  // Create a buffer to receive the minified string. Make sure that there is enough room (length bytes).  
+  // Create a buffer to receive the minified string. Make sure that there is enough room (length bytes).
  std::unique_ptr<char[]> buffer{new char[length]};
  size_t new_length{}; // It will receive the minified length.
  auto error = simdjson::minify(some_string, length, buffer.get(), new_length);
@@ -329,10 +332,10 @@ index allows you to select the indexed node. Within objects, the string value of
 select the value. If your keys contain the characters '/' or '~', they must be escaped as '~1' and
 '~0' respectively. An empty JSON Path refers to the whole document.

-We also extend the JSON Pointer support to include *relative* paths.  
+We also extend the JSON Pointer support to include *relative* paths.
 You can apply a JSON path to any node and the path gets interpreted relatively, as if the currrent node were a whole JSON document.

-Consider the following example: 
+Consider the following example:

 ```c++
 auto cars_json = R"( [
@@ -370,14 +373,31 @@ When you use the code this way, it is your responsibility to check for error bef
 result: if there is an error, the result value will not be valid and using it will caused undefined
 behavior.

-We can write a "quick start" example where we attempt to parse a file and access some data, without triggering exceptions:
+We can write a "quick start" example where we attempt to parse the following JSON file and access some data, without triggering exceptions:
+```JavaScript
+{
+  "statuses": [
+    {
+      "id": 505874924095815700
+    },
+    {
+      "id": 505874922023837700
+    }
+  ],
+  "search_metadata": {
+    "count": 100
+  }
+}
+```
+
+Our program loads the file, selects value corresponding to key "search_metadata" which expected to be an object, and then
+it selects the key "count" within that object.

 ```C++
 #include "simdjson.h"

 int main(void) {
  simdjson::dom::parser parser;
-
  simdjson::dom::element tweets;
  auto error = parser.load("twitter.json").get(tweets);
  if (error) { std::cerr << error << std::endl; return EXIT_FAILURE; }
@@ -391,6 +411,31 @@ int main(void) {
 }
 ```

+The following is a similar example where one wants to get the id of the first tweet without
+triggering exceptions. To do this, we use `["statuses"].at(0)["id"]`. We break that expression down:
+
+- Get the list of tweets (the `"statuses"` key of the document) using `["statuses"]`). The result is expected to be an array.
+- Get the first tweet using `.at(0)`. The result is expected to be an object.
+- Get the id of the tweet using ["id"]. We expect the value to be a non-negative integer.
+
+Observe how we use the `at` method when querying an index into an array, and not the bracket operator.
+
+```C++
+#include "simdjson.h"
+
+int main(void) {
+  simdjson::dom::parser parser;
+  simdjson::dom::element tweets;
+  auto error = parser.load("twitter.json").get(tweets);
+  if(error) { std::cerr << error << std::endl; return EXIT_FAILURE; }
+  uint64_t identifier;
+  error = tweets["statuses"].at(0)["id"].get(identifier);
+  if(error) { std::cerr << error << std::endl; return EXIT_FAILURE; }
+  std::cout << identifier << std::endl;
+  return EXIT_SUCCESS;
+}
+```
+
 ### Error Handling Example

 This is how the example in "Using the Parsed JSON" could be written using only error code checking:
@@ -522,6 +567,21 @@ dom::element doc = parser.parse(json); // Throws an exception if there was an er
 When used this way, a `simdjson_error` exception will be thrown if an error occurs, preventing the
 program from continuing if there was an error.

+
+If one is willing to trigger exceptions, it is possible to write simpler code:
+
+```C++
+#include "simdjson.h"
+
+int main(void) {
+  simdjson::dom::parser parser;
+  simdjson::dom::element tweets = parser.load("twitter.json");
+  std::cout << "ID: " << tweets["statuses"].at(0)["id"] << std::endl;
+  return EXIT_SUCCESS;
+}
+```
+
+
 Tree Walking and JSON Element Types
 -----------------------------------

@@ -602,7 +662,7 @@ for (dom::element doc : docs) {
 // Prints 1 2 3
 ```

-In-memory ndjson strings can be parsed as well, with `parser.parse_many(string)`: 
+In-memory ndjson strings can be parsed as well, with `parser.parse_many(string)`:


 ```c++
@@ -619,7 +679,7 @@ for (dom::element doc : docs) {


 Unlike `parser.parse`, both `parser.load_many(filename)` and `parser.parse_many(string)` may parse
-"on demand" (lazily). That is, no parsing may have been done before you enter the loop 
+"on demand" (lazily). That is, no parsing may have been done before you enter the loop
 `for (dom::element doc : docs) {` and you should expect the parser to only ever fully parse one JSON
 document at a time.

@@ -657,8 +717,8 @@ The simdjson library is fully compliant with  the [RFC 8259](https://www.tbray.o
 - The only insignificant whitespace characters allowed are the space, the horizontal tab, the line feed and the carriage return. In particular, a JSON document may not contain an unespaced null character.
 - A single string or a single number is considered to be a valid JSON document.
 - We fully validate the numbers according to the JSON specification. For example,  the string `01` is not valid JSON document since the specification states that *leading zeros are not allowed*.
- The specification allows implementations to set limits on the range and precision of numbers accepted.  We support 64-bit floating-point numbers as well as integer values. 
-  - We parse integers and floating-point numbers as separate types which allows us to support all signed (two complement's) 64-bit integers, like a Java `long` or a C/C++ `long long` and all 64-bit unsigned integers. When we cannot represent exactly an integer as a signed or unsigned 64-bit value, we reject the JSON document. 
+- The specification allows implementations to set limits on the range and precision of numbers accepted.  We support 64-bit floating-point numbers as well as integer values.
+  - We parse integers and floating-point numbers as separate types which allows us to support all signed (two complement's) 64-bit integers, like a Java `long` or a C/C++ `long long` and all 64-bit unsigned integers. When we cannot represent exactly an integer as a signed or unsigned 64-bit value, we reject the JSON document.
  - We support the full range of 64-bit floating-point numbers (binary64). The values range from `std::numeric_limits<double>::lowest()`  to `std::numeric_limits<double>::max()`, so from -1.7976e308 all the way to 1.7975e308. Extreme values (less or equal to -1e308, greater or equal to 1e308) are rejected: we refuse to parse the input document. Numbers are parsed with with a perfect accuracy (ULP 0): the nearest floating-point value is chosen, rounding to even when needed. If you serialized your floating-point numbers with 17 significant digits in a standard compliant manner, the simdjson library is guaranteed to recovere the example same numbers, exactly.
 - The specification states that JSON text exchanged between systems that are not part of a closed ecosystem MUST be encoded using UTF-8. The simdjson library does full UTF-8 validation as part of the parsing. The specification states that implementations MUST NOT add a byte order mark: the simdjson library rejects documents starting with a  byte order mark.
 - The simdjson library validates string content for unescaped characters. Unescaped line breaks and tabs in strings are not allowed.
@@ -7,7 +7,7 @@ An overview of what you need to know to use simdjson, with examples.
 Requirements
 ------------------

- A recent compiler (LLVM clang6 or better, GNU GCC 7 or better) on a 64-bit (ARM or x64 Intel/AMD) POSIX systems such as macOS, freeBSD or Linux. We require that the compiler supports the C++11 standard or better.
+- A recent compiler (LLVM clang6 or better, GNU GCC 7 or better) on a 64-bit (PPC, ARM or x64 Intel/AMD) POSIX systems such as macOS, freeBSD or Linux. We require that the compiler supports the C++11 standard or better.
 - Visual Studio 2017 or better under 64-bit Windows. Users should target a 64-bit build (x64) instead of a 32-bit build (x86). We support the LLVM clang compiler under Visual Studio (clangcl) as well as as the regular Visual Studio compiler.

 Including simdjson
@@ -47,7 +47,7 @@ include(FetchContent)
 FetchContent_Declare(
  simdjson
  GIT_REPOSITORY https://github.com/simdjson/simdjson.git
-  GIT_TAG  v0.4.7
+  GIT_TAG  v0.6.1
  GIT_SHALLOW TRUE)

 set(SIMDJSON_JUST_LIBRARY ON CACHE INTERNAL "")
@@ -56,7 +56,7 @@ set(SIMDJSON_BUILD_STATIC ON CACHE INTERNAL "")
 FetchContent_MakeAvailable(simdjson)
 ```

-You should replace `GIT_TAG  v0.5.0` by the version you need. If you omit `GIT_TAG  v0.5.0`, you will work from the main branch of simdjson: we recommend that if you are working on production code, 
+You should replace `GIT_TAG  v0.6.1` by the version you need. If you omit `GIT_TAG  v0.6.1`, you will work from the main branch of simdjson: we recommend that if you are working on production code,

 Elsewhere in your project, you can  declare dependencies on simdjson with lines such as these:

@@ -93,10 +93,13 @@ dom::element doc = parser.parse("[1,2,3]"_padded); // parse a string, the _padde
 ```

 The parsed document resulting from the `parser.load` and `parser.parse` calls depends on the `parser` instance. Thus the `parser` instance must remain in scope. Furthermore, you must have at most one parsed document in play per `parser` instance.
+You cannot copy a `parser` instance, you may only move it.
+
+If you need to keep a document around long term, you can keep or move the parser instance. Note that moving a parser instance, or keeping one in a movable data structure like vector or map, can cause any outstanding `element`, `object` or `array` instances to be invalidated. If you need to store a parser in a movable data structure, you should use a `std::unique_ptr` to avoid this invalidation(e.g., `std::unique_ptr<dom::parser> parser(new dom::parser{})`).

 During the`load` or `parse` calls, neither the input file nor the input string are ever modified. After calling `load` or `parse`, the source (either a file or a string) can be safely discarded. All of the JSON data is stored in the `parser` instance.  The parsed document is also immutable in simdjson: you do not modify it by accessing it.

-For best performance, a `parser` instance should be reused over several files: otherwise you will needlessly reallocate memory, an expensive process. It is also possible to avoid entirely memory allocations during parsing when using simdjson. 
+For best performance, a `parser` instance should be reused over several files: otherwise you will needlessly reallocate memory, an expensive process. It is also possible to avoid entirely memory allocations during parsing when using simdjson.


 If you need a lower-level interface, you may call the function `parser.parse(const char * p, size_t l)` on a pointer `p` while specifying the
@@ -218,7 +221,7 @@ available, we define the macro `SIMDJSON_HAS_STRING_VIEW`.

 When we detect that it is unavailable,
 we use [string-view-lite](https://github.com/martinmoene/string-view-lite) as a
-substitute. In such cases, we use the type alias `using string_view = nonstd::string_view;` to  
+substitute. In such cases, we use the type alias `using string_view = nonstd::string_view;` to
 offer the same API, irrespective of the compiler and standard library. The macro
 `SIMDJSON_HAS_STRING_VIEW` will be *undefined* to indicate that we emulate `string_view`.

@@ -311,10 +314,10 @@ index allows you to select the indexed node. Within objects, the string value of
 select the value. If your keys contain the characters '/' or '~', they must be escaped as '~1' and
 '~0' respectively. An empty JSON Path refers to the whole document.

-We also extend the JSON Pointer support to include *relative* paths.  
+We also extend the JSON Pointer support to include *relative* paths.
 You can apply a JSON path to any node and the path gets interpreted relatively, as if the currrent node were a whole JSON document.

-Consider the following example: 
+Consider the following example:

 ```c++
 auto cars_json = R"( [
@@ -351,14 +354,33 @@ When you use the code this way, it is your responsibility to check for error bef
 result: if there is an error, the result value will not be valid and using it will caused undefined
 behavior.

-We can write a "quick start" example where we attempt to parse a file and access some data, without triggering exceptions:

+
+We can write a "quick start" example where we attempt to parse the following JSON file and access some data, without triggering exceptions:
+```JavaScript
+{
+  "statuses": [
+    {
+      "id": 505874924095815700
+    },
+    {
+      "id": 505874922023837700
+    }
+  ],
+  "search_metadata": {
+    "count": 100
+  }
+}
 ```
+
+Our program loads the file, selects value corresponding to key "search_metadata" which expected to be an object, and then
+it selects the key "count" within that object.
+
+```C++
 #include "simdjson.h"

 int main(void) {
  simdjson::dom::parser parser;
-
  simdjson::dom::element tweets;
  auto error = parser.load("twitter.json").get(tweets);
  if (error) { std::cerr << error << std::endl; return EXIT_FAILURE; }
@@ -372,6 +394,32 @@ int main(void) {
 }
 ```

+
+The following is a similar example where one wants to get the id of the first tweet without
+triggering exceptions. To do this, we use `["statuses"].at(0)["id"]`. We break that expression down:
+
+- Get the list of tweets (the `"statuses"` key of the document) using `["statuses"]`). The result is expected to be an array.
+- Get the first tweet using `.at(0)`. The result is expected to be an object.
+- Get the id of the tweet using ["id"]. We expect the value to be a non-negative integer.
+
+Observe how we use the `at` method when querying an index into an array, and not the bracket operator.
+
+```
+#include "simdjson.h"
+
+int main(void) {
+  simdjson::dom::parser parser;
+  simdjson::dom::element tweets;
+  auto error = parser.load("twitter.json").get(tweets);
+  if(error) { std::cerr << error << std::endl; return EXIT_FAILURE; }
+  uint64_t identifier;
+  error = tweets["statuses"].at(0)["id"].get(identifier);
+  if(error) { std::cerr << error << std::endl; return EXIT_FAILURE; }
+  std::cout << identifier << std::endl;
+  return EXIT_SUCCESS;
+}
+```
+
 ### Error Handling Example

 This is how the example in "Using the Parsed JSON" could be written using only error code checking:
@@ -503,6 +551,21 @@ dom::element doc = parser.parse(json); // Throws an exception if there was an er
 When used this way, a `simdjson_error` exception will be thrown if an error occurs, preventing the
 program from continuing if there was an error.

+
+
+If one is willing to trigger exceptions, it is possible to write simpler code:
+
+```
+#include "simdjson.h"
+
+int main(void) {
+  simdjson::dom::parser parser;
+  simdjson::dom::element tweets = parser.load("twitter.json");
+  std::cout << "ID: " << tweets["statuses"].at(0)["id"] << std::endl;
+  return EXIT_SUCCESS;
+}
+```
+
 Tree Walking and JSON Element Types
 -----------------------------------

@@ -584,7 +647,7 @@ for (dom::element doc : docs) {
 ```


-In-memory ndjson strings can be parsed as well, with `parser.parse_many(string)`: 
+In-memory ndjson strings can be parsed as well, with `parser.parse_many(string)`:


 ```
@@ -601,7 +664,7 @@ for (dom::element doc : docs) {


 Unlike `parser.parse`, both `parser.load_many(filename)` and `parser.parse_many(string)` may parse
-"on demand" (lazily). That is, no parsing may have been done before you enter the loop 
+"on demand" (lazily). That is, no parsing may have been done before you enter the loop
 `for (dom::element doc : docs) {` and you should expect the parser to only ever fully parse one JSON
 document at a time.

@@ -20,20 +20,22 @@ The current implementations are:
 * haswell: AVX2 (2013 Intel Haswell or later)
 * westmere: SSE4.2 (2010 Westmere or later).
 * arm64: 64-bit ARMv8-A NEON
+* ppc64: 64-bit POWER8 and POWER9 with VSX and ALTIVEC extensions. Both big endian and little endian are implemented, depends on the compiler you are using.
 * fallback: A generic implementation that runs on any 64-bit processor.

 In many cases, you don't know where your compiled binary is going to run, so simdjson automatically
 compiles *all* the implementations into the executable. On Intel, it will include 3 implementations
-(haswell, westmere and fallback), and on ARM it will include 2 (arm64 and fallback).
+(haswell, westmere and fallback), on ARM it will include 2 (arm64 and fallback), and on PPC it will include 2 (ppc64 and fallback).

 If you know more about where you're going to run and want to save the space, you can disable any of
 these implementations at compile time with `-DSIMDJSON_IMPLEMENTATION_X=0` (where X is HASWELL,
-WESTMERE, ARM64 and FALLBACK).
+WESTMERE, ARM64, PPC64 and FALLBACK).

 The simdjson library automatically sets header flags for each implementation as it compiles; there
 is no need to set architecture-specific flags yourself (e.g., `-mavx2`, `/AVX2`  or
 `-march=haswell`), and it may even break runtime dispatch and your binaries will fail to run on
-older processors.
+older processors. _Note:_ for POWER9 processors make sure you compile it with `-mcpu=power9` and `-mtune=power9` to
+get maximum performance.

 Runtime CPU Detection
 ---------------------
@@ -71,7 +73,7 @@ And look them up by name:
 ```c++
 cout << simdjson::available_implementations["fallback"]->description() << endl;
 ```
-Though the fallback implementation should always be available, others might be missing. When 
+Though the fallback implementation should always be available, others might be missing. When
 an implementation is not available, the bracket call `simdjson::available_implementations[name]`
 will return the null pointer.

@@ -5,12 +5,12 @@ A Better Way to Parse Documents?
 Whether we parse JSON or XML, or any other serialized format, there are relatively few common strategies:

 - The most established approach is the construction of document-object-model (DOM).
- Another established approach is a event-based approach (like SAX, SAJ). 
- Another popular approach is the schema-based deserialization model. 
+- Another established approach is a event-based approach (like SAX, SAJ).
+- Another popular approach is the schema-based deserialization model.

-We propose an approach that is as easy to use and often as flexible as the DOM approach, yet as fast and 
-efficient as the schema-based or event-based approaches. We call this new approach "On Demand". The 
-simdjson On Demand API offers a familiar, friendly DOM API and 
+We propose an approach that is as easy to use and often as flexible as the DOM approach, yet as fast and
+efficient as the schema-based or event-based approaches. We call this new approach "On Demand". The
+simdjson On Demand API offers a familiar, friendly DOM API and
 provides the performance of just-in-time parsing on top of the simdjson superior performance.

 To achieve ease of use, we mimicked the *form* of a traditional DOM API: you can iterate over
@@ -81,10 +81,10 @@ type, we avoid branch mispredictions related to data type determination and impr


 We expect users of an On Demand API to work in terms of a JSON dialect, which is a set of expectations and
-specifications that come in addition to the [JSON specification](https://www.rfc-editor.org/rfc/rfc8259.txt). 
+specifications that come in addition to the [JSON specification](https://www.rfc-editor.org/rfc/rfc8259.txt).
 The On Demand approach is designed around several principles:

-* **Streaming (\*):** It avoids preparsing values, keeping the memory usage and the latency down. 
+* **Streaming (\*):** It avoids preparsing values, keeping the memory usage and the latency down.
 * **Forward-Only:** To prevent reiteration of the same values and to keep the number of variables down (literally), only a single index is maintained and everything uses it (even if you have nested for loops). This means when you are going through an array of arrays, for example, that the inner array loop will advance the index to the next comma, and the array can just pick it up and look at it.
 * **Natural Iteration:** A JSON array or object can be iterated with a normal C++ for loop. Nested arrays and objects are supported by nested for loops.
 * **Use-Specific Parsing:** Parsing is always specific to the type required by the programmer. For example, if the programmer asks for an unsigned integer, we just start parsing digits. If there were no digits, we toss an error. There are even different parsers for `double`, `uint64_t` and `int64_t` values. This use-specific parsing avoids the branchiness of a generic "type switch," and makes the code more inlineable and compact.
@@ -98,14 +98,14 @@ approaches to parsing and parser APIs in use today.
 ### DOM Parsers

 Many of the most usable, popular JSON APIs (including simdjson) deserialize into a **DOM**: an intermediate tree of
-objects, arrays and values. In this model, we convert the input data all at once into a tree-like structure (the DOM). 
-The DOM is then accessed by the programmer like any other in-memory data structure. The resulting API let 
-you refer to each array or object separately, using familiar techniques like iteration (`for (auto value : array)`) 
+objects, arrays and values. In this model, we convert the input data all at once into a tree-like structure (the DOM).
+The DOM is then accessed by the programmer like any other in-memory data structure. The resulting API let
+you refer to each array or object separately, using familiar techniques like iteration (`for (auto value : array)`)
 or indexing (`object["key"]`). In some cases, the values are even deserialized directly into familiar C++ constructs like vectors and
 maps.

 The DOM approach is conceptually simple and "programmer friendly". Using the
-DOM tree is often easy enough that many users use the DOM as-is instead of creating their own
+DOM tree is often easy enough that many users use the DOM as-is instead of creating
 their own custom data structures.

 The DOM approach was the only way to parse JSON documents up to version 0.6 of the simdjson library.
@@ -144,14 +144,14 @@ of several gigabytes per second. However, in some instances, it may be possible
 The event-based model (originally from the "Streaming API for XML") uses streaming to eliminate the cost of
 parsing and storing the entire JSON. In the event-based model, a core JSON engine parses the JSON document
 piece by piece, but instead of stuffing values in a DOM tree, it passes each value to a callback function,
-letting the user decide for themselves how to handle it. In such a model, the programmer may need to provide functions 
-for all possible events (a number, a string, a new object, a new array, the array ends, the object ends, and so on). 
-This allows programmers to work with much larger files without running out of memory. 
+letting the user decide for themselves how to handle it. In such a model, the programmer may need to provide functions
+for all possible events (a number, a string, a new object, a new array, the array ends, the object ends, and so on).
+This allows programmers to work with much larger files without running out of memory.

 The drawback is complexity: event-based APIs generally have you define a single callback for each type
 (e.g. `string_field(std::string_view key, std::string_view value)`). Because of this, the programmer suffers
 from context blindness: when they find a string they have to check where it is before they know what to
-do with it. Is this string the text of the tweet, the screen name, or something else? Are we even in 
+do with it. Is this string the text of the tweet, the screen name, or something else? Are we even in
 a tweet right now, or is this from some other place in the document
 entirely? Though an event-based approach may allow superior performance, it is demanding of the programmer
 who must efficiently keep track of its current state within the JSON input.
@@ -196,7 +196,7 @@ sax::parser parser;
 parser.parse(twitter_callbacks());
 ```

-This is a large amount of code, requiring mental gymnastics even to read. An actual implementation is  harder to write 
+This is a large amount of code, requiring mental gymnastics even to read. An actual implementation is  harder to write
 and to maintain.


@@ -221,7 +221,7 @@ choice, as well as a parser to deserialize the JSON into those structs. Some suc
 define your own data structures (`struct`) and they let a preprocessor inspects it and generates a custom JSON parser for it.
 Though not all of these schema-based parser generators generate a parser or even optimize for
 streaming, but they are *able* to in principle. Unlike the DOM and the event-based models, a schema-based approach assumes
- that the structure of the document is known at compile-time. 
+ that the structure of the document is known at compile-time.


 Pros of the schema-based approach:
@@ -244,9 +244,9 @@ the parser does not. This means it has to look at each value blind with a big "s
 statement, asking "is this a number? A string? A boolean? An array? An object?"

 In modern processors, this kind of switch statement can make your program run slower
-than it needs to because of the high cost of branch misprediction. Indeed, modern processor 
-cores rely on speculative execution for speed. They "read ahead" in your program, predicting 
-which instructions to run as soon as the data is available. A single-threaded program can 
+than it needs to because of the high cost of branch misprediction. Indeed, modern processor
+cores rely on speculative execution for speed. They "read ahead" in your program, predicting
+which instructions to run as soon as the data is available. A single-threaded program can
 execute 2, 3 or even more instructions per cycle--largely because of speculative execution.

 Unfortunately, when the processor mispredicts the instructions, typically due to a mispredicted
@@ -258,7 +258,7 @@ Type blindness means that the processor has to guess, for every JSON value, whet
 an object, number, string or boolean since these correspond to distinct code paths.
 Though some JSON files have predictable content, we find in practice that many JSON files
 stress the branch prediction. Though branch predictors improve with each new generation of processors,
-the cost of branch mispredictions also tends to increase as pipelines expand, and the processors become 
+the cost of branch mispredictions also tends to increase as pipelines expand, and the processors become
 able to schedule longer streams of instructions.

 On Demand parsing is tailor-made to solve this problem at the source, parsing values only after the
@@ -297,12 +297,12 @@ To help visualize the algorithm, we'll walk through the example C++ given at the
   ```c++
   auto doc = parser.iterate(json);
   ```
-   
+
   Since this is the first time this parser has been used, `iterate()` first allocates internal
   parser buffers if this is the first time through. When reusing an existing parser, allocation
-   only happens if the new document is bigger than internal buffers can handle. The On Demand 
+   only happens if the new document is bigger than internal buffers can handle. The On Demand
   API only ever allocates memory in the `iterate()` function call.
-   
+
   The simdjson library then preprocesses the JSON text at high speed, finding all tokens (i.e. the starting
   position of any JSON value, as well as any important operators like `,`, `:`, `]` or `}`).

@@ -320,7 +320,7 @@ To help visualize the algorithm, we'll walk through the example C++ given at the

   NOTE: You should always have such a `document` instance (here `doc`) and it should remain in scope for the duration
   of your parsing function. E.g., you should not use the returned document as a temporary (e.g., `auto x = parser.iterate(json).get_object();`)
-   followed by other operations as the destruction of the `document` instance makes all of the derived instances 
+   followed by other operations as the destruction of the `document` instance makes all of the derived instances
   ill-defined.


@@ -357,16 +357,16 @@ To help visualize the algorithm, we'll walk through the example C++ given at the
  }
  ```
   What is not explained in this code expansion is *error chaining*.
-   Generally, you can use `document` methods on a `simdjson_result<...>` value; any errors will 
+   Generally, you can use `document` methods on a `simdjson_result<...>` value; any errors will
   just be passed down the chain. Many method calls
   can be chained in this manner. So `for (object tweet : doc["statuses"])`, which is the equivalent of
-   `object tweet = *(doc.get_object()["statuses"].get_array().begin()).get_object()`, could fail in any of 
+   `object tweet = *(doc.get_object()["statuses"].get_array().begin()).get_object()`, could fail in any of
   6 method calls, and the error will only be checked at the end,
   when you attempt to cast the final `simdjson_result<object>` to object. Upon casting, an exception is
   thrown if there was an error.

   NOTE: while the document can be queried once for a key as if it were an object, it is not an actual object
-   instance. If you need to treat it as an object (e.g., to query more than one keys), you can cast it as 
+   instance. If you need to treat it as an object (e.g., to query more than one keys), you can cast it as
   such `ondemand::object root_object = doc.get_object();`.


@@ -403,7 +403,7 @@ To help visualize the algorithm, we'll walk through the example C++ given at the

   `["screen_name"]` then converts to object, checking for `{`, and finds `"screen_name"`.

-   To convert the result to usable string (i.e., the screen name `lemire`), the characters are written to the document's 
+   To convert the result to usable string (i.e., the screen name `lemire`), the characters are written to the document's
   string buffer (after possibly escaping them), which now has *two* string_views pointing into it, and looks like `first!\0lemire\0`.

   Finally, the temporary user object is destroyed, causing it to skip the remainder of the object
@@ -454,7 +454,7 @@ To help visualize the algorithm, we'll walk through the example C++ given at the

   At the end of the loop, the `tweet` is first destroyed, skipping the remainder of the tweet
   object (`}`).
-   
+
   The `iter++` instruction from `for (ondemand::object tweet : doc["statuses"])` then checks whether there are
   more values and finds that there are none (`]`). It marks the array iteration as finished and the for
   loop terminates.
@@ -467,7 +467,7 @@ Design Features
 ### String Parsing

 When the user requests strings, we unescape them to a single string buffer much like the DOM parser
-so that users enjoy the same string performance as the core simdjson. We do not write the length to the 
+so that users enjoy the same string performance as the core simdjson. We do not write the length to the
 string buffer, however; that is stored in the `string_view` instance we return to the user.

 ```C++
@@ -489,11 +489,11 @@ case with `std::string`) but be mindful that the life cycle of these `string_vie
 parser instance. If the parser instance is destroyed or reused for a new JSON document, these strings are no longer valid.

 We iterate through object instances using `field` instances which represent key-value pairs. The value
-is accessible by the `value()` method whereas the key is accessible by the `key()` method. 
-The keys are treated differently than values are made available as as special type `raw_json_string` 
-which is a lightweight type that is meant to be used on a temporary basis, amost solely for 
+is accessible by the `value()` method whereas the key is accessible by the `key()` method.
+The keys are treated differently than values are made available as as special type `raw_json_string`
+which is a lightweight type that is meant to be used on a temporary basis, amost solely for
 direct raw ASCII comparisons (`field.key() == "mykey"`). If you occasionally need to access and store the
-unescaped key values, you may use the `unescaped_key()` method. Once you have called `unescaped_key()` method, 
+unescaped key values, you may use the `unescaped_key()` method. Once you have called `unescaped_key()` method,
 neither the `key()` nor the `unescaped_key()` methods should be called: the current field instance
 has no longer a key (that is by design). Like other strings, the resulting `std::string_view` generated
 from the `unescaped_key()` method has a lifecycle tied to the `parser` instance: once the parser
@@ -512,7 +512,7 @@ for(auto field : doc.get_object())  {
 The On Demand API is powerful. To compensate, we add some safeguards to ensure that it can be used without fear
 in production systems:

-  - If the value fails to be parsed as one type, the program can try to parse it as something else until the program succeeds. Thus 
+  - If the value fails to be parsed as one type, the program can try to parse it as something else until the program succeeds. Thus
    the programmer can engineer fall back routines.
  - If the value succeeds in being parsed or converted to a type, the program cannot try again. An attempt to parse the same node twice will
    cause the program to abort. We put this safety measure in the API to prevent double iteration of an array which
@@ -533,8 +533,8 @@ in production systems:
    // parent owns the focus
    ondemand::object c1 = parent["child1"];
    // c1 owns the focus
-    // 
-    if(std::string_view(c1["name"]) != "John") { ... } 
+    //
+    if(std::string_view(c1["name"]) != "John") { ... }
    // c2 attempts to grab the focus from parent but fails
    ondemand::object c2 = parent["child2"];
    // c2 is now in an unsafe state and the following line would be unsafe
@@ -552,7 +552,7 @@ in production systems:
    {
      ondemand::object c1 = parent["child1"];
      // c1 grabbed the focus from parent
-      if(std::string_view(c1["name"]) != "John") { return false; } 
+      if(std::string_view(c1["name"]) != "John") { return false; }
    }
    // c1 went out of scope, so its destructor was called and the focus
    // was handed back to parent.
@@ -604,7 +604,7 @@ At this time we recommend the On Demand API in the following cases:
 3. Speed and efficiency are of the utmost importance. Keep in mind that the core simdjson API is highly efficient so adopting the On Demand API is not necessary for high efficiency.
 4. As a developer, you value a clean, flexible and maintainable API.

-Good applications for the On Demand API might be: 
+Good applications for the On Demand API might be:

 * You are working from pre-existing large JSON files that have been vetted. You expect them to be well formed according to a known JSON dialect and to have a consistent layout. For example, you might be doing biomedical research or machine learning on top of static data dumps in JSON.
 * You have a closed system on predetermined hardware. Both the generation and the consumption of JSON data is within your system. Your team controls both the software that produces the JSON and the software the parses it, your team knows and control the hardware. Thus you can fully test your system.
@@ -613,7 +613,7 @@ Good applications for the On Demand API might be:
 ## Checking Your CPU Selection

 Given that the On Demand API does not offer runtime dispatching, your code is compiled against a specific CPU target. You should
-verify that the code is compiled against the target you expect: `haswell` (AVX2 x64 processors), `westmere` (SSE4 x64 processors), `arm64` (64-bit ARM), `fallback` (others). Under x64 processors, many programmers will want to target `haswell` whereas under ARM,
+verify that the code is compiled against the target you expect: `haswell` (AVX2 x64 processors), `westmere` (SSE4 x64 processors), `arm64` (64-bit ARM), `ppc64` (64-bit POWER), `fallback` (others). Under x64 processors, many programmers will want to target `haswell` whereas under ARM,
 most programmers will want to target `arm64`. The `fallback` is probably only good for testing purposes, not for deployment.

 ```C++
@@ -626,4 +626,4 @@ the `CMAKE_CXX_FLAGS` variable:
 ```
 cmake  -DCMAKE_CXX_FLAGS="-march=haswell" -B build_haswell
 cmake --build build_haswell
-```
+```
@@ -13,6 +13,8 @@ Contents
 - [Support](#support)
 - [API](#api)
 - [Use cases](#use-cases)
+- [Tracking your position](#tracking-your-position)
+- [Incomplete streams](#incomplete-streams)

 Motivation
 -----------
@@ -158,3 +160,58 @@ From [jsonlines.org](http://jsonlines.org/examples/):
    ```
    JSON Lines' biggest strength is in handling lots of similar nested data structures. One .jsonl file is easier to
    work with than a directory full of XML files.
+
+
+Tracking your position
+-----------
+
+Some users would like to know where the document they parsed is in the input array of bytes.
+It is possible to do so by accessing directly the iterator and calling its `current_index()`
+method which reports the location (in bytes) of the current document in the input stream.
+
+Let us illustrate the idea with code:
+
+
+```C++
+    auto json = R"([1,2,3]  {"1":1,"2":3,"4":4} [1,2,3]  )"_padded;
+    simdjson::dom::parser parser;
+    simdjson::dom::document_stream stream;
+    ASSERT_SUCCESS( parser.parse_many(json).get(stream) );
+    auto i = stream.begin();
+    for(; i != stream.end(); ++i) {
+        auto doc = *i;
+        if(!doc.error()) {
+          std::cout << "got full document at " << i.current_index() << std::endl;
+        }
+    }
+    size_t index = i.current_index();
+    if(index != 38) {
+      std::cerr << "Expected to stop after the three full documents " << std::endl;
+      std::cerr << "index = " << index << std::endl;
+      return false;
+    }
+```
+
+This code will print:
+```
+got full document at 0
+got full document at 9
+got full document at 29
+```
+
+The last call to `i.current_index()` return the byte index 38, which is just beyond
+the last document.
+
+Incomplete streams
+-----------
+
+Some users may need to work with truncated streams while tracking their location in the stream.
+The same code, with the `current_index()` will work. However, the last block (by default 1MB)
+terminates with an unclosed string, then no JSON document within this last block will validate.
+In particular, it means that if your input string is `[1,2,3]  {"1":1,"2":3,"4":4} [1,2` then
+no JSON document will be successfully parsed. The error `simdjson::UNCLOSED_STRING` will be
+given (even with the first JSON document). It is then your responsability to terminate the input
+maybe by appending the missing data at the end of the truncated string, or by copying the truncated
+data before the continuing input.
+
+
@@ -157,7 +157,7 @@ Downclocking

 SIMD instructions are the public transportation of computing. Instead of using 4 distinct instructions to add numbers, you can replace them with a single instruction that does the same work. Though the one instruction is slightly more expensive, the energy used per unit of work is much less with SIMD. If you can increase your speed using SIMD instructions (NEON, SSE, AVX), you should expect to reduce your power usage.

-The SIMD instructions that simdjson relies upon (SSE and AVX under x64, NEON under ARM) are routinely part of runtime libraries (e.g., [Go](https://golang.org/src/runtime/memmove_amd64.s), [Glibc](https://github.com/ihtsae/glibc/commit/5f3d0b78e011d2a72f9e88b0e9ef5bc081d18f97), [LLVM](https://github.com/llvm/llvm-project/blob/96f3ea0d21b48ca088355db10d4d1a2e9bc9f884/lldb/tools/debugserver/source/MacOSX/i386/DNBArchImplI386.cpp), [Rust](https://github.com/rust-lang/rust/commit/070fad1701fb36b112853b0a6a9787a7bb7ff34c), [Java](http://hg.openjdk.java.net/jdk8u/jdk8u/hotspot/file/c1374141598c/src/cpu/x86/vm/stubGenerator_x86_64.cpp#l1297), [PHP](https://github.com/php/php-src/blob/e5cb53ec68603d4dbdd780fd3ecfca943b4fd383/ext/standard/string.c)). What distinguishes the simdjson library is that it is built from the ground up to benefit from these instructions. 
+The SIMD instructions that simdjson relies upon (SSE and AVX under x64, NEON under ARM, ALTIVEC under PPC) are routinely part of runtime libraries (e.g., [Go](https://golang.org/src/runtime/memmove_amd64.s), [Glibc](https://github.com/ihtsae/glibc/commit/5f3d0b78e011d2a72f9e88b0e9ef5bc081d18f97), [LLVM](https://github.com/llvm/llvm-project/blob/96f3ea0d21b48ca088355db10d4d1a2e9bc9f884/lldb/tools/debugserver/source/MacOSX/i386/DNBArchImplI386.cpp), [Rust](https://github.com/rust-lang/rust/commit/070fad1701fb36b112853b0a6a9787a7bb7ff34c), [Java](http://hg.openjdk.java.net/jdk8u/jdk8u/hotspot/file/c1374141598c/src/cpu/x86/vm/stubGenerator_x86_64.cpp#l1297), [PHP](https://github.com/php/php-src/blob/e5cb53ec68603d4dbdd780fd3ecfca943b4fd383/ext/standard/string.c)). What distinguishes the simdjson library is that it is built from the ground up to benefit from these instructions.


 You should not expect the simdjson library to cause *downclocking* of your recent Intel CPU cores.
@@ -167,7 +167,7 @@ On some Intel processors, using SIMD instructions in a sustained manner on the s
 - [Whenever 512-bit AVX-512 instructions are used](https://lemire.me/blog/2018/09/07/avx-512-when-and-how-to-use-these-new-instructions/).
 - Whenever heavy 256-bit or wider instructions are used. Heavy instructions are those involving floating point operations or integer multiplications (since these execute on the floating point unit).

-The simdjson library does not currently support AVX-512 instructions and it does not make use of heavy 256-bit instructions. We do use vectorized multiplications, but only using 128-bit registers. Thus there should be no downclocking due to simdjson on recent processors. 
+The simdjson library does not currently support AVX-512 instructions and it does not make use of heavy 256-bit instructions. We do use vectorized multiplications, but only using 128-bit registers. Thus there should be no downclocking due to simdjson on recent processors.

 You may still be worried about which SIMD instruction set is used by simdjson.  Thankfully,  [you can always determine and change which architecture-specific implementation is used](implementation-selection.md) by simdjson. Thus even if your CPU supports AVX2, you do not need to use AVX2. You are in control.

@@ -1,7 +1,7 @@

-# Tape structure in simdjson 
+# Tape structure in simdjson

-We parse a JSON document to a tape. A tape is an array of 64-bit values. Each node encountered in the JSON document is written to the tape using one or more 64-bit tape elements; the layout of the tape is in "document order": elements are stored as they are encountered in the JSON document. 
+We parse a JSON document to a tape. A tape is an array of 64-bit values. Each node encountered in the JSON document is written to the tape using one or more 64-bit tape elements; the layout of the tape is in "document order": elements are stored as they are encountered in the JSON document.

 Throughout, little endian encoding is assumed. The tape is indexed starting at 0 (the first element is at index 0).

@@ -70,7 +70,7 @@ The following is a dump of the content of the tape, with the first number of eac
 Most tape elements are written as `('c' << 56) + x` where `'c'` is some ASCII character determining the type of the element (out of 't', 'f', 'n', 'l', 'u', 'd', '"', '{', '}', '[', ']' ,'r') and where `x` is a 56-bit value called the payload. The payload is normally interpreted as an unsigned 56-bit integer. Note that 56-bit integers can be quite large.


-Performance consideration: We believe that accessing the tape in regular units of 64 bits is more important for performance than saving memory. 
+Performance consideration: We believe that accessing the tape in regular units of 64 bits is more important for performance than saving memory.

 ## Simple JSON values

@@ -91,7 +91,7 @@ Integer values are represented as two 64-bit tape elements:
 Float values are represented as two 64-bit tape elements:
 - The 64-bit value `('d' << 56)` followed by the 64-bit double value literally in standard IEEE 754 notation.

-Performance consideration: We store numbers of the main tape because we believe that locality of reference is helpful for performance. 
+Performance consideration: We store numbers of the main tape because we believe that locality of reference is helpful for performance.

 ## Root node

@@ -109,20 +109,20 @@ Hint: We can read the first tape element to determine the length of the tape.

 We prefix the string data itself by a 32-bit header to be interpreted as a 32-bit integer. It indicates the length of the string. The actual string data starts at an offset of 4 bytes.

-We store string values using UTF-8 encoding with null termination on a separate tape. A string value is represented on the main tape as the 64-bit tape element `('"' << 56) + x` where the payload `x` is the location on the string tape of the null-terminated string. 
+We store string values using UTF-8 encoding with null termination on a separate tape. A string value is represented on the main tape as the 64-bit tape element `('"' << 56) + x` where the payload `x` is the location on the string tape of the null-terminated string.

-## Arrays 
+## Arrays

 JSON arrays are represented using two 64-bit tape elements.

- The first 64-bit tape element contains the value `('[' << 56) + x` where the payload `x` is 1 + the index of the second 64-bit tape element on the tape. 
+- The first 64-bit tape element contains the value `('[' << 56) + x` where the payload `x` is 1 + the index of the second 64-bit tape element on the tape.
 - The second 64-bit tape element contains the value `(']' << 56) + x` where the payload `x` contains the index of the first 64-bit tape element on the tape.

 All the content of the array is located between these two tape elements, including arrays and objects.

 Performance consideration: We can skip the content of an array entirely by accessing the first 64-bit tape element, reading the payload and moving to the corresponding index on the tape.

-## Objects 
+## Objects

 JSON objects are represented using two 64-bit tape elements.

@@ -131,6 +131,6 @@ JSON objects are represented using two 64-bit tape elements.

 In-between these two tape elements, we alternate between key (which must be strings) and values. A value could be an object or an array.

-All the content of the object is located between these two tape elements, including arrays and objects. 
+All the content of the object is located between these two tape elements, including arrays and objects.

 Performance consideration: We can skip the content of an object entirely by accessing the first 64-bit tape element, reading the payload and moving to the corresponding index on the tape.
@@ -57,4 +57,9 @@ IF(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
  add_quickstart_test(quickstart_noexceptions quickstart_noexceptions.cpp "" true)
  add_quickstart_test(quickstart_noexceptions11 quickstart_noexceptions.cpp c++11 true)
  set_property( TEST quickstart_noexceptions APPEND PROPERTY LABELS acceptance compile )
+
+  add_quickstart_test(quickstart2_noexceptions quickstart2_noexceptions.cpp "" true)
+  add_quickstart_test(quickstart2_noexceptions11 quickstart2_noexceptions.cpp c++11 true)
+  set_property( TEST quickstart2_noexceptions APPEND PROPERTY LABELS acceptance compile )
+
 endif()
@@ -0,0 +1,7 @@
+#include "simdjson.h"
+
+int main(void) {
+  simdjson::dom::parser parser;
+  simdjson::dom::element tweets = parser.load("twitter.json");
+  std::cout << "ID: " << tweets["statuses"].at(0)["id"] << std::endl;
+}
@@ -0,0 +1,13 @@
+#include "simdjson.h"
+
+int main(void) {
+  simdjson::dom::parser parser;
+  simdjson::dom::element tweets;
+  auto error = parser.load("twitter.json").get(tweets);
+  if(error) { std::cerr << error << std::endl; return EXIT_FAILURE; }
+  uint64_t identifier;
+  error = tweets["statuses"].at(0)["id"].get(identifier);
+  if(error) { std::cerr << error << std::endl; return EXIT_FAILURE; }
+  std::cout << identifier << std::endl;
+  return EXIT_SUCCESS;
+}
@@ -57,6 +57,9 @@ if(ENABLE_FUZZING)
  implement_fuzzer(fuzz_implementations) # parses and serializes again, compares across implementations
  implement_fuzzer(fuzz_minify)          # minify *with* parsing
  implement_fuzzer(fuzz_minifyimpl)      # minify *without* parsing, plus compare implementations
+  implement_fuzzer(fuzz_ndjson)          # the ndjson api
+  implement_fuzzer(fuzz_ondemand)
+  implement_fuzzer(fuzz_padded)
  implement_fuzzer(fuzz_parser)
  implement_fuzzer(fuzz_print_json)
  implement_fuzzer(fuzz_utf8)            # utf8 verification, compares across implementations
@@ -105,6 +105,14 @@ struct FuzzData {
    return {};
  }

+  // consumes the rest of the data as a string view
+  std::string_view remainder_as_stringview() {
+    std::string_view ret{chardata(),Size};
+    Data+=Size;
+    Size=0;
+    return ret;
+  }
+
  // split the remainder of the data into string views,
  std::vector<std::string_view> splitIntoStrings() {
    std::vector<std::string_view> ret;
@@ -83,3 +83,18 @@ build-sanitizers/fuzz/fuzz_parser my_testcase.json
 ```
 In case this does not reproduce the bug, you may want to proceed with reproducing using the oss-fuzz tools. See the instructions [here](https://google.github.io/oss-fuzz/advanced-topics/reproducing/).

+# Minimizing and cleansing crashes
+If a crashing case is found, it is useful to minimize it and cleanse it (replace irrelevant bytes with spaces).
+
+```shell
+build-sanitizers-O0/fuzz/fuzz_ndjson out/ndjson
+# ...crashes and writes the crash-... file
+# minimize it:
+build-sanitizers-O0/fuzz/fuzz_ndjson crash-xxxxxxx -minimize_crash=1 -exact_artifact_path=minimized_crash -max_total_time=100
+
+# replace irrelevant parts with space:
+build-sanitizers-O0/fuzz/fuzz_ndjson minimized_crash -cleanse_crash=1 -exact_artifact_path=cleansed_crash
+
+# use/share cleansed_crash
+
+```
@@ -4,71 +4,110 @@
 # - different sanitizers
 # - different build options
 # - reproduce build, for running through valgrind
+#
+# Set environment variable CLANGSUFFIX to select clang version (example: "-11")

 # fail on error
-set -eu
+set -e

 unset CXX CC CFLAGS CXXFLAGS LDFLAGS

 me=$(basename $0)

+
+if [ -z $CLANGSUFFIX ] ; then
+    # the default clang version is set low enough to be found on current Debian stable (Buster)
+    CLANGSUFFIX=-8
+fi
+
+# detect unset variables
+set -u
+
 # common options
-CLANGVER=-9
-COMMON="-GNinja -DCMAKE_CXX_COMPILER=clang++$CLANGVER -DCMAKE_C_COMPILER=clang$CLANGVER -DSIMDJSON_BUILD_STATIC=Off -DENABLE_FUZZING=On -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_GIT=Off"
+CXX_CLAGS_COMMON=-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+COMMON="-GNinja -DCMAKE_CXX_COMPILER=clang++$CLANGSUFFIX -DCMAKE_C_COMPILER=clang$CLANGSUFFIX -DSIMDJSON_BUILD_STATIC=Off -DENABLE_FUZZING=On -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_DISABLE_DEPRECATED_API=On"

 # A replay build, as plain as it gets. For use with valgrind/gdb.
 variant=replay
 if [ ! -d build-$variant ] ; then
    mkdir build-$variant
    cd build-$variant
-    
+
    cmake .. \
 	  $COMMON \
 	  -DCMAKE_BUILD_TYPE=Debug \
 	  -DSIMDJSON_FUZZ_LINKMAIN=On
-    
+
    ninja all_fuzzers
    cd ..
 fi


 # A fuzzer with sanitizers. For improved capability to find bugs.
-variant=sanitizers
+# About the optimization level: (measured for fuzz_atpointer)
+# -O0 gives  4000 executions/s
+# -O1 gives 20000 executions/s
+# -O2 gives 32000 executions/s
+# -O3 gives 32000 executions/s
+# for reference, the release build (without sanitizers, but with fuzzing instrumentation)
+# gives 80000 executions/s.
+# A low level is good for debugging. A higher level gets more work done.
+# Different levels may uncover different types of bugs, see this interesting
+# thread: https://github.com/google/oss-fuzz/issues/2295#issuecomment-481493392
+# Oss-fuzz uses -O1 so it may be relevant to use something else than that,
+# to do something oss-fuzz doesn't.
+variant=sanitizers-O3

    if [ ! -d build-$variant ] ; then
-	
+
 	mkdir build-$variant
 	cd build-$variant
-	
 	cmake .. \
 	      $COMMON \
-	      -DCMAKE_CXX_FLAGS="-fsanitize=fuzzer-no-link,address,undefined -fno-sanitize-recover=undefined" \
-	      -DCMAKE_C_FLAGS="-fsanitize=fuzzer-no-link,address,undefined -fno-sanitize-recover=undefined" \
+	      -DCMAKE_CXX_FLAGS="-O3 -fsanitize=fuzzer-no-link,address,undefined -fno-sanitize-recover=undefined $CXX_CLAGS_COMMON" \
+	      -DCMAKE_C_FLAGS="-O3 -fsanitize=fuzzer-no-link,address,undefined -fno-sanitize-recover=undefined" \
 	      -DCMAKE_BUILD_TYPE=Debug \
 	      -DSIMDJSON_FUZZ_LINKMAIN=Off \
 	      -DSIMDJSON_FUZZ_LDFLAGS="-fsanitize=fuzzer"
-	
+
 	ninja all_fuzzers
 	cd ..
    fi

+variant=sanitizers-O0
+
+    if [ ! -d build-$variant ] ; then
+
+	mkdir build-$variant
+	cd build-$variant
+	cmake .. \
+	      $COMMON \
+	      -DCMAKE_CXX_FLAGS="-O0 -fsanitize=fuzzer-no-link,address,undefined -fno-sanitize-recover=undefined $CXX_CLAGS_COMMON" \
+	      -DCMAKE_C_FLAGS="-O0 -fsanitize=fuzzer-no-link,address,undefined -fno-sanitize-recover=undefined" \
+	      -DCMAKE_BUILD_TYPE=Debug \
+	      -DSIMDJSON_FUZZ_LINKMAIN=Off \
+	      -DSIMDJSON_FUZZ_LDFLAGS="-fsanitize=fuzzer"
+
+	ninja all_fuzzers
+	cd ..
+    fi


 # A fast fuzzer, for fast exploration rather than finding bugs.
 variant=fast
 if [ ! -d build-$variant ] ; then
-	
+
 	mkdir build-$variant
 	cd build-$variant
-	
+
 	cmake .. \
 	      $COMMON \
-	      -DCMAKE_CXX_FLAGS="-fsanitize=fuzzer-no-link" \
+	      -DCMAKE_CXX_FLAGS="-fsanitize=fuzzer-no-link $CXX_CLAGS_COMMON" \
 	      -DCMAKE_C_FLAGS="-fsanitize=fuzzer-no-link" \
 	      -DCMAKE_BUILD_TYPE=Release \
 	      -DSIMDJSON_FUZZ_LINKMAIN=Off \
 	      -DSIMDJSON_FUZZ_LDFLAGS="-fsanitize=fuzzer"
-	
+
 	ninja all_fuzzers
 	cd ..
    fi
@@ -14,7 +14,7 @@ export OUT=$(pwd)/ossfuzz-out
 export CC=clang
 export CXX="clang++"
 export CFLAGS="-fsanitize=fuzzer-no-link"
-export CXXFLAGS="-fsanitize=fuzzer-no-link,address,undefined -O1"
+export CXXFLAGS="-fsanitize=fuzzer-no-link,address,undefined -O1 -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION"
 export LIB_FUZZING_ENGINE="-fsanitize=fuzzer"

 $ossfuzz
@@ -58,7 +58,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
    simdjson_unused auto v = elem fun;                                                         \
    break;                                                                     \
  }
+#if SIMDJSON_EXCEPTIONS
  try {
+#endif

    switch (action) {
      CASE(0, type);
@@ -122,9 +124,11 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
 #undef CASE
 #undef CASE2

+#if SIMDJSON_EXCEPTIONS
  } catch (std::exception &) {
    // do nothing
  }
+#endif

  return 0;
 }
@@ -73,7 +73,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
    // or what hardware it runs on
    constexpr std::size_t Nimplementations_max=3;
    const std::size_t Nimplementations = supported_implementations.size();
-    
+
    if(Nimplementations>Nimplementations_max) {
        //there is another backend added, please bump Nimplementations_max!
        std::abort();
@@ -0,0 +1,31 @@
+#include "simdjson.h"
+#include <cstddef>
+#include <cstdint>
+#include <string>
+
+#include "FuzzUtils.h"
+#include "NullBuffer.h"
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
+  FuzzData fd(Data, Size);
+  const auto batch_size=static_cast<size_t>(fd.getInt<0,1000>());
+  const auto json=simdjson::padded_string{fd.remainder_as_stringview()};
+  simdjson::dom::parser parser;
+#if SIMDJSON_EXCEPTIONS
+  try {
+#endif
+    simdjson::dom::document_stream docs;
+    if(parser.parse_many(json,batch_size).get(docs)) {
+      return 0;
+    }
+
+  size_t bool_count=0;
+  for (auto doc : docs) {
+    bool_count+=doc.is_bool();
+  }
+#if SIMDJSON_EXCEPTIONS
+  } catch(...) {
+  }
+#endif
+  return 0;
+}
@@ -0,0 +1,74 @@
+#include "FuzzUtils.h"
+#include "simdjson.h"
+#include <cstddef>
+#include <cstdint>
+#include <string>
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
+  FuzzData fd(Data, Size);
+  const int action = fd.getInt<0, 12>();
+
+  // split the remainder of the document into strings
+  auto strings = fd.splitIntoStrings();
+  while (strings.size() < 1) {
+    strings.emplace_back();
+  }
+#if SIMDJSON_EXCEPTIONS
+  try {
+#endif
+    simdjson::builtin::ondemand::parser parser;
+    simdjson::padded_string padded(strings[0]);
+    auto doc = parser.iterate(padded);
+    if (doc.error()) {
+      return 0;
+    }
+    for (auto item : doc) {
+      switch (action) {
+      case 0: {
+        simdjson_unused auto x = item.get_string();
+      } break;
+      case 1: {
+        simdjson_unused auto x = item.get_bool();
+      } break;
+      case 2: {
+        simdjson_unused auto x = item.get_array();
+      } break;
+      case 3: {
+        simdjson_unused auto x = item.get_int64();
+      } break;
+      case 4: {
+        simdjson_unused auto x = item.get_double();
+      } break;
+      case 5: {
+        simdjson_unused auto x = item.get_object();
+      } break;
+      case 6: {
+        simdjson_unused auto x = item.get_uint64();
+      } break;
+      case 7: {
+        simdjson_unused auto x = item.get_raw_json_string();
+      } break;
+      case 8: {
+        simdjson_unused auto x = item.is_null();
+      } break;
+      case 9: {
+        simdjson_unused auto x = item.begin();
+      } break;
+      case 10: {
+        simdjson_unused auto x = item.end();
+      } break;
+      case 11: {
+        for (auto e : item) {
+          simdjson_unused auto x = e.is_null();
+        }
+      } break;
+      default:;
+      }
+    }
+#if SIMDJSON_EXCEPTIONS
+  } catch (...) {
+  }
+#endif
+
+  return 0;
+}
@@ -0,0 +1,54 @@
+#include "FuzzUtils.h"
+#include "simdjson.h"
+#include <cstddef>
+#include <cstdint>
+#include <fstream>
+#include <string>
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
+  FuzzData fd(Data, Size);
+
+  const int action = fd.getInt<0, 10>();
+
+  const auto s1 = fd.get<size_t>();
+  const auto s2 = fd.get<size_t>();
+  switch (action) {
+  case 0: {
+    simdjson_unused simdjson::padded_string p(s1);
+  } break;
+  case 1: {
+    // operator== with temp value
+    simdjson_unused simdjson::padded_string p1(s1);
+    simdjson_unused simdjson::padded_string p2(s2);
+    p1 = std::move(p2);
+  } break;
+  case 2: {
+    // swap
+    simdjson_unused simdjson::padded_string p1(s1);
+    simdjson_unused simdjson::padded_string p2(s2);
+    p1.swap(p2);
+  } break;
+  case 3: {
+    simdjson_unused simdjson::padded_string p(s1);
+    p.data();
+  } break;
+  case 4: {
+    simdjson::padded_string p(s1);
+    simdjson_unused auto sv = static_cast<std::string_view>(p);
+  } break;
+  case 5: {
+    // load from file.
+    const std::string filename = "/dev/shm/fuzz_padded.tmp";
+    {
+      std::ofstream file(filename);
+      assert(file);
+      const long ssize = static_cast<long>(fd.Size);
+      file.write(fd.chardata(), ssize);
+      assert(file.tellp() == ssize);
+    }
+    simdjson_unused auto data = simdjson::padded_string::load(filename);
+  } break;
+  default:;
+  }
+  return 0;
+}
@@ -25,6 +25,7 @@ if [ ! -d $bdir ] ; then
          -DCMAKE_BUILD_TYPE=Debug \
          -DSIMDJSON_BUILD_STATIC=On \
          -DENABLE_FUZZING=On \
+          -DSIMDJSON_DISABLE_DEPRECATED_API=On \
 	  -DSIMDJSON_FUZZ_LINKMAIN=On
    ninja all_fuzzers
    cd ..
@@ -0,0 +1,64 @@
+#!/bin/sh
+
+# usage: fuzzer crash
+
+
+fuzzer=$1
+
+if [ ! -x $fuzzer ] ; then
+  echo "arg 1 should be a fuzzer executable"
+  exit 1
+fi
+
+crash=$2
+if [ ! -e $crash ] ; then
+  echo "arg2 should be a crashing test case"
+  exit 1
+fi
+if [ -d $crash ] ; then
+  echo "crash should be a file, not a dir"
+  exit 1
+fi
+
+echo "checking that the crash crashes..."
+origreport=$(mktemp --tmpdir orig_crash_report.XXXXXXXXX)
+if $fuzzer $crash >$origreport 2>&1; then
+  echo "your crash does not crash."
+  exit 1
+else
+  echo "...it does."
+fi
+
+sizeofcrash=$(stat --format=%s $crash)
+echo "starting to minimize crash of size $sizeofcrash..."
+minimized=$(mktemp --tmpdir minimized_crash.XXXXXXXXX)
+$fuzzer -minimize_crash=1 -exact_artifact_path=$minimized -max_total_time=5 $crash  >/dev/null 2>&1
+
+sizeofminimized=$(stat --format=%s $minimized)
+echo "got it down to $sizeofminimized"
+
+echo "checking that the minimized crash crashes..."
+report=$(mktemp --tmpdir minimized_crash_report.XXXXXXXXX)
+if $fuzzer $minimized >$report 2>&1; then
+  echo "your minimized crash does not crash."
+  exit 1
+else
+ echo "...it does."
+fi
+
+echo "starting cleansing..."
+cleansed=$(mktemp --tmpdir cleansed_crash.XXXXXXXXX)
+cleansingreport=$(mktemp --tmpdir cleansing_output.XXXXXXXXX)
+$fuzzer $minimized -cleanse_crash=1 -exact_artifact_path=$cleansed >$cleansingreport 2>&1
+
+echo "checking that the cleansed crash crashes..."
+report=$(mktemp --tmpdir cleansed_crash_report.XXXXXXXXX)
+if $fuzzer $cleansed >$report 2>&1; then
+  echo "your cleansed crash $cleansed does not crash. see cleansing report: $cleansingreport"
+  exit 1
+else
+  echo "....it does."
+fi
+
+echo "your minimized and cleansed crash (report $origreport) is here: $cleansed and the report for the cleansed crash is here: $report"
+
@@ -30,8 +30,8 @@ cmake .. \
 -DENABLE_FUZZING=On \
 -DSIMDJSON_COMPETITION=Off \
 -DSIMDJSON_FUZZ_LINKMAIN=Off \
-DSIMDJSON_GIT=Off \
 -DSIMDJSON_GOOGLE_BENCHMARKS=Off \
+-DSIMDJSON_DISABLE_DEPRECATED_API=On \
 -DSIMDJSON_FUZZ_LDFLAGS=$LIB_FUZZING_ENGINE

 cmake --build . --target all_fuzzers
@@ -27,7 +27,11 @@ if [ ! -d out ] ; then
  tar xf corpus.tar && rm corpus.tar
 fi

-builddir=build-sanitizers
+# By default, use the debug friendly variant since this script is intended
+# for developers reproducing bugs/validating fixes locally.
+builddir=build-sanitizers-O0
+#...but feel free to use this one instead, if you want to build coverage fast.
+#builddir=build-fast

 if [ ! -d $builddir ] ; then
  fuzz/build_fuzzer_variants.sh
@@ -5,7 +5,7 @@
 * @mainpage
 *
 * Check the [README.md](https://github.com/lemire/simdjson/blob/master/README.md#simdjson--parsing-gigabytes-of-json-per-second).
- * 
+ *
 * Sample code. See https://github.com/simdjson/simdjson/blob/master/doc/basics.md for more examples.

    #include "simdjson.h"
@@ -21,7 +21,7 @@
        {  "12345" : {"a":12.34, "b":56.78, "c": 9998877}   },
        {  "12545" : {"a":11.44, "b":12.78, "c": 11111111}  }
        ] )"_padded;
-  
+
      for (simdjson::dom::object obj : parser.parse(abstract_json)) {
        for(const auto& key_value : obj) {
          cout << "key: " << key_value.key << " : ";
@@ -85,6 +85,7 @@ SIMDJSON_DISABLE_UNDESIRED_WARNINGS
 #include "simdjson/arm64.h"
 #include "simdjson/haswell.h"
 #include "simdjson/westmere.h"
+#include "simdjson/ppc64.h"
 #include "simdjson/fallback.h"
 #include "simdjson/builtin.h"

@@ -12,7 +12,7 @@ NO_SANITIZE_UNDEFINED
 simdjson_really_inline int trailing_zeroes(uint64_t input_num) {
 #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
  unsigned long ret;
-  // Search the mask data from least significant bit (LSB) 
+  // Search the mask data from least significant bit (LSB)
  // to the most significant bit (MSB) for a set bit (1).
  _BitScanForward64(&ret, input_num);
  return (int)ret;
@@ -30,7 +30,7 @@ simdjson_really_inline uint64_t clear_lowest_bit(uint64_t input_num) {
 simdjson_really_inline int leading_zeroes(uint64_t input_num) {
 #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
  unsigned long leading_zero = 0;
-  // Search the mask data from most significant bit (MSB) 
+  // Search the mask data from most significant bit (MSB)
  // to least significant bit (LSB) for a set bit (1).
  if (_BitScanReverse64(&leading_zero, input_num))
    return (int)(63 - leading_zero);
@@ -13,15 +13,15 @@ namespace {
 simdjson_really_inline uint64_t prefix_xor(uint64_t bitmask) {
  /////////////
  // We could do this with PMULL, but it is apparently slow.
-  //  
+  //
  //#ifdef __ARM_FEATURE_CRYPTO // some ARM processors lack this extension
  //return vmull_p64(-1ULL, bitmask);
  //#else
  // Analysis by @sebpop:
  // When diffing the assembly for src/stage1_find_marks.cpp I see that the eors are all spread out
-  // in between other vector code, so effectively the extra cycles of the sequence do not matter 
+  // in between other vector code, so effectively the extra cycles of the sequence do not matter
  // because the GPR units are idle otherwise and the critical path is on the FP side.
-  // Also the PMULL requires two extra fmovs: GPR->FP (3 cycles in N1, 5 cycles in A72 ) 
+  // Also the PMULL requires two extra fmovs: GPR->FP (3 cycles in N1, 5 cycles in A72 )
  // and FP->GPR (2 cycles on N1 and 5 cycles on A72.)
  ///////////
  bitmask ^= bitmask << 1;
@@ -9,7 +9,7 @@ namespace {
 // credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/
 static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) {
  uint64_t val;
-  memcpy(&val, chars, sizeof(uint64_t));
+  std::memcpy(&val, chars, sizeof(uint64_t));
  val = (val & 0x0F0F0F0F0F0F0F0F) * 2561 >> 8;
  val = (val & 0x00FF00FF00FF00FF) * 6553601 >> 16;
  return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32);
@@ -423,7 +423,7 @@ simdjson_really_inline int8x16_t make_int8x16_t(int8_t x1,  int8_t x2,  int8_t x
    simd8x64(const simd8x64<T>& o) = delete; // no copy allowed
    simd8x64<T>& operator=(const simd8<T> other) = delete; // no assignment allowed
    simd8x64() = delete; // no default constructor allowed
-    
+
    simdjson_really_inline simd8x64(const simd8<T> chunk0, const simd8<T> chunk1, const simd8<T> chunk2, const simd8<T> chunk3) : chunks{chunk0, chunk1, chunk2, chunk3} {}
    simdjson_really_inline simd8x64(const T ptr[64]) : chunks{simd8<T>::load(ptr), simd8<T>::load(ptr+16), simd8<T>::load(ptr+32), simd8<T>::load(ptr+48)} {}

--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Daniel Lemire	cbacec0760	Releasing 0.7.0.	2020-12-04 18:56:15 -05:00
0xflotus	82fcf141f7	fix: small typo error (#1328 )	2020-12-03 13:16:45 -05:00
Daniel Lemire	aaa12c8fb6	more thread testing (#1324 ) * Adding a few tests. * More tests. * Trailing. * More links/documentations.	2020-12-02 11:33:36 -05:00
Daniel Lemire	9b5150daef	Fixing issue 1247 (#1325 )	2020-12-01 19:09:55 -05:00
Daniel Lemire	9304d88920	Prototype test for issue 1299: using parse_many, find the location of the end of the last document (#1301 ) * Prototype test for issue 1299. * This improves the documentation. * Removing trailing white spaces. * Removing trailing spaces * Trailing.	2020-12-01 15:59:20 -05:00
Paul Dreik	725ca010e7	add ndjson fuzzer (#1304 ) * add ndjson fuzzer * reproduce #1310 in the newly added unit test Had to replace the input, because: 1) the fuzzer uses the first part of the input to determine the batch_size to use, so that has to be cut off 2) the master now protects against low values of batch_size I also made the test not return early, so the error is triggered.	2020-12-01 15:58:41 -05:00
Daniel Lemire	59c857e969	Merge branch 'master' of github.com:simdjson/simdjson	2020-12-01 15:31:44 -05:00
Daniel Lemire	45c9136443	Fixed odd issue.	2020-12-01 15:31:24 -05:00
Daniel Lemire	bc4087ac96	Hunting for data races in document_stream (#1323 ) * Adding matching tests. * Actually adding thread testing actions.	2020-12-01 12:40:15 -05:00
Daniel Lemire	d9c4191e8a	This should fix an issue we have with unclosed strings with document_stream (#1321 ) * This should fix an issue we have with unclosed strings. * Patching the fallback kernel. * Better guarding the code. * Patching the fallback.	2020-11-30 13:43:57 -05:00
Daniel Lemire	3fa40b8dc2	Adding an example corresponding to issue 1316 (documentation enhancement) (#1317 ) * Adding an example. * Updated other doc file. * Trying to take into account @jkeiser's comments. * Some people prefer empty final lines.	2020-11-27 17:40:29 -05:00
Daniel Lemire	dc69bc28ae	Trying to verify recent document stream issues. (#1318 ) * Trying to verify recent document stream issues. * Adding another one. * More thorough tests. * Removing trailing spaces. * Working toward exposing some issues. * Tweaking.	2020-11-27 17:04:10 -05:00
Daniel Lemire	53577f11e1	Explicit worker disposal.	2020-11-27 15:39:43 -05:00
Daniel Lemire	1abb64f6f6	This fixes issue 1302 (#1303 ) * This verifies issue 1302 * I believe that this fixes the issue. * Let us do it differently. * Better comments.	2020-11-27 14:19:36 -05:00
Mikhail Matrosov	b442b29ccf	Mark comparison operators as const (#1320 ) Co-authored-by: Mikhail Matrosov <mikhail.matrosov@aimtech.com>	2020-11-27 14:18:42 -05:00
Daniel Lemire	5609851747	This will guard the batch_size so it cannot be so low as to accidentally burn through your CPU. (#1319 ) * This will guard the batch_size so it cannot be so low as to accidentally burn through your CPU.	2020-11-27 09:37:54 -05:00
Paul Dreik	68a8004518	don't memcpy after failed alloc (#1315 ) Should fix https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=27675	2020-11-21 07:52:24 +01:00
Paul Dreik	0b39e3a6cf	add fuzzer for padded_string (#1312 ) This also fixes an overflow problem.	2020-11-19 16:51:56 +01:00
Paul Dreik	924ad2d592	when amalagamating, replace the SIMDJSON_IMPLEMENTATION (#1306 )	2020-11-17 20:50:49 +01:00
Paul Dreik	5202d07a77	add script and instructions for minimizing and cleansing fuzz crashes (#1305 )	2020-11-17 20:49:29 +01:00
friendlyanon	b632107a7a	Use the org's own PPA for CMake (#1295 ) Co-authored-by: friendlyanon <friendlyanon@users.noreply.github.com>	2020-11-10 13:55:59 -05:00
Paul Dreik	f62ca21dd1	enable boost json (#1292 ) * bump boost.json and see if it works in simdjson CI * enable boost json * clean up * add boost json to deps * use boost if std::string_view is available * add build with c++20 * use docker image which has the proper libc++ installed	2020-11-10 13:55:04 -05:00
friendlyanon	553befa012	Disable Github CI in feature branches (#1298 ) Forks that would like to contribute via PRs from feature branches needlessly run CI on those branches on top of the PRs. This is a waste of resources. Co-authored-by: friendlyanon <friendlyanon@users.noreply.github.com>	2020-11-10 13:52:58 -05:00
Paul Dreik	def624a50c	update version tag (#1297 )	2020-11-08 10:17:06 -05:00
Daniel Lemire	268f26a84f	Tweak of the documentation.	2020-11-07 16:33:12 -05:00
Paul Dreik	d9456b3030	fix numeric_limits usage problem on windows caused by the max macro (#1293 ) See https://stackoverflow.com/questions/2789481/problem-calling-stdmax/2789509#2789509	2020-11-06 06:55:27 +01:00
Daniel Lemire	f195f849d0	Helping contributors. (#1289 )	2020-11-05 10:27:35 -05:00
friendlyanon	669e5b91b4	Improve CI workflows (#1291 ) * Fix Cirrus CI CMake is not installed at this point yet. * Add caching in Circle CI * Add caching to the MinGW Github workflows * Fix Circle CI config Co-authored-by: friendlyanon <friendlyanon@users.noreply.github.com>	2020-11-05 08:42:00 -05:00
Daniel Lemire	a06e0958ca	We do not want the amalgamate.py script to run ever if you do not have at least Python 3. (#1290 )	2020-11-04 13:52:52 -05:00
friendlyanon	a56149e79a	Cache CMake dependencies in CI (#1282 ) Co-authored-by: friendlyanon <friendlyanon@users.noreply.github.com>	2020-11-04 13:35:33 -05:00
friendlyanon	c805fc28a4	Remove git modules (#1258 ) * Bump minimum CMake version * Remove unnecessary git checks * Move benchmark options where they are used * Declare helper functions for dependencies The custom solution here is tailored for fast configure times, but only works for dependencies on Github. * Import dependencies using the declared commands * Remove git submodules * Call target_link_libraries properly target_link_libraries must not be called without a requirement specifier. * Fix includes for competition Co-authored-by: friendlyanon <friendlyanon@users.noreply.github.com>	2020-11-04 13:34:29 -05:00
Daniel Lemire	218c274090	Updating main branch for legacy libc++ support (#1288 ) * Updating main branch for legacy libc++ support * Adopting * Removing unnecessary math header. * Updating the single-header files so we can pass the new tests. * Portable infinite-value detection is hard. * Working toward disabling boost json selectively. * Selectively disabling Boost JSON * More work toward selectively disabling boost json.	2020-11-04 12:24:42 -05:00
Tyler Kennedy	921c79f26f	Add method to get the total number of slots used by an array in the t… (#1253 ) * Add method to get the total number of slots used by an array in the tape. * Whoops	2020-11-04 10:35:54 -05:00
Paul Dreik	af4db55e66	remove trailing whitespace (#1284 )	2020-11-03 21:48:09 +01:00
Paul Dreik	9f78559cc8	detect trailing whitespace in a CI job (#1268 )	2020-11-03 20:41:19 +01:00
Paul Dreik	23b4bc93aa	move amalgamate from bash to python (#1278 ) This is much faster (from 3.5 to 0.14 seconds)	2020-11-03 07:35:16 +01:00
Daniel Lemire	6321df078a	This fixes very minor "mistakes". (#1281 )	2020-11-02 20:04:04 -05:00
Convery	e6258377d9	Additional define to silence the 32-bit portability warning. (#1279 ) Under compilers like MSVC the `#pragma message` about portability will be issued for each translation module; regardless of whether or not `SIMDJSON_PORTABILITY_H` has already been defined. As such a new define `SIMDJSON_NO_PORTABILITY_WARNING`, can be defined prior to the inclusion of `<simdjson.h>` to silence it.	2020-11-02 08:37:13 -05:00
Jeremy Ong	b3a26fd0cc	Do not enable -fPIC when compiling on Windows (#1280 ) Regardless of the compiler, Windows targets do not support -fPIC, as position independent code is already implicitly enabled. Compiling simdjson with Clang on Windows will error because -fPIC is an unsupported option for target 'x86_64-pc-windows-msvc'.	2020-11-02 08:18:04 -05:00
Paul Dreik	47669566da	fix unintended early return in ondemand unit test loops (#1263 ) fix uninteded early return in ondemand unit test loops go through and fix warnings appearing in qtcreator, qualify with std::, add const, abort on error get rid of ulp_distance, not needed anymore when parsing is exact	2020-11-01 19:08:40 +01:00
Paul Dreik	03a1bb0a5b	detect if amalgamation is needed (#1269 ) output amalgamation date to be locale and timezone independent	2020-11-01 19:07:15 +01:00
Paul Dreik	54ffbbe7db	remove asserts from compute_float_64 (#1276 )	2020-11-01 18:34:42 +01:00
Paul Dreik	0b82f07115	fix segfault in numberparsing #1273 (#1274 ) This was a read overflow.	2020-11-01 18:27:21 +01:00
Paul Dreik	265db2e533	fix non ascii sources (#1275 ) Master does not build because of non-ascii sources, merging without waiting for CI.	2020-11-01 11:14:01 +01:00
Paul Dreik	2b2abf4895	reamalgamate	2020-11-01 07:03:17 +01:00
Paul Dreik	f93fb21c95	optionally disable deprecated apis (#1271 ) Introduce cmake option SIMDJSON_DISABLE_DEPRECATED_API (default Off) which turns off deprecated simdjson api functions by setting the macro SIMDJSON_DISABLE_DEPRECATED_API. For non-cmake users, users will have to set SIMDJSON_DISABLE_DEPRECATED_API by some other means to disable the api. Closes #1264	2020-11-01 06:38:52 +01:00
Paul Dreik	b7fe764e6c	fuzz more of generic/numberparsing.h (#1272 ) make the ondemand fuzzer use more of the api	2020-11-01 06:29:16 +01:00
Paul Dreik	500e4c3572	fuzz with the intended clang version (#1267 ) This builds the CI fuzzers with the intended clang version. It also allows users to set the clang version locally, in case they need to. It also switches the CI fuzzers to use an optimized sanitizer build, to do something oss-fuzz doesn't and get more done in the short time the CI fuzzer runs.	2020-10-31 08:22:49 +01:00
friendlyanon	55281c01fb	Fix unintended Drone CI break (#1266 ) * Use 'set -e' as it is portable * Install git on the i386 images * Remove CMake PPA from arm64 images CMake's PPA doesn't actually distribute arm64 binary packages. See related issues: https://gitlab.kitware.com/cmake/cmake/-/issues/17923 https://gitlab.kitware.com/cmake/cmake/-/issues/20122 * Use Debian Buster instead of Ubuntu Bionic Beaver Buster has all the packages in apt that is needed, which is convenient * Install Clang 6.0 explicitly Make sure that what gets installed is clang 6.0 as the step name suggests. https://packages.debian.org/buster/clang-6.0 * Install CMake from buster-backports Co-authored-by: friendlyanon <friendlyanon@users.noreply.github.com>	2020-10-30 22:47:40 -04:00
Daniel Lemire	a8bf10ea5a	Minor patch.	2020-10-30 14:51:50 -04:00
friendlyanon	9f60093b41	Update CMake to at least 3.16 in Drone CI (#1261 ) * Add script for CMake PPA * Call the CMake PPA script in Drone CI "apt-get update -qq" can be omitted, as that command is already called by the script to pull in necessary packages for the CMake GPG keys. * Remove sudo calls in the CMake PPA script This script is intended to be run in Docker images, where the default user is already root. * Use echo instead of printf * Use /etc/os-release instead of lsb_release lsd_release could be installed, but os-release is just more convenient to grab the version code from at this point. * On Debian images grab CMake from buster-backports It's not wise to mix Ubuntu PPAs with Debian and buster-backports has CMake 3.16, which is recent enough for our purposes. Co-authored-by: friendlyanon <friendlyanon@users.noreply.github.com>	2020-10-30 13:55:35 -04:00
Daniel Lemire	562d4f4f58	Fixing a minor logical error.	2020-10-29 16:42:50 -04:00
Paul Dreik	ac87437588	fuzz the on demand api (#1220 )	2020-10-29 19:14:44 +01:00
Daniel Lemire	b1444b4dfb	Mostly tiny changes, with one optimization to fallback for number parsing. (#1265 ) * Mostly tiny changes, with one optimization to fallback for number parsing. * Missed an update.	2020-10-29 11:18:11 -04:00
friendlyanon	ed21875083	Remove old CMake version (#1259 ) The images used by github already come with a sufficiently recent CMake version instead of a very outdated one. See: https://github.com/actions/virtual-environments Co-authored-by: friendlyanon <friendlyanon@users.noreply.github.com>	2020-10-28 09:44:13 -04:00
Daniel Lemire	0249b5b311	Update CONTRIBUTORS	2020-10-27 18:44:28 -04:00
Danila Kutenin	f46a0f64f2	PPC64 support (#1254 ) * Initial PPC64 support * Add travis CI * Fix outdated cmake version for travis * Fix indendtation * Try another workaround for outdated cmake in travis * Try beta cmake * Add dash before beta * Use builtin snaps * Use cmake as rocksdb * Test cmake on bionic * Remove unnecessary things from travis * Remove unnecessary things from travis * Another try of compiler install * Add all major compilers * Add all major compilers * Add all major compilers * Tweak travis a bit * Typo * More robust travis * Typos typos typos * Add fewer compilers, add non specific build for clang and gcc, should be the final config * CMAKE_FLAGS is in incorrect place * Remove default implementation * Limit build thread number * Fall back prefix_xor to a usual implementation, no performance boost is noticed * Test for power9 as it is the main architecture for OpenPOWER right now * Add to documentation to build with power9 as the implementation is compatible but compiler optimizations is not * Replace ARM with PPC in the comment	2020-10-27 18:43:39 -04:00
Jonathan Wakely	1fd0447dbb	Remove repeated words (#1252 )	2020-10-26 20:41:01 -04:00
Jeremy Ong	9856201f5c	Disable exceptions when compiling with MSVC (#1256 ) Projects that link simdjson from MSVC with exceptions off will include simdjson headers which transitively include STL headers. The MSVC STL stipulates that _HAS_EXCEPTIONS=0 be defined or code requiring exceptions will be enabled. This change adds a new job to the appveyor build matrix to verify the build and tests with exceptions disabled, and disables exceptions at the compiler level when SIMDJSON_EXCEPTIONS is specified to OFF.	2020-10-26 14:11:25 -04:00
Daniel Lemire	a75c07065f	Fix for issue 1246. We document the relationship between parser instances and elements (#1250 ) * Fix for issue 1246. * Adopting John's wording.	2020-10-26 08:40:45 -04:00