release candidate (#1132)

* This improves our documentation.

* Removing tags for doxygen.

* You need a recent cmake remark.

.appveyor.yml

@@ -8,8 +8,6 @@ environment:
   matrix:
     - job_name: VS2019
       CMAKE_ARGS:  -A %Platform%
-    - job_name: VS2019CLANG
-      CMAKE_ARGS:  -A %Platform% -T ClangCL
     - job_name: VS2019ARM
       CMAKE_ARGS: -A ARM64 -DCMAKE_CROSSCOMPILING=1 -D SIMDJSON_GOOGLE_BENCHMARKS=OFF # Does Google Benchmark builds under VS ARM?
     - job_name: VS2017 (Static, No Threads)
@@ -20,6 +18,10 @@ environment:
       platform: Win32
       CMAKE_ARGS:  -A %Platform% -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_ENABLE_THREADS=ON # This should be the default. Testing anyway.
       CTEST_ARGS: -E checkperf
+    - job_name: VS2015
+      image: Visual Studio 2015
+      CMAKE_ARGS:  -A %Platform% -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_ENABLE_THREADS=OFF
+      CTEST_ARGS: -E checkperf
 
 build_script:
   - mkdir build

.circleci/config.yml

@@ -2,15 +2,6 @@ version: 2.1
 
 # Reusable image / compiler definitions
 executors:
-  gcc7:
-    docker:
-      - image: gcc:7
-        environment:
-          CXX: g++
-          CC: gcc
-          BUILD_FLAGS: -j
-          CTEST_FLAGS: -j4 --output-on-failure
-
   gcc8:
     docker:
       - image: conanio/gcc8
@@ -110,24 +101,23 @@ commands:
       - run: cd build && make install
       - run: echo -e '#include <simdjson.h>\nint main(int argc,char**argv) {simdjson::dom::parser parser;simdjson::dom::element tweets = parser.load(argv[1]); }' > tmp.cpp && c++ -Ibuild/destination/include -Lbuild/destination/lib -std=c++17 -Wl,-rpath,build/destination/lib -o linkandrun tmp.cpp -lsimdjson && ./linkandrun jsonexamples/twitter.json
 
+  cmake_installed_test_cxx20: # assuming that it was installed, this tries to build using C++20
+    steps:
+      - run: echo -e '#include <simdjson.h>\nint main(int argc,char**argv) {simdjson::dom::parser parser;simdjson::dom::element tweets = parser.load(argv[1]); }' > tmp.cpp && c++ -Ibuild/destination/include -Lbuild/destination/lib -std=c++20 -Wl,-rpath,build/destination/lib -o linkandrun tmp.cpp -lsimdjson && ./linkandrun jsonexamples/twitter.json
+
 jobs:
 
   # static
-  gcc7:
-    description: Build and run tests on GCC 7 and AVX 2 with a cmake static build
-    executor: gcc7
-    environment: { CMAKE_FLAGS: -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_BUILD_STATIC=ON }
-    steps: [ install_cmake, cmake_test, cmake_install_test ]
   justlib-gcc10:
     description: Build just the library, install it and do a basic test
     executor: gcc10
-    environment: { CMAKE_FLAGS: -SIMDJSON_JUST_LIBRARY=ON }
-    steps: [ cmake_build, cmake_install_test ]
+    environment: { CMAKE_FLAGS: -DSIMDJSON_JUST_LIBRARY=ON }
+    steps: [ cmake_build, cmake_install_test, cmake_installed_test_cxx20 ]
   gcc10:
     description: Build and run tests on GCC 10 and AVX 2 with a cmake static build, this test performance regression
     executor: gcc10
     environment: { CMAKE_FLAGS: -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_BUILD_STATIC=ON }
-    steps: [ cmake_test_all, cmake_install_test ]
+    steps: [ cmake_test_all, cmake_install_test, cmake_installed_test_cxx20 ]
   clang6:
     description: Build and run tests on clang 6 and AVX 2 with a cmake static build
     executor: clang6
@@ -137,13 +127,13 @@ jobs:
     description: Build and run tests on clang 10 and AVX 2 with a cmake static build
     executor: clang10
     environment: { CMAKE_FLAGS: -DSIMDJSON_GOOGLE_BENCHMARKS=ON  -DSIMDJSON_BUILD_STATIC=ON }
-    steps: [ cmake_test, cmake_install_test ]
+    steps: [ cmake_test, cmake_install_test, cmake_installed_test_cxx20 ]
   # libcpp
   libcpp-clang10:
     description: Build and run tests on clang 10 and AVX 2 with a cmake static build and libc++
     executor: clang10
     environment: { CMAKE_FLAGS: -DSIMDJSON_USE_LIBCPP=ON  -DSIMDJSON_BUILD_STATIC=ON }
-    steps: [ cmake_test, cmake_install_test ]
+    steps: [ cmake_test, cmake_install_test, cmake_installed_test_cxx20 ]
   # sanitize
   sanitize-gcc10:
     description: Build and run tests on GCC 10 and AVX 2 with a cmake sanitize build
@@ -207,18 +197,13 @@ jobs:
     executor: gcc10
     environment: { CXXFLAGS: -march=nehalem }
     steps: [ cmake_test ]
-  no-computed-goto-gcc10:
-    description: Build, run tests and check performance on GCC 7 with -DSIMDJSON_NO_COMPUTED_GOTO=true
-    executor: gcc10
-    environment: { CXXFLAGS: -DSIMDJSON_NO_COMPUTED_GOTO=true }
-    steps: [ cmake_test ]
 
 workflows:
   version: 2.1
   build_and_test:
     jobs:
       # full multi-implementation tests
-      - gcc7
+      #- gcc7 tested on GitHub actions
       - gcc10 # do not delete this as it tests our performance
       - clang6
       #- clang10 # this gets tested a lot below
@@ -241,7 +226,6 @@ workflows:
       # quicker make single-implementation tests
       - arch-haswell-gcc10
       - arch-nehalem-gcc10
-      - no-computed-goto-gcc10
 
       # testing "just the library"
       - justlib-gcc10

.cirrus.yml

@@ -14,8 +14,8 @@ task:
   build_script:
     - mkdir build
     - cd build
-    - cmake ..
-    - make -j4
+    - cmake -DSIMDJSON_BASH=OFF  -DSIMDJSON_GIT=OFF ..
+    - make
   test_script:
     - cd build
-    - ctest -j4 --output-on-failure -E checkperf
+    - ctest --output-on-failure -E checkperf

.gitattributes

@@ -80,15 +80,15 @@
 .gitignore     export-ignore
 
 # Sources
-*.c     text diff=c
-*.cc    text diff=cpp
-*.cxx   text diff=cpp
-*.cpp   text diff=cpp
-*.c++   text diff=cpp
-*.hpp   text diff=cpp
-*.h     text diff=c
-*.h++   text diff=cpp
-*.hh    text diff=cpp
+*.c     text eol=lf diff=c
+*.cc    text eol=lf diff=cpp
+*.cxx   text eol=lf diff=cpp
+*.cpp   text eol=lf diff=cpp
+*.c++   text eol=lf diff=cpp
+*.hpp   text eol=lf diff=cpp
+*.h     text eol=lf diff=c
+*.h++   text eol=lf diff=cpp
+*.hh    text eol=lf diff=cpp
 
 # Compiled Object files
 *.slo   binary

.github/ISSUE_TEMPLATE/bug_report.md

@@ -2,7 +2,7 @@
 name: Bug report
 about: Create a report to help us improve
 title: ''
-labels: bug
+labels: bug (unverified)
 assignees: ''
 
 ---
@@ -17,14 +17,21 @@ Before submitting an issue, please ensure that you have read the documentation:
 **Describe the bug**
 A clear and concise description of what the bug is.
 
+Note that a compiler warning is not a bug. 
+
 **To Reproduce**
 Steps to reproduce the behaviour: provide a code sample if possible.
 
+If we cannot reproduce the issue, then we cannot address it.
+
+Note that a stack trace from your own program is not enough.
+
 **Configuration (please complete the following information if relevant):**
  - OS: [e.g. Ubuntu 16.04.6 LTS]
  - Compiler [e.g. Apple clang version 11.0.3 (clang-1103.0.32.59) x86_64-apple-darwin19.4.0]
  - Version [e.g. 22]
 
+We support up-to-date 64-bit ARM and x64 FreeBSD, macOS, Windows and Linux systems. Please ensure that your configuration is supported before labelling the issue as a bug. In particular, we do not support legacy 32-bit systems. 
 
 **Indicate whether you are willing or able to provide a bug fix as a pull request**
 

.github/ISSUE_TEMPLATE/feature_request.md

@@ -2,7 +2,7 @@
 name: Feature request
 about: Suggest an idea for this project
 title: ''
-labels: feature request
+labels: ''
 assignees: ''
 
 ---

.github/workflows/mingw-ci.yml

@@ -0,0 +1,53 @@
+name: MinGW32-CI
+
+on: [push, pull_request]
+
+# Important: scoop will either install 32-bit GCC or 64-bit GCC, not both.
+
+# It is important to build static libraries because cmake is not smart enough under Windows/mingw to take care of the path. So
+# with a dynamic library, you could get failures due to the fact that the EXE can't find its DLL.
+
+jobs:
+  ci:
+    name: windows-gcc
+    runs-on: windows-2016
+
+    env:
+      CMAKE_GENERATOR: Ninja # This is critical, try ' cmake  -GNinja-DSIMDJSON_BUILD_STATIC=ON .. ' if using the command line
+      CC: gcc
+      CXX: g++
+
+    steps: # To reproduce what is below, start a powershell with administrative rights, using scoop *is* a good idea
+      - uses: actions/checkout@v2
+
+      - uses: actions/cache@v2 # we cache the scoop setup with 32-bit GCC
+        id: cache
+        with:
+          path: |
+            C:\ProgramData\scoop
+          key: scoop32 # static key: should be good forever
+      - name: Setup Windows # This should almost never run if the cache works.
+        if: steps.cache.outputs.cache-hit != 'true'
+        shell: powershell
+        run: |
+          Invoke-Expression (New-Object System.Net.WebClient).DownloadString('https://get.scoop.sh')
+          scoop install sudo --global
+          sudo scoop install git --global 
+          sudo scoop install ninja --global
+          sudo scoop install cmake --global
+          sudo scoop install gcc --arch 32bit --global
+          $env:path
+          Write-Host 'Everything has been installed, you are good!'
+      - name: Build and Test 32-bit x86
+        shell: powershell
+        run: |
+          $ENV:PATH="C:\ProgramData\scoop\shims;C:\ProgramData\scoop\apps\gcc\current\bin;C:\ProgramData\scoop\apps\ninja\current;$ENV:PATH"
+          g++ --version
+          cmake --version
+          ninja --version
+          git --version
+          mkdir build32
+          cd build32
+          cmake -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_ENABLE_THREADS=OFF ..
+          cmake --build . --target parse_many_test jsoncheck basictests numberparsingcheck stringparsingcheck errortests integer_tests pointercheck --verbose
+          ctest -R "(parse_many_test|jsoncheck|basictests|stringparsingcheck|numberparsingcheck|errortests|integer_tests|pointercheck)"  --output-on-failure

.github/workflows/mingw64-ci.yml

@@ -0,0 +1,59 @@
+name: MinGW64-CI
+
+on: [push, pull_request]
+
+# Important: scoop will either install 32-bit GCC or 64-bit GCC, not both.
+
+# It is important to build static libraries because cmake is not smart enough under Windows/mingw to take care of the path. So
+# with a dynamic library, you could get failures due to the fact that the EXE can't find its DLL.
+
+jobs:
+  ci:
+    name: windows-gcc
+    runs-on: windows-2016
+
+    env:
+      CMAKE_GENERATOR: Ninja # This is critical, try ' cmake  -GNinja-DSIMDJSON_BUILD_STATIC=ON .. ' if using the command line
+      CC: gcc
+      CXX: g++
+
+    steps: # To reproduce what is below, start a powershell with administrative rights, using scoop *is* a good idea
+      - uses: actions/checkout@v2
+
+      - uses: actions/cache@v2 # we cache the scoop setup with 64-bit GCC
+        id: cache
+        with:
+          path: |
+            C:\ProgramData\scoop
+          key: scoop64 # static key: should be good forever
+      - name: Setup Windows # This should almost never run if the cache works.
+        if: steps.cache.outputs.cache-hit != 'true'
+        shell: powershell
+        run: |
+          Invoke-Expression (New-Object System.Net.WebClient).DownloadString('https://get.scoop.sh')
+          scoop install sudo --global
+          sudo scoop install git --global
+          sudo scoop install ninja --global
+          sudo scoop install cmake --global
+          sudo scoop install gcc --arch 64bit --global
+          $env:path
+          Write-Host 'Everything has been installed, you are good!'
+      - name: Build and Test 64-bit x64
+        shell: powershell
+        run: |
+          $ENV:PATH="C:\ProgramData\scoop\shims;C:\ProgramData\scoop\apps\gcc\current\bin;C:\ProgramData\scoop\apps\ninja\current;$ENV:PATH"
+          g++ --version
+          cmake --version
+          ninja --version
+          git --version
+          mkdir build64
+          cd build64
+          cmake -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_ENABLE_THREADS=OFF ..
+          cmake --build . --target parse_many_test jsoncheck basictests numberparsingcheck stringparsingcheck errortests integer_tests pointercheck --verbose
+          ctest -R "(parse_many_test|jsoncheck|basictests|stringparsingcheck|numberparsingcheck|errortests|integer_tests|pointercheck)"  --output-on-failure
+          cd ..
+          mkdir build64debug
+          cd build64debug
+          cmake -DCMAKE_BUILD_TYPE=Debug -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_ENABLE_THREADS=OFF ..
+          cmake --build . --target parse_many_test jsoncheck basictests numberparsingcheck stringparsingcheck errortests integer_tests pointercheck --verbose
+          ctest -R "(parse_many_test|jsoncheck|basictests|stringparsingcheck|numberparsingcheck|errortests|integer_tests|pointercheck)"  --output-on-failure

.github/workflows/msys2.yml

@@ -0,0 +1,44 @@
+name: MSYS2-CI
+
+on: [push, pull_request]
+
+jobs:
+  windows-mingw:
+    name: ${{ matrix.msystem }}
+    runs-on: windows-latest
+    defaults:
+      run:
+        shell: msys2 {0}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - msystem: "MINGW64"
+            install: mingw-w64-x86_64-cmake mingw-w64-x86_64-ninja mingw-w64-x86_64-gcc
+            type: Release
+          - msystem: "MINGW32"
+            install: mingw-w64-i686-cmake mingw-w64-i686-ninja mingw-w64-i686-gcc
+            type: Release
+          - msystem: "MINGW64"
+            install: mingw-w64-x86_64-cmake mingw-w64-x86_64-ninja mingw-w64-x86_64-gcc
+            type: Debug
+          - msystem: "MINGW32"
+            install: mingw-w64-i686-cmake mingw-w64-i686-ninja mingw-w64-i686-gcc
+            type: Debug
+    env:
+      CMAKE_GENERATOR: Ninja
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: msys2/setup-msys2@v2
+        with:
+          update: true
+          msystem: ${{ matrix.msystem }}
+          install: ${{ matrix.install }}
+      - name: Build and Test
+        run: |
+          mkdir build
+          cd build
+          cmake -DCMAKE_BUILD_TYPE=${{ matrix.type }} -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_DO_NOT_USE_THREADS_NO_MATTER_WHAT=ON ..
+          cmake --build . --verbose
+          ctest -j4 --output-on-failure -E checkperf

.github/workflows/ubuntu18.yml

@@ -0,0 +1,22 @@
+name: Ubuntu 18.04 CI (GCC 7)
+
+on: [push, pull_request]
+
+jobs:
+  ubuntu-build:
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup cmake
+        uses: jwlawson/actions-setup-cmake@v1.0
+        with:
+          cmake-version: '3.9.x'
+      - name: Use cmake
+        run: |
+          mkdir build &&
+          cd build &&
+          cmake  -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_BUILD_STATIC=ON -DCMAKE_INSTALL_PREFIX:PATH=destination ..  &&
+          cmake --build .   &&
+          ctest -j --output-on-failure  &&
+          make install  &&
+          echo -e '#include <simdjson.h>\nint main(int argc,char**argv) {simdjson::dom::parser parser;simdjson::dom::element tweets = parser.load(argv[1]); }' > tmp.cpp && c++ -Idestination/include -Ldestination/lib -std=c++17 -Wl,-rpath,destination/lib -o linkandrun tmp.cpp -lsimdjson && ./linkandrun jsonexamples/twitter.json

.github/workflows/ubuntu20.yml

@@ -0,0 +1,22 @@
+name: Ubuntu 20.04 CI (GCC 9)
+
+on: [push, pull_request]
+
+jobs:
+  ubuntu-build:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup cmake
+        uses: jwlawson/actions-setup-cmake@v1.0
+        with:
+          cmake-version: '3.9.x'
+      - name: Use cmake
+        run: |
+          mkdir build &&
+          cd build &&
+          cmake  -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_BUILD_STATIC=ON -DCMAKE_INSTALL_PREFIX:PATH=destination ..  &&
+          cmake --build .   &&
+          ctest -j --output-on-failure  &&
+          make install  &&
+          echo -e '#include <simdjson.h>\nint main(int argc,char**argv) {simdjson::dom::parser parser;simdjson::dom::element tweets = parser.load(argv[1]); }' > tmp.cpp && c++ -Idestination/include -Ldestination/lib -std=c++17 -Wl,-rpath,destination/lib -o linkandrun tmp.cpp -lsimdjson && ./linkandrun jsonexamples/twitter.json

.github/workflows/vs16-ci.yml

@@ -0,0 +1,26 @@
+name: VS16-CI
+
+on: [push, pull_request]
+
+jobs:
+  ci:
+    name: windows-vs16
+    runs-on: windows-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: 'Run CMake with VS16'
+      uses: lukka/run-cmake@v2
+      with:
+        cmakeListsOrSettingsJson: CMakeListsTxtAdvanced
+        cmakeListsTxtPath: '${{ github.workspace }}/CMakeLists.txt'
+        buildDirectory: "${{ github.workspace }}/../../_temp/windows"
+        cmakeBuildType: Release   
+        buildWithCMake: true
+        cmakeGenerator: VS16Win64 
+        cmakeAppendedArgs: -DSIMDJSON_COMPETITION=OFF
+        buildWithCMakeArgs: --config Release  
+        
+    - name: 'Run CTest'
+      run: ctest -C Release  -E checkperf  --output-on-failure 
+      working-directory: "${{ github.workspace }}/../../_temp/windows"
+  

.github/workflows/vs16-clang-ci.yml

@@ -0,0 +1,25 @@
+name: VS16-CLANG-CI
+
+on: [push, pull_request]
+
+jobs:
+  ci:
+    name: windows-vs16
+    runs-on: windows-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: 'Run CMake with VS16'
+      uses: lukka/run-cmake@v2
+      with:
+        cmakeListsOrSettingsJson: CMakeListsTxtAdvanced
+        cmakeListsTxtPath: '${{ github.workspace }}/CMakeLists.txt'
+        buildDirectory: "${{ github.workspace }}/../../_temp/windows"
+        cmakeBuildType: Release   
+        buildWithCMake: true
+        cmakeGenerator: VS16Win64 
+        cmakeAppendedArgs:  -T ClangCL -DSIMDJSON_COMPETITION=OFF  -DSIMDJSON_BUILD_STATIC=ON
+        buildWithCMakeArgs: --config Release  
+        
+    - name: 'Run CTest'
+      run: ctest -C Release -E checkperf  --output-on-failure 
+      working-directory: "${{ github.workspace }}/../../_temp/windows"

.github/workflows/vs16-ninja-ci.yml

@@ -0,0 +1,26 @@
+name: VS16-Ninja-CI
+
+on: [push, pull_request]
+
+jobs:
+  ci:
+    name: windows-vs16
+    runs-on: windows-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: 'Run CMake with VS16'
+      uses: lukka/run-cmake@v2
+      with:
+        cmakeListsOrSettingsJson: CMakeListsTxtAdvanced
+        cmakeListsTxtPath: '${{ github.workspace }}/CMakeLists.txt'
+        buildDirectory: "${{ github.workspace }}/../../_temp/windows"
+        cmakeBuildType: Release   
+        buildWithCMake: true
+        cmakeGenerator: VS16Win64 
+        cmakeAppendedArgs:  -G Ninja -DSIMDJSON_COMPETITION=OFF   -DSIMDJSON_BUILD_STATIC=ON
+        buildWithCMakeArgs: --config Release  
+        
+    - name: 'Run CTest'
+      run: ctest -C Release  -E checkperf  --output-on-failure 
+      working-directory: "${{ github.workspace }}/../../_temp/windows"
+  

CMakeLists.txt

@@ -6,10 +6,11 @@ project(simdjson
 )
 
 set(PROJECT_VERSION_MAJOR 0)
-set(PROJECT_VERSION_MINOR 4)
-set(PROJECT_VERSION_PATCH 1)
-set(SIMDJSON_LIB_VERSION "0.4.1" CACHE STRING "simdjson library version")
-set(SIMDJSON_LIB_SOVERSION "2" CACHE STRING "simdjson library soversion")
+set(PROJECT_VERSION_MINOR 5)
+set(PROJECT_VERSION_PATCH 0)
+set(SIMDJSON_SEMANTIC_VERSION "0.5.0" CACHE STRING "simdjson semantic version")
+set(SIMDJSON_LIB_VERSION "3.0.0" CACHE STRING "simdjson library version")
+set(SIMDJSON_LIB_SOVERSION "3" CACHE STRING "simdjson library soversion")
 set(SIMDJSON_GITHUB_REPOSITORY https://github.com/simdjson/simdjson)
 
 include(GNUInstallDirs)
@@ -30,7 +31,7 @@ if(NOT(SIMDJSON_JUST_LIBRARY))
   add_subdirectory(jsonchecker)
   add_subdirectory(jsonexamples)
   add_library(test-data INTERFACE)
-  target_link_libraries(test-data INTERFACE jsonchecker-data jsonexamples-data)
+  target_link_libraries(test-data INTERFACE jsonchecker-data jsonchecker-minefield-data jsonexamples-data)
 endif()
 
 #
@@ -45,6 +46,7 @@ if(NOT(SIMDJSON_JUST_LIBRARY))
   add_subdirectory(tools)  ## This needs to be before tests because of cxxopts
   add_subdirectory(singleheader)
 endif()
+install(FILES singleheader/simdjson.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
 
 #
 # Compile tools / tests / benchmarks

CONTRIBUTING.md

@@ -41,6 +41,7 @@ We have few hard rules, but we have some:
 - Printing to standard output or standard error (`stderr`, `stdout`, `std::cerr`, `std::cout`) in the core library is forbidden. This follows from the [Writing R Extensions](https://cran.r-project.org/doc/manuals/R-exts.html) manual which states that "Compiled code should not write to stdout or stderr".
 - Calls to `abort()` are forbidden in the core library. This follows from the [Writing R Extensions](https://cran.r-project.org/doc/manuals/R-exts.html) manual which states that "Under no circumstances should your compiled code ever call abort or exit".
 - All source code files (.h, .cpp) must be ASCII.
+- All C macros introduced in public headers need to be prefixed with either `SIMDJSON_` or `simdjson_`.
 
 Tools, tests and benchmarks are not held to these same strict rules.
 
@@ -71,6 +72,12 @@ Pull requests are always invited. However, we ask that you follow these guidelin
 
 If the benefits of your proposed code remain unclear, we may choose to discard your code: that is not an insult, we frequently discard our own code. We may also consider various alternatives and choose another path. Again, that is not an insult or a sign that you have wasted your time.
 
+Style
+-----
+
+Our formatting style is inspired by the LLVM style.
+The simdjson library is written using the snake case: when a variable or a function is a phrase,  each space is replaced by an underscore character, and the first letter of each word written in lowercase.  Compile-time constants are written entirely in uppercase with the same underscore convention.
+
 Code of Conduct
 ---------------
 

Doxyfile

@@ -38,7 +38,7 @@ PROJECT_NAME           = simdjson
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = "0.4.1"
+PROJECT_NUMBER         = "0.5.0"
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a

HACKING.md

@@ -1,11 +1,38 @@
 Hacking simdjson
 ================
 
-Here is wisdom about how to build, test and run simdjson from within the repository. *Users* of
-simdjson should use the released simdjson.h and simdjson.cpp files.
+Here is wisdom about how to build, test and run simdjson from within the repository. This is mostly useful for people who plan to contribute simdjson, or maybe study the design.
 
 If you plan to contribute to simdjson, please read our [CONTRIBUTING](https://github.com/simdjson/simdjson/blob/master/CONTRIBUTING.md) guide.
 
+
+Design notes
+------------------------------
+
+The parser works in two stages:
+
+- Stage 1. (Find marks) Identifies quickly structure elements, strings, and so forth. We validate UTF-8 encoding at that stage.
+- Stage 2. (Structure building) Involves constructing a "tree" of sort (materialized as a tape) to navigate through the data. Strings and numbers are parsed at this stage.
+
+
+The role of stage 1 is to identify pseudo-structural characters as quickly as possible. A character is pseudo-structural if and only if:
+
+1. Not enclosed in quotes, AND
+2. Is a non-whitespace character, AND
+3. Its preceding character is either:
+   (a) a structural character, OR
+   (b) whitespace OR
+   (c) the final quote in a string.
+
+This helps as we redefine some new characters as pseudo-structural such as the characters 1, G, n in the following:
+
+> { "foo" : 1.5, "bar" : 1.5 GEOFF_IS_A_DUMMY bla bla , "baz", null }
+
+Stage 1 also does unicode validation.
+
+Stage 2 handles all of the rest: number parsings, recognizing atoms like true, false, null, and so forth.
+
+
 Directory Structure and Source
 ------------------------------
 
@@ -13,13 +40,13 @@ simdjson's source structure, from the top level, looks like this:
 
 * **CMakeLists.txt:** The main build system.
 * **include:** User-facing declarations and inline definitions (most user-facing functions are inlined).
-  * simdjson.h: A "master include" that includes files from include/simdjson/. This is equivalent to
+  * simdjson.h: A "main include" that includes files from include/simdjson/. This is equivalent to
     the distributed simdjson.h.
   * simdjson/*.h: Declarations for public simdjson classes and functions.
-  * simdjson/inline/*.h: Definitions for public simdjson classes and functions.
+  * simdjson/*-inl.h: Definitions for public simdjson classes and functions.
 * **src:** The source files for non-inlined functionality (e.g. the architecture-specific parser
   implementations).
-  * simdjson.cpp: A "master source" that includes all implementation files from src/. This is
+  * simdjson.cpp: A "main source" that includes all implementation files from src/. This is
     equivalent to the distributed simdjson.cpp.
   * arm64/|fallback/|haswell/|westmere/: Architecture-specific implementations. All functions are
     Each architecture defines its own namespace, e.g. simdjson::haswell.
@@ -38,7 +65,9 @@ Other important files and directories:
 * **.drone.yml:** Definitions for Drone CI.
 * **.appveyor.yml:** Definitions for Appveyor CI (Windows).
 * **.circleci:** Definitions for Circle CI.
-* **amalgamate.sh:** Generates singleheader/simdjson.h and singleheader/simdjson.cpp for release.
+* **.github/workflows:** Definitions for GitHub Actions (CI).
+* **singleheader:** Contains generated `simdjson.h` and `simdjson.cpp` that we release. The files `singleheader/simdjson.h` and `singleheader/simdjson.cpp` should never be edited by hand.
+* **singleheader/amalgamate.sh:** Generates `singleheader/simdjson.h` and `singleheader/simdjson.cpp` for release (bash script). 
 * **benchmark:** This is where we do benchmarking. Benchmarking is core to every change we make; the
   cardinal rule is don't regress performance without knowing exactly why, and what you're trading
   for it. Many of our benchmarks are microbenchmarks. We are effectively doing controlled scientific experiments for the purpose of understanding what affects our performance. So we simplify as much as possible. We try to avoid irrelevant factors such as page faults, interrupts, unnnecessary system calls. We recommend checking the performance as follows:
@@ -64,18 +93,24 @@ Other important files and directories:
 * **jsonchecker:** A set of JSON files used to check different functionality of the parser.
   * **pass*.json:** Files that should pass validation.
   * **fail*.json:** Files that should fail validation.
+  * **jsonchecker/minefield/y_*.json:** Files that should pass validation.
+  * **jsonchecker/minefield/n_*.json:** Files that should fail validation.
 * **jsonexamples:** A wide spread of useful, real-world JSON files with different characteristics
   and sizes.
-* **singleheader:** Contains generated simdjson.h and simdjson.cpp that we release.
 * **test:** The tests are here. basictests.cpp and errortests.cpp are the primary ones.
-* **tools:** Source for executables that can be distributed with simdjson
+* **tools:** Source for executables that can be distributed with simdjson. Some examples:
+  * `json2json mydoc.json` parses the document, constructs a model and then dumps back the result to standard output.
+  * `json2json -d mydoc.json` parses the document, constructs a model and then dumps model (as a tape) to standard output. The tape format is described in the accompanying file `tape.md`.
+  * `minify mydoc.json` minifies the JSON document, outputting the result to standard output. Minifying means to remove the unneeded white space characters.
+  *`jsonpointer mydoc.json <jsonpath> <jsonpath> ... <jsonpath>` parses the document, constructs a model and then processes a series of [JSON Pointer paths](https://tools.ietf.org/html/rfc6901). The result is itself a JSON document.
+
+
 > **Don't modify the files in singleheader/ directly; these are automatically generated.**
->
-> While we distribute those files on release, we *maintain* the files under include/ and src/.
+
 
 While simdjson distributes just two files from the singleheader/ directory, we *maintain* the code in
-multiple files under include/ and src/. include/simdjson.h and src/simdjson.cpp are the "spine" for
-these, and you can include
+multiple files under include/ and src/. The files include/simdjson.h and src/simdjson.cpp are the "spine" for
+these, and you can include them as if they were the corresponding singleheader/ files.
 
 
 
@@ -124,11 +159,11 @@ We also handle the special case where a user is compiling using LLVM clang under
 
 
 
-Regenerating Single Headers From Master
+Regenerating Single-Header Files
 ---------------------------------------
 
-simdjson.h and simdjson.cpp are not always up to date in master. To ensure you have the latest copy,
-you can regenerate them by running this at the top level:
+The simdjson.h and simdjson.cpp files in the singleheader directory are not always up-to-date with the rest of the code; they are only ever
+systematically regenerated on releases. To ensure you have the latest code, you can regenerate them by running this at the top level:
 
 ```bash
 mkdir build
@@ -137,13 +172,15 @@ cmake ..
 cmake --build . --target amalgamate
 ```
 
-The amalgamator is at `amalgamate.sh` at the top level. It generates singleheader/simdjson.h by
+You need to have a working bash on your system.
+
+The amalgamator script is `amalgamate.sh` generates singleheader/simdjson.h by
 reading through include/simdjson.h, copy/pasting each header file into the amalgamated file at the
 point it gets included (but only once per header). singleheader/simdjson.cpp is generated from
 src/simdjson.cpp the same way, except files under generic/ may be included and copy/pasted multiple
 times.
 
-### Usage (CMake on 64-bit platforms like Linux, freeBSD or macOS)
+### Usage (CMake on 64-bit platforms like Linux, FreeBSD or macOS)
 
 Requirements: In addition to git, we require a recent version of CMake as well as bash.
 
@@ -156,7 +193,7 @@ brew install cmake
 apt-get update -qq
 apt-get install -y cmake
 ```
-3. On freeBSD, you might be able to install bash and CMake as follows:
+3. On FreeBSD, you might be able to install bash and CMake as follows:
 ```
 pkg update -f
 pkg install bash
@@ -206,15 +243,15 @@ Note that the name of directory (`build`) is arbitrary, you can name it as you w
 
 
 
-### Usage (CMake on 64-bit Windows using Visual Studio)
+### Usage (CMake on 64-bit Windows using Visual Studio 2019)
 
-We assume you have a common 64-bit Windows PC with at least Visual Studio 2017 and an x64 processor with AVX2 support (2013 Intel Haswell or later) or SSE 4.2 + CLMUL (2010 Westmere or later).
+We assume you have a common 64-bit Windows PC with at least Visual Studio 2019.
 
 - Grab the simdjson code from GitHub, e.g., by cloning it using [GitHub Desktop](https://desktop.github.com/).
 - Install [CMake](https://cmake.org/download/). When you install it, make sure to ask that `cmake` be made available from the command line. Please choose a recent version of cmake.
 - Create a subdirectory within simdjson, such as `build`.
 - Using a shell, go to this newly created directory. You can start a shell directly from GitHub Desktop (Repository > Open in Command Prompt).
-- Type `cmake -DCMAKE_GENERATOR_PLATFORM=x64 ..` in the shell while in the `build` repository. (Alternatively, if you want to build a DLL, you may use the command line `cmake -DCMAKE_GENERATOR_PLATFORM=x64 -DSIMDJSON_BUILD_STATIC=OFF ..`.)
+- Type `cmake  ..` in the shell while in the `build` repository. 
 - This last command (`cmake ...`) created a Visual Studio solution file in the newly created directory (e.g., `simdjson.sln`). Open this file in Visual Studio. You should now be able to build the project and run the tests. For example, in the `Solution Explorer` window (available from the `View` menu), right-click `ALL_BUILD` and select `Build`. To test the code, still in the `Solution Explorer` window, select `RUN_TESTS` and select `Build`.
 
 
@@ -235,380 +272,11 @@ Furthermore, if you have installed LLVM clang on Windows, for example as a compo
 - `cmake --build . -config Release`
 
 
-### Usage (Using `vcpkg` on 64-bit Windows, Linux and macOS)
-
-[vcpkg](https://github.com/Microsoft/vcpkg) users on Windows, Linux and macOS can download and install `simdjson` with one single command from their favorite shell.
-
-On 64-bit Linux and macOS:
-
-```
-$ ./vcpkg install simdjson
-```
-
-will build and install `simdjson` as a static library.
-
-On Windows (64-bit):
-
-```
-.\vcpkg.exe install simdjson:x64-windows
-```
-
-will build and install `simdjson` as a shared library.
-
-```
-.\vcpkg.exe install simdjson:x64-windows-static
-```
-
-will build and install `simdjson` as a static library.
-
-These commands will also print out instructions on how to use the library from MSBuild or CMake-based projects.
-
-If you find the version of `simdjson` shipped with `vcpkg` is out-of-date, feel free to report it to
-`vcpkg` community either by submitting an issue or by creating a PR.
-
-### Usage (Docker)
-
-One can run tests and benchmarks using docker. It especially makes sense under Linux. Privileged
-access may be needed to get performance counters.
-
-```
-git clone https://github.com/simdjson/simdjson.git
-cd simdjson
-docker build -t simdjson .
-docker run --privileged -t simdjson
-```
-
-## Architecture and Design Notes
-
-### Requirements
-
-- 64-bit platforms like Linux or macOS, as well as Windows through Visual Studio 2017 or later.
-- Any 64-bit processor:
-  - AVX2 (i.e., Intel processors starting with the Haswell microarchitecture released 2013 and AMD
-    processors starting with the Zen microarchitecture released 2017),
-  - SSE 4.2 and CLMUL (i.e., Intel processors going back to Westmere released in 2010 or AMD
-    processors starting with the Jaguar used in the PS4 and XBox One),
-  - 64-bit ARM processor (ARMv8-A NEON): this covers a wide range of mobile processors, including
-    all Apple processors currently available for sale, going as far back as the iPhone 5s (2013).
-  - Any 64-bit processor (simdjson has a fallback generic 64-bit implementation that is still super
-    fast).
-- A recent C++ compiler (e.g., GNU GCC or LLVM CLANG or Visual Studio 2017), we assume C++17. GNU
-  GCC 7 or better or LLVM's clang 6 or better.
-- Some benchmark scripts assume bash and other common utilities, but they are optional.
-
-### Scope
-
-We provide a fast parser, that fully validates an input according to various specifications.
-The parser builds a useful immutable (read-only) DOM (document-object model) which can be later accessed.
-
-To simplify the engineering, we make some assumptions.
-
-- We support UTF-8 (and thus ASCII), nothing else (no Latin, no UTF-16). We do not believe this is a
-  genuine limitation, because we do not think there is any serious application that needs to process
-  JSON data without an ASCII or UTF-8 encoding. If the UTF-8 contains a leading BOM, it should be
-  omitted: the user is responsible for detecting and skipping the BOM; UTF-8 BOMs are discouraged.
-- All strings in the JSON document may have up to 4294967295 bytes in UTF-8 (4GB). To enforce this
-  constraint, we refuse to parse a document that contains more than 4294967295 bytes (4GB). This
-  should accommodate most JSON documents.
-- As allowed by the specification, we allow repeated keys within an object (other parsers like
-  sajson do the same).
-- [The simdjson library is fast for JSON documents spanning a few bytes up to many megabytes](https://github.com/lemire/simdjson/issues/312).
-
-_We do not aim to provide a general-purpose JSON library._ A library like RapidJSON offers much more
-than just parsing, it helps you generate JSON and offers various other convenient functions. We
-merely parse the document. This may change in the future.
-
-### Features
-
-- The input string is unmodified. (Parsers like sajson and RapidJSON use the input string as a buffer.)
-- We parse integers and floating-point numbers as separate types which allows us to support large signed 64-bit integers in [-9223372036854775808,9223372036854775808), like a Java `long` or a C/C++ `long long` and large unsigned integers up to the value 18446744073709551615. Among the parsers that differentiate between integers and floating-point numbers, not all support 64-bit integers. (For example, sajson rejects JSON files with integers larger than or equal to 2147483648. RapidJSON will parse a file containing an overly long integer like 18446744073709551616 as a floating-point number.) When we cannot represent exactly an integer as a signed or unsigned 64-bit value, we reject the JSON document.
-- We support the full range of 64-bit floating-point numbers (binary64). The values range from ` std::numeric_limits<double>::lowest()`  to `std::numeric_limits<double>::max()`, so from -1.7976e308 all the way to 1.7975e308. Extreme values (less or equal to -1e308, greater or equal to 1e308) are rejected: we refuse to parse the input document.
-- We test for accurate float parsing with a perfect accuracy (ULP 0). Many parsers offer only approximate floating parsing. For example, RapidJSON also offers the option of accurate float parsing (`kParseFullPrecisionFlag`) but it comes at a significant performance penalty compared to the default settings. By default, RapidJSON tolerates an error of 3 ULP.
-- We do full UTF-8 validation as part of the parsing. (Parsers like fastjson, gason and dropbox json11 do not do UTF-8 validation. The sajson parser does incomplete UTF-8 validation, accepting code point
-sequences like 0xb1 0x87.)
-- We fully validate the numbers. (Parsers like gason and ultranjson will accept `[0e+]` as valid JSON.)
-- We validate string content for unescaped characters. (Parsers like fastjson and ultrajson accept unescaped line breaks and tabs in strings.)
-- We fully validate the white-space characters outside of the strings. Parsers like RapidJSON will accept JSON documents with null characters outside of strings.
-
-### Architecture
-
-The parser works in two stages:
-
-- Stage 1. (Find marks) Identifies quickly structure elements, strings, and so forth. We validate UTF-8 encoding at that stage.
-- Stage 2. (Structure building) Involves constructing a "tree" of sort (materialized as a tape) to navigate through the data. Strings and numbers are parsed at this stage.
-
-### Remarks on JSON parsing
-
-- The JSON spec defines what a JSON parser is:
-  > A JSON parser transforms a JSON text into another representation. A JSON parser MUST accept all texts that conform to the JSON grammar. A JSON parser MAY accept non-JSON forms or extensions. An implementation may set limits on the size of texts that it accepts. An implementation may set limits on the maximum depth of nesting. An implementation may set limits on the range and precision of numbers. An implementation may set limits on the length and character contents of strings.
-
-* JSON is not JavaScript:
-
-  > All JSON is Javascript but NOT all Javascript is JSON. So {property:1} is invalid because property does not have double quotes around it. {'property':1} is also invalid, because it's single quoted while the only thing that can placate the JSON specification is double quoting. JSON is even fussy enough that {"property":.1} is invalid too, because you should have of course written {"property":0.1}. Also, don't even think about having comments or semicolons, you guessed it: they're invalid. (credit:https://github.com/elzr/vim-json)
-
-* The structural characters are:
-
-      begin-array     =  [ left square bracket
-      begin-object    =  { left curly bracket
-      end-array       =  ] right square bracket
-      end-object      =  } right curly bracket
-      name-separator  = : colon
-      value-separator = , comma
-
-### Pseudo-structural elements
-
-A character is pseudo-structural if and only if:
-
-1. Not enclosed in quotes, AND
-2. Is a non-whitespace character, AND
-3. Its preceding character is either:
-   (a) a structural character, OR
-   (b) whitespace.
-
-This helps as we redefine some new characters as pseudo-structural such as the characters 1, G, n in the following:
-
-> { "foo" : 1.5, "bar" : 1.5 GEOFF_IS_A_DUMMY bla bla , "baz", null }
-
-
-
-### UTF-8 validation (lookup2)
-
-The simdjson library relies on the lookup2 algorithm for UTF-8 validation on x64 platforms.
-
-This algorithm validate the length of multibyte characters (that each multibyte character has the right number of continuation characters, and that all continuation characters are part of a multibyte  character).
-
-####  Algorithm
-
-This algorithm compares *expected* continuation characters with *actual* continuation bytes, and emits an error anytime there is a mismatch.
-
-For example, in the string "𝄞₿֏ab", which has a 4-, 3-, 2- and 1-byte
-characters, the file will look like this:
-
-| Character             | 𝄞  |    |    |    | ₿  |    |    | ֏  |    | a  | b  |
-|-----------------------|----|----|----|----|----|----|----|----|----|----|----|
-| Character Length      |  4 |    |    |    |  3 |    |    |  2 |    |  1 |  1 |
-| Byte                  | F0 | 9D | 84 | 9E | E2 | 82 | BF | D6 | 8F | 61 | 62 |
-| is_second_byte        |    |  X |    |    |    |  X |    |    |  X |    |    |
-| is_third_byte         |    |    |  X |    |    |    |  X |    |    |    |    |
-| is_fourth_byte        |    |    |    |  X |    |    |    |    |    |    |    |
-| expected_continuation |    |  X |  X |  X |    |  X |  X |    |  X |    |    |
-| is_continuation       |    |  X |  X |  X |    |  X |  X |    |  X |    |    |
-
-The errors here are basically (Second Byte OR Third Byte OR Fourth Byte == Continuation):
-
-- **Extra Continuations:** Any continuation that is not a second, third or fourth byte is not
-  part of a valid 2-, 3- or 4-byte character and is thus an error. It could be that it's just
-  floating around extra outside of any character, or that there is an illegal 5-byte character,
-  or maybe it's at the beginning of the file before any characters have started; but it's an
-  error in all these cases.
-- **Missing Continuations:** Any second, third or fourth byte that *isn't* a continuation is an error, because that means
-  we started a new character before we were finished with the current one.
-
-####  Getting the Previous Bytes
-
-Because we want to know if a byte is the *second* (or third, or fourth) byte of a multibyte
-character, we need to "shift the bytes" to find that out. This is what they mean:
-
-- `is_continuation`: if the current byte is a continuation.
-- `is_second_byte`: if 1 byte back is the start of a 2-, 3- or 4-byte character.
-- `is_third_byte`: if 2 bytes back is the start of a 3- or 4-byte character.
-- `is_fourth_byte`: if 3 bytes back is the start of a 4-byte character.
-
-We use shuffles to go n bytes back, selecting part of the current `input` and part of the
-`prev_input` (search for `.prev<1>`, `.prev<2>`, etc.). These are passed in by the caller
-function, because the 1-byte-back data is used by other checks as well.
-
-####   Getting the Continuation Mask
-
-Once we have the right bytes, we have to get the masks. To do this, we treat UTF-8 bytes as
-numbers, using signed `<` and `>` operations to check if they are continuations or leads.
-In fact, we treat the numbers as *signed*, partly because it helps us, and partly because
-Intel's SIMD presently only offers signed `<` and `>` operations (not unsigned ones).
-
-In UTF-8, bytes that start with the bits 110, 1110 and 11110 are 2-, 3- and 4-byte "leads,"
-respectively, meaning they expect to have 1, 2 and 3 "continuation bytes" after them.
-Continuation bytes start with 10, and ASCII (1-byte characters) starts with 0.
-
-When treated as signed numbers, they look like this:
-
-| Type         | High Bits  | Binary Range | Signed |
-|--------------|------------|--------------|--------|
-| ASCII        | `0`        | `01111111`   |   127  |
-|              |            | `00000000`   |     0  |
-| 4+-Byte Lead | `1111`     | `11111111`   |    -1  |
-|              |            | `11110000    |   -16  |
-| 3-Byte Lead  | `1110`     | `11101111`   |   -17  |
-|              |            | `11100000    |   -32  |
-| 2-Byte Lead  | `110`      | `11011111`   |   -33  |
-|              |            | `11000000    |   -64  |
-| Continuation | `10`       | `10111111`   |   -65  |
-|              |            | `10000000    |  -128  |
-
-This makes it pretty easy to get the continuation mask! It's just a single comparison:
-
-```
-is_continuation = input < -64`
-```
-
-We can do something similar for the others, but it takes two comparisons instead of one: "is
-the start of a 4-byte character" is `< -32` and `> -65`, for example. And 2+ bytes is `< 0` and
-`> -64`. Surely we can do better, they're right next to each other!
-
-####  Getting the is_xxx Masks: Shifting the Range
-
-Notice *why* continuations were a single comparison. The actual *range* would require two
-comparisons--`< -64` and `> -129`--but all characters are always greater than -128, so we get
-that for free. In fact, if we had *unsigned* comparisons, 2+, 3+ and 4+ comparisons would be
-just as easy: 4+ would be `> 239`, 3+ would be `> 223`, and 2+ would be `> 191`.
-
-Instead, we add 128 to each byte, shifting the range up to make comparison easy. This wraps
-ASCII down into the negative, and puts 4+-Byte Lead at the top:
-
-| Type                 | High Bits  | Binary Range | Signed |
-|----------------------|------------|--------------|-------|
-| 4+-Byte Lead (+ 127) | `0111`     | `01111111`   |   127 |
-|                      |            | `01110000    |   112 |
-|----------------------|------------|--------------|-------|
-| 3-Byte Lead (+ 127)  | `0110`     | `01101111`   |   111 |
-|                      |            | `01100000    |    96 |
-|----------------------|------------|--------------|-------|
-| 2-Byte Lead (+ 127)  | `010`      | `01011111`   |    95 |
-|                      |            | `01000000    |    64 |
-|----------------------|------------|--------------|-------|
-| Continuation (+ 127) | `00`       | `00111111`   |    63 |
-|                      |            | `00000000    |     0 |
-|----------------------|------------|--------------|-------|
-| ASCII (+ 127)        | `1`        | `11111111`   |    -1 |
-|                      |            | `10000000`   |  -128 |
-|----------------------|------------|--------------|-------|
-
-*Now* we can use signed `>` on all of them:
-
-```
-prev1 = input.prev<1>
-prev2 = input.prev<2>
-prev3 = input.prev<3>
-prev1_flipped = input.prev<1>(prev_input) ^ 0x80; // Same as `+ 128`
-prev2_flipped = input.prev<2>(prev_input) ^ 0x80; // Same as `+ 128`
-prev3_flipped = input.prev<3>(prev_input) ^ 0x80; // Same as `+ 128`
-is_second_byte = prev1_flipped > 63;2+-byte lead
-is_third_byte  = prev2_flipped > 95;3+-byte lead
-is_fourth_byte = prev3_flipped > 111; // 4+-byte lead
-```
-
-NOTE: we use `^ 0x80` instead of `+ 128` in the code, which accomplishes the same thing, and even takes the same number
-of cycles as `+`, but on many Intel architectures can be parallelized better (you can do 3
-`^`'s at a time on Haswell, but only 2 `+`'s).
-
-That doesn't look like it saved us any instructions, did it? Well, because we're adding the
-same number to all of them, we can save one of those `+ 128` operations by assembling
-`prev2_flipped` out of prev 1 and prev 3 instead of assembling it from input and adding 128
-to it. One more instruction saved!
-
-```
-prev1 = input.prev<1>
-prev3 = input.prev<3>
-prev1_flipped = prev1 ^ 0x80; // Same as `+ 128`
-prev3_flipped = prev3 ^ 0x80; // Same as `+ 128`
-prev2_flipped = prev1_flipped.concat<2>(prev3_flipped): // <shuffle: take the first 2 bytes from prev1 and the rest from prev3  
-```
-
-####  Bringing It All Together: Detecting the Errors
-
-At this point, we have `is_continuation`, `is_first_byte`, `is_second_byte` and `is_third_byte`.
-All we have left to do is check if they match!
-
-```
-return (is_second_byte | is_third_byte | is_fourth_byte) ^ is_continuation;
-```
-
-But wait--there's more. The above statement is only 3 operations, but they *cannot be done in
-parallel*. You have to do 2 `|`'s and then 1 `&`. Haswell, at least, has 3 ports that can do
-bitwise operations, and we're only using 1!
-
-####  Epilogue: Addition For Booleans
-
-There is one big case the above code doesn't explicitly talk about--what if is_second_byte
-and is_third_byte are BOTH true? That means there is a 3-byte and 2-byte character right next
-to each other (or any combination), and the continuation could be part of either of them!
-Our algorithm using `&` and `|` won't detect that the continuation byte is problematic.
-
-Never fear, though. If that situation occurs, we'll already have detected that the second
-leading byte was an error, because it was supposed to be a part of the preceding multibyte
-character, but it *wasn't a continuation*.
-
-We could stop here, but it turns out that we can fix it using `+` and `-` instead of `|` and
-`&`, which is both interesting and possibly useful (even though we're not using it here). It
-exploits the fact that in SIMD, a *true* value is -1, and a *false* value is 0. So those
-comparisons were giving us numbers!
-
-Given that, if you do `is_second_byte + is_third_byte + is_fourth_byte`, under normal
-circumstances you will either get 0 (0 + 0 + 0) or -1 (-1 + 0 + 0, etc.). Thus,
-`(is_second_byte + is_third_byte + is_fourth_byte) - is_continuation` will yield 0 only if
-*both* or *neither* are 0 (0-0 or -1 - -1). You'll get 1 or -1 if they are different. Because
-*any* nonzero value is treated as an error (not just -1), we're just fine here :)
-
-Further, if *more than one* multibyte character overlaps,
-`is_second_byte + is_third_byte + is_fourth_byte` will be -2 or -3! Subtracting `is_continuation`
-from *that* is guaranteed to give you a nonzero value (-1, -2 or -3). So it'll always be
-considered an error.
-
-One reason you might want to do this is parallelism. ^ and | are not associative, so
-(A | B | C) ^ D will always be three operations in a row: either you do A | B -> | C -> ^ D, or
-you do B | C -> | A -> ^ D. But addition and subtraction *are* associative: (A + B + C) - D can
-be written as `(A + B) + (C - D)`. This means you can do A + B and C - D at the same time, and
-then adds the result together. Same number of operations, but if the processor can run
-independent things in parallel (which most can), it runs faster.
-
-This doesn't help us on Intel, but might help us elsewhere: on Haswell, at least, | and ^ have
-a super nice advantage in that more of them can be run at the same time (they can run on 3
-ports, while + and - can run on 2)! This means that we can do A | B while we're still doing C,
-saving us the cycle we would have earned by using +. Even more, using an instruction with a
-wider array of ports can help *other* code run ahead, too, since these instructions can "get
-out of the way," running on a port other instructions can't.
-
-####  Epilogue II: One More Trick
-
-There's one more relevant trick up our sleeve, it turns out: it turns out on Intel we can "pay
-for" the (prev<1> + 128) instruction, because it can be used to save an instruction in
-check_special_cases()--but we'll talk about that there :)
-
-
-
-
-## About the Project
-
-### Bindings and Ports of simdjson
-
-We distinguish between "bindings" (which just wrap the C++ code) and a port to another programming language (which reimplements everything).
-
-
-- [ZippyJSON](https://github.com/michaeleisel/zippyjson): Swift bindings for the simdjson project.
-- [pysimdjson](https://github.com/TkTech/pysimdjson): Python bindings for the simdjson project.
-- [simdjson-rs](https://github.com/Licenser/simdjson-rs): Rust port.
-- [simdjson-rust](https://github.com/SunDoge/simdjson-rust): Rust wrapper (bindings).
-- [SimdJsonSharp](https://github.com/EgorBo/SimdJsonSharp): C# version for .NET Core (bindings and full port).
-- [simdjson_nodejs](https://github.com/luizperes/simdjson_nodejs): Node.js bindings for the simdjson project.
-- [simdjson_php](https://github.com/crazyxman/simdjson_php): PHP bindings for the simdjson project.
-- [simdjson_ruby](https://github.com/saka1/simdjson_ruby): Ruby bindings for the simdjson project.
-- [simdjson-go](https://github.com/minio/simdjson-go): Go port using Golang assembly.
-- [rcppsimdjson](https://github.com/eddelbuettel/rcppsimdjson): R bindings.
-
-### Tools
-
-- `json2json mydoc.json` parses the document, constructs a model and then dumps back the result to standard output.
-- `json2json -d mydoc.json` parses the document, constructs a model and then dumps model (as a tape) to standard output. The tape format is described in the accompanying file `tape.md`.
-- `minify mydoc.json` minifies the JSON document, outputting the result to standard output. Minifying means to remove the unneeded white space characters.
-- `jsonpointer mydoc.json <jsonpath> <jsonpath> ... <jsonpath>` parses the document, constructs a model and then processes a series of [JSON Pointer paths](https://tools.ietf.org/html/rfc6901). The result is itself a JSON document.
-
-
 ### Various References
 
-- [Google double-conv](https://github.com/google/double-conversion/)
 - [How to implement atoi using SIMD?](https://stackoverflow.com/questions/35127060/how-to-implement-atoi-using-simd)
 - [Parsing JSON is a Minefield 💣](http://seriot.ch/parsing_json.php)
 - https://tools.ietf.org/html/rfc7159
-- The Mison implementation in rust https://github.com/pikkr/pikkr
 - http://rapidjson.org/md_doc_sax.html
 - https://github.com/Geal/parser_benchmarks/tree/master/json
 - Gron: A command line tool that makes JSON greppable https://news.ycombinator.com/item?id=16727665
@@ -622,31 +290,3 @@ Inspiring links:
 - https://auth0.com/blog/beating-json-performance-with-protobuf/
 - https://gist.github.com/shijuvar/25ad7de9505232c87034b8359543404a
 - https://github.com/frankmcsherry/blog/blob/master/posts/2018-02-11.md
-
-Validating UTF-8 takes no more than 0.7 cycles per byte:
-
-- https://github.com/lemire/fastvalidate-utf-8 https://lemire.me/blog/2018/05/16/validating-utf-8-strings-using-as-little-as-0-7-cycles-per-byte/
-
-### Academic References
-
-- T.Mühlbauer, W.Rödiger, R.Seilbeck, A.Reiser, A.Kemper, and T.Neumann. Instant loading for main memory databases. PVLDB, 6(14):1702–1713, 2013. (SIMD-based CSV parsing)
-- Mytkowicz, Todd, Madanlal Musuvathi, and Wolfram Schulte. "Data-parallel finite-state machines." ACM SIGARCH Computer Architecture News. Vol. 42. No. 1. ACM, 2014.
-- Lu, Yifan, et al. "Tree structured data processing on GPUs." Cloud Computing, Data Science & Engineering-Confluence, 2017 7th International Conference on. IEEE, 2017.
-- Sidhu, Reetinder. "High throughput, tree automata based XML processing using FPGAs." Field-Programmable Technology (FPT), 2013 International Conference on. IEEE, 2013.
-- Dai, Zefu, Nick Ni, and Jianwen Zhu. "A 1 cycle-per-byte XML parsing accelerator." Proceedings of the 18th annual ACM/SIGDA international symposium on Field programmable gate arrays. ACM, 2010.
-- Lin, Dan, et al. "Parabix: Boosting the efficiency of text processing on commodity processors." High Performance Computer Architecture (HPCA), 2012 IEEE 18th International Symposium on. IEEE, 2012. http://parabix.costar.sfu.ca/export/1783/docs/HPCA2012/final_ieee/final.pdf
-- Deshmukh, V. M., and G. R. Bamnote. "An empirical evaluation of optimization parameters in XML parsing for performance enhancement." Computer, Communication and Control (IC4), 2015 International Conference on. IEEE, 2015.
-- Moussalli, Roger, et al. "Efficient XML Path Filtering Using GPUs." ADMS@ VLDB. 2011.
-- Jianliang, Ma, et al. "Parallel speculative dom-based XML parser." High Performance Computing and Communication & 2012 IEEE 9th International Conference on Embedded Software and Systems (HPCC-ICESS), 2012 IEEE 14th International Conference on. IEEE, 2012.
-- Li, Y., Katsipoulakis, N.R., Chandramouli, B., Goldstein, J. and Kossmann, D., 2017. Mison: a fast JSON parser for data analytics. Proceedings of the VLDB Endowment, 10(10), pp.1118-1129. http://www.vldb.org/pvldb/vol10/p1118-li.pdf
-- Cameron, Robert D., et al. "Parallel scanning with bitstream addition: An xml case study." European Conference on Parallel Processing. Springer, Berlin, Heidelberg, 2011.
-- Cameron, Robert D., Kenneth S. Herdy, and Dan Lin. "High performance XML parsing using parallel bit stream technology." Proceedings of the 2008 conference of the center for advanced studies on collaborative research: meeting of minds. ACM, 2008.
-- Shah, Bhavik, et al. "A data parallel algorithm for XML DOM parsing." International XML Database Symposium. Springer, Berlin, Heidelberg, 2009.
-- Cameron, Robert D., and Dan Lin. "Architectural support for SWAR text processing with parallel bit streams: the inductive doubling principle." ACM Sigplan Notices. Vol. 44. No. 3. ACM, 2009.
-- Amagasa, Toshiyuki, Mana Seino, and Hiroyuki Kitagawa. "Energy-Efficient XML Stream Processing through Element-Skipping Parsing." Database and Expert Systems Applications (DEXA), 2013 24th International Workshop on. IEEE, 2013.
-- Medforth, Nigel Woodland. "icXML: Accelerating Xerces-C 3.1. 1 using the Parabix Framework." (2013).
-- Zhang, Qiang Scott. Embedding Parallel Bit Stream Technology Into Expat. Diss. Simon Fraser University, 2010.
-- Cameron, Robert D., et al. "Fast Regular Expression Matching with Bit-parallel Data Streams."
-- Lin, Dan. Bits filter: a high-performance multiple string pattern matching algorithm for malware detection. Diss. School of Computing Science-Simon Fraser University, 2010.
-- Yang, Shiyang. Validation of XML Document Based on Parallel Bit Stream Technology. Diss. Applied Sciences: School of Computing Science, 2013.
-- N. Nakasato, "Implementation of a parallel tree method on a GPU", Journal of Computational Science, vol. 3, no. 3, pp. 132-141, 2012.

README.md

@@ -1,8 +1,9 @@
-[![Build Status](https://cloud.drone.io/api/badges/simdjson/simdjson/status.svg)](https://cloud.drone.io/simdjson/simdjson) 
 [![Fuzzing Status](https://oss-fuzz-build-logs.storage.googleapis.com/badges/simdjson.svg)](https://bugs.chromium.org/p/oss-fuzz/issues/list?sort=-opened&q=proj%3Asimdjson&can=2)
-[![Build status](https://ci.appveyor.com/api/projects/status/ae77wp5v3lebmu6n/branch/master?svg=true)](https://ci.appveyor.com/project/lemire/simdjson-jmmti/branch/master)
-[![CirrusCI](https://api.cirrus-ci.com/github/simdjson/simdjson.svg)](https://cirrus-ci.com/github/simdjson/simdjson)
-[![][license img]][license]  [![Doxygen Documentation](https://img.shields.io/badge/docs-doxygen-green.svg)](https://simdjson.org/api/0.3.1/index.html)
+![Ubuntu 18.04 CI](https://github.com/simdjson/simdjson/workflows/Ubuntu%2018.04%20CI%20(GCC%207)/badge.svg)
+[![Ubuntu 20.04 CI](https://github.com/simdjson/simdjson/workflows/Ubuntu%2020.04%20CI%20(GCC%209)/badge.svg)](https://simdjson.org/plots.html)
+![VS16-CI](https://github.com/simdjson/simdjson/workflows/VS16-CI/badge.svg)
+![MinGW64-CI](https://github.com/simdjson/simdjson/workflows/MinGW64-CI/badge.svg)
+[![][license img]][license]  [![Doxygen Documentation](https://img.shields.io/badge/docs-doxygen-green.svg)](https://simdjson.org/api/0.5.0/index.html)
 
 simdjson : Parsing gigabytes of JSON per second
 ===============================================
@@ -69,7 +70,7 @@ Usage documentation is available:
 * [Performance](doc/performance.md) shows some more advanced scenarios and how to tune for them.
 * [Implementation Selection](doc/implementation-selection.md) describes runtime CPU detection and
   how you can work with it.
-* [API](https://simdjson.org/api/0.3.1/annotated.html) contains the automatically generated API documentation.
+* [API](https://simdjson.org/api/0.5.0/annotated.html) contains the automatically generated API documentation.
 
 Performance results
 -------------------
@@ -127,6 +128,7 @@ Real-world usage
 - [Microsoft FishStore](https://github.com/microsoft/FishStore)
 - [Yandex ClickHouse](https://github.com/yandex/ClickHouse)
 - [Clang Build Analyzer](https://github.com/aras-p/ClangBuildAnalyzer)
+- [Shopify HeapProfiler](https://github.com/Shopify/heap-profiler)
 
 If you are planning to use simdjson in a product, please work from one of our releases.
 
@@ -136,6 +138,7 @@ Bindings and Ports of simdjson
 We distinguish between "bindings" (which just wrap the C++ code) and a port to another programming language (which reimplements everything).
 
 - [ZippyJSON](https://github.com/michaeleisel/zippyjson): Swift bindings for the simdjson project.
+- [libpy_simdjson](https://github.com/gerrymanoim/libpy_simdjson/): high-speed Python bindings for simdjson using [libpy](https://github.com/quantopian/libpy).
 - [pysimdjson](https://github.com/TkTech/pysimdjson): Python bindings for the simdjson project.
 - [simdjson-rs](https://github.com/simd-lite): Rust port.
 - [simdjson-rust](https://github.com/SunDoge/simdjson-rust): Rust wrapper (bindings).
@@ -143,6 +146,7 @@ We distinguish between "bindings" (which just wrap the C++ code) and a port to a
 - [simdjson_nodejs](https://github.com/luizperes/simdjson_nodejs): Node.js bindings for the simdjson project.
 - [simdjson_php](https://github.com/crazyxman/simdjson_php): PHP bindings for the simdjson project.
 - [simdjson_ruby](https://github.com/saka1/simdjson_ruby): Ruby bindings for the simdjson project.
+- [fast_jsonparser](https://github.com/anilmaurya/fast_jsonparser): Ruby bindings for the simdjson project.
 - [simdjson-go](https://github.com/minio/simdjson-go): Go port using Golang assembly.
 - [rcppsimdjson](https://github.com/eddelbuettel/rcppsimdjson): R bindings.
 

RELEASES.md

@@ -1,3 +1,39 @@
+# 0.5
+
+## Highlights
+
+Performance
+* Faster and simpler UTF-8 validation with the lookup4 algorithm https://github.com/simdjson/simdjson/pull/993
+* We improved the performance of simdjson under Visual Studio by about 25%. Users will still get better performance with clang-cl (+30%) but the gap has been reduced. https://github.com/simdjson/simdjson/pull/1031
+
+Code usability
+* In `parse_many`, when parsing streams of JSON documetns, we give to the users runtime control as to whether threads are used (via the parser.threaded attribute). https://github.com/simdjson/simdjson/issues/925
+* Prefixed public macros to avoid name clashes with other libraries. https://github.com/simdjson/simdjson/issues/1035
+* Better documentation regarding package managers (brew, MSYS2, conan, apt, vcpkg, FreeBSD package manager, etc.).
+* Better documentation regarding  CMake usage.
+
+Standards
+* We improved standard compliance with respect to both the JSON RFC 8259 and JSON Pointer RFC 6901. We added the at_pointer method to nodes for standard-compliant JSON Pointer queries. The legacy `at(std::string_view)` method remains but is deprecated since it is not standard-compliant as per  RFC 6901.
+* We removed computed GOTOs without sacrificing performance thus improving the C++ standard compliance (since computed GOTOs are compiler-specific extensions).
+* Better support for C++20 https://github.com/simdjson/simdjson/pull/1050
+
+# 0.4
+
+## Highlights
+
+- Test coverage has been greatly improved and we have resolved many static-analysis warnings on different systems.
+- We added a fast (8GB/s) minifier that works directly on JSON strings.
+- We added fast (10GB/s) UTF-8 validator that works directly on strings (any strings, including non-JSON).
+- The array and object elements have a constant-time size() method.
+- Performance improvements to the API (type(), get<>()).
+- The parse_many function (ndjson) has been entirely reworked. It now uses a single secondary thread instead of several new threads.
+- We have introduced a faster UTF-8 validation algorithm (lookup3) for all kernels (ARM, x64 SSE, x64 AVX).
+- C++11 support for older compilers and systems.
+- FreeBSD support (and tests).
+- We support the clang front-end compiler (clangcl) under Visual Studio.
+- It is now possible to target ARM platforms under Visual Studio.
+- The simdjson library will never abort or print to standard output/error.
+
 # 0.3
 
 ## Highlights

benchmark/CMakeLists.txt

@@ -1,5 +1,13 @@
 include_directories( . linux )
-link_libraries(simdjson simdjson-flags simdjson-windows-headers test-data)
+link_libraries(simdjson-windows-headers test-data)
+
+
+if (TARGET benchmark::benchmark)
+  add_executable(bench_sax bench_sax.cpp)
+  target_link_libraries(bench_sax simdjson-internal-flags simdjson-include-source benchmark::benchmark)
+endif (TARGET benchmark::benchmark)
+
+link_libraries(simdjson simdjson-flags)
 add_executable(benchfeatures benchfeatures.cpp)
 add_executable(get_corpus_benchmark get_corpus_benchmark.cpp)
 add_executable(perfdiff perfdiff.cpp)
@@ -14,12 +22,6 @@ target_compile_definitions(parse_nonumberparsing PRIVATE SIMDJSON_SKIPNUMBERPARS
 add_executable(parse_nostringparsing parse.cpp)
 target_compile_definitions(parse_nostringparsing PRIVATE SIMDJSON_SKIPSTRINGPARSING)
 
-if (TARGET benchmark::benchmark)
-  link_libraries(benchmark::benchmark)
-  add_executable(bench_parse_call bench_parse_call.cpp)
-  add_executable(bench_dom_api bench_dom_api.cpp)
-endif()
-
 if (TARGET competition-all)
   add_executable(distinctuseridcompetition distinctuseridcompetition.cpp)
   target_link_libraries(distinctuseridcompetition competition-core)
@@ -34,4 +36,10 @@ if (TARGET competition-all)
   target_compile_definitions(allparsingcompetition PRIVATE ALLPARSER)
 endif()
 
+if (TARGET benchmark::benchmark)
+  link_libraries(benchmark::benchmark)
+  add_executable(bench_parse_call bench_parse_call.cpp)
+  add_executable(bench_dom_api bench_dom_api.cpp)
+endif()
+
 include(checkperf.cmake)

benchmark/bench_dom_api.cpp

@@ -11,8 +11,68 @@ const padded_string EMPTY_ARRAY("[]", 2);
 const char *TWITTER_JSON = SIMDJSON_BENCHMARK_DATA_DIR "twitter.json";
 const char *NUMBERS_JSON = SIMDJSON_BENCHMARK_DATA_DIR "numbers.json";
 
+static void recover_one_string(State& state) {
+  dom::parser parser;
+  const std::string_view data = "\"one string\"";
+  padded_string docdata{data};
+  // we do not want mem. alloc. in the loop.
+  auto error = parser.allocate(docdata.size());
+  if(error) {
+      cout << error << endl;
+      return;
+  }
+  dom::element doc;
+  if ((error = parser.parse(docdata).get(doc))) {
+    cerr << "could not parse string" << error << endl;
+    return;
+  }
+  for (SIMDJSON_UNUSED auto _ : state) {
+      std::string_view v;
+      error = doc.get(v);
+      if (error) {
+        cerr << "could not get string" << error << endl;
+        return;
+      }
+      benchmark::DoNotOptimize(v);
+  }
+}
+BENCHMARK(recover_one_string);
 
 
+static void serialize_twitter(State& state) {
+  dom::parser parser;
+  padded_string docdata;
+  auto error = padded_string::load(TWITTER_JSON).get(docdata);
+  if(error) {
+      cerr << "could not parse twitter.json" << error << endl;
+      return;
+  }
+  // we do not want mem. alloc. in the loop.
+  if((error = parser.allocate(docdata.size()))) {
+      cout << error << endl;
+      return;
+  }
+  dom::element doc;
+  if ((error = parser.parse(docdata).get(doc))) {
+    cerr << "could not parse twitter.json" << error << endl;
+    return;
+  }
+  size_t bytes = 0;
+  for (SIMDJSON_UNUSED auto _ : state) {
+    std::string serial = simdjson::minify(doc);
+    bytes += serial.size();
+    benchmark::DoNotOptimize(serial);
+  }
+  // Gigabyte: https://en.wikipedia.org/wiki/Gigabyte
+  state.counters["Gigabytes"] = benchmark::Counter(
+	        double(bytes), benchmark::Counter::kIsRate,
+	        benchmark::Counter::OneK::kIs1000); // For GiB : kIs1024
+  state.counters["docs"] = Counter(double(state.iterations()), benchmark::Counter::kIsRate);
+}
+BENCHMARK(serialize_twitter)->Repetitions(10)->ComputeStatistics("max", [](const std::vector<double>& v) -> double {
+    return *(std::max_element(std::begin(v), std::end(v)));
+  })->DisplayAggregatesOnly(true);
+
 static void numbers_scan(State& state) {
   // Prints the number of results in twitter.json
   dom::parser parser;
@@ -22,7 +82,7 @@ static void numbers_scan(State& state) {
     cerr << "could not read " << NUMBERS_JSON << " as an array: " << error << endl;
     return;
   }
-  for (UNUSED auto _ : state) {
+  for (SIMDJSON_UNUSED auto _ : state) {
     std::vector<double> container;
     for (auto e : arr) {
       double x;
@@ -44,7 +104,7 @@ static void numbers_size_scan(State& state) {
     cerr << "could not read " << NUMBERS_JSON << " as an array: " << error << endl;
     return;
   }
-  for (UNUSED auto _ : state) {
+  for (SIMDJSON_UNUSED auto _ : state) {
     std::vector<double> container;
     container.resize(arr.size());
     size_t pos = 0;
@@ -70,7 +130,7 @@ static void numbers_type_scan(State& state) {
     cerr << "could not read " << NUMBERS_JSON << " as an array" << endl;
     return;
   }
-  for (UNUSED auto _ : state) {
+  for (SIMDJSON_UNUSED auto _ : state) {
     std::vector<double> container;
     for (auto e : arr) {
       dom::element_type actual_type = e.type();
@@ -96,7 +156,7 @@ static void numbers_type_size_scan(State& state) {
     cerr << "could not read " << NUMBERS_JSON << " as an array: " << error << endl;
     return;
   }
-  for (UNUSED auto _ : state) {
+  for (SIMDJSON_UNUSED auto _ : state) {
     std::vector<double> container;
     container.resize(arr.size());
     size_t pos = 0;
@@ -121,7 +181,7 @@ static void numbers_load_scan(State& state) {
   dom::parser parser;
   dom::array arr;
   simdjson::error_code error;
-  for (UNUSED auto _ : state) {
+  for (SIMDJSON_UNUSED auto _ : state) {
     // this may hit the disk, but probably just once
     if ((error = parser.load(NUMBERS_JSON).get(arr))) {
       cerr << "could not read " << NUMBERS_JSON << " as an array: " << error << endl;
@@ -144,7 +204,7 @@ static void numbers_load_size_scan(State& state) {
   dom::parser parser;
   dom::array arr;
   simdjson::error_code error;
-  for (UNUSED auto _ : state) {
+  for (SIMDJSON_UNUSED auto _ : state) {
     // this may hit the disk, but probably just once
     if ((error = parser.load(NUMBERS_JSON).get(arr))) {
       cerr << "could not read " << NUMBERS_JSON << " as an array" << endl;
@@ -173,7 +233,7 @@ static void numbers_exceptions_scan(State& state) {
   // Prints the number of results in twitter.json
   dom::parser parser;
   dom::array arr = parser.load(NUMBERS_JSON);
-  for (UNUSED auto _ : state) {
+  for (SIMDJSON_UNUSED auto _ : state) {
     std::vector<double> container;
     for (double x : arr) {
       container.push_back(x);
@@ -188,7 +248,7 @@ static void numbers_exceptions_size_scan(State& state) {
   // Prints the number of results in twitter.json
   dom::parser parser;
   dom::array arr = parser.load(NUMBERS_JSON);
-  for (UNUSED auto _ : state) {
+  for (SIMDJSON_UNUSED auto _ : state) {
     std::vector<double> container;
     container.resize(arr.size());
     size_t pos = 0;
@@ -208,7 +268,7 @@ static void numbers_type_exceptions_scan(State& state) {
   // Prints the number of results in twitter.json
   dom::parser parser;
   dom::array arr = parser.load(NUMBERS_JSON);
-  for (UNUSED auto _ : state) {
+  for (SIMDJSON_UNUSED auto _ : state) {
     std::vector<double> container;
     for (auto e : arr) {
       dom::element_type actual_type = e.type();
@@ -227,7 +287,7 @@ static void numbers_type_exceptions_size_scan(State& state) {
   // Prints the number of results in twitter.json
   dom::parser parser;
   dom::array arr = parser.load(NUMBERS_JSON);
-  for (UNUSED auto _ : state) {
+  for (SIMDJSON_UNUSED auto _ : state) {
     std::vector<double> container;
     container.resize(arr.size());
     size_t pos = 0;
@@ -248,7 +308,7 @@ BENCHMARK(numbers_type_exceptions_size_scan);
 static void numbers_exceptions_load_scan(State& state) {
   // Prints the number of results in twitter.json
   dom::parser parser;
-  for (UNUSED auto _ : state) {
+  for (SIMDJSON_UNUSED auto _ : state) {
     // this may hit the disk, but probably just once
     dom::array arr = parser.load(NUMBERS_JSON);
     std::vector<double> container;
@@ -264,7 +324,7 @@ BENCHMARK(numbers_exceptions_load_scan);
 static void numbers_exceptions_load_size_scan(State& state) {
   // Prints the number of results in twitter.json
   dom::parser parser;
-  for (UNUSED auto _ : state) {
+  for (SIMDJSON_UNUSED auto _ : state) {
     // this may hit the disk, but probably just once
     dom::array arr = parser.load(NUMBERS_JSON);
     std::vector<double> container;
@@ -285,7 +345,7 @@ static void twitter_count(State& state) {
   // Prints the number of results in twitter.json
   dom::parser parser;
   dom::element doc = parser.load(TWITTER_JSON);
-  for (UNUSED auto _ : state) {
+  for (SIMDJSON_UNUSED auto _ : state) {
     uint64_t result_count = doc["search_metadata"]["count"];
     if (result_count != 100) { return; }
   }
@@ -298,7 +358,7 @@ static void iterator_twitter_count(State& state) {
   // Prints the number of results in twitter.json
   padded_string json = padded_string::load(TWITTER_JSON);
   ParsedJson pj = build_parsed_json(json);
-  for (UNUSED auto _ : state) {
+  for (SIMDJSON_UNUSED auto _ : state) {
     ParsedJson::Iterator iter(pj);
     // uint64_t result_count = doc["search_metadata"]["count"];
     if (!iter.move_to_key("search_metadata")) { return; }
@@ -316,7 +376,7 @@ static void twitter_default_profile(State& state) {
   // Count unique users with a default profile.
   dom::parser parser;
   dom::element doc = parser.load(TWITTER_JSON);
-  for (UNUSED auto _ : state) {
+  for (SIMDJSON_UNUSED auto _ : state) {
     set<string_view> default_users;
     for (dom::object tweet : doc["statuses"]) {
       dom::object user = tweet["user"];
@@ -334,7 +394,7 @@ static void twitter_image_sizes(State& state) {
   dom::parser parser;
   dom::element doc = parser.load(TWITTER_JSON);
   simdjson::error_code error;
-  for (UNUSED auto _ : state) {
+  for (SIMDJSON_UNUSED auto _ : state) {
     set<tuple<uint64_t, uint64_t>> image_sizes;
     for (dom::object tweet : doc["statuses"]) {
       dom::array media;
@@ -359,7 +419,7 @@ static void error_code_twitter_count(State& state) noexcept {
   simdjson::error_code error;
   dom::element doc;
   if ((error = parser.load(TWITTER_JSON).get(doc))) { return; }
-  for (UNUSED auto _ : state) {
+  for (SIMDJSON_UNUSED auto _ : state) {
     uint64_t value;
     if ((error = doc["search_metadata"]["count"].get(value))) { return; }
     if (value != 100) { return; }
@@ -373,7 +433,7 @@ static void error_code_twitter_default_profile(State& state) noexcept {
   simdjson::error_code error;
   dom::element doc;
   if ((error = parser.load(TWITTER_JSON).get(doc))) { std::cerr << error << std::endl; return; }
-  for (UNUSED auto _ : state) {
+  for (SIMDJSON_UNUSED auto _ : state) {
     set<string_view> default_users;
 
     dom::array tweets;
@@ -403,7 +463,7 @@ static void iterator_twitter_default_profile(State& state) {
   auto error = padded_string::load(TWITTER_JSON).get(json);
   if (error) { std::cerr << error << std::endl; return; }
   ParsedJson pj = build_parsed_json(json);
-  for (UNUSED auto _ : state) {
+  for (SIMDJSON_UNUSED auto _ : state) {
     set<string_view> default_users;
     ParsedJson::Iterator iter(pj);
 
@@ -444,7 +504,7 @@ static void error_code_twitter_image_sizes(State& state) noexcept {
   simdjson::error_code error;
   dom::element doc;
   if ((error = parser.load(TWITTER_JSON).get(doc))) { std::cerr << error << std::endl; return; }
-  for (UNUSED auto _ : state) {
+  for (SIMDJSON_UNUSED auto _ : state) {
     set<tuple<uint64_t, uint64_t>> image_sizes;
     dom::array statuses;
     if ((error = doc["statuses"].get(statuses))) { return; }
@@ -476,7 +536,7 @@ static void iterator_twitter_image_sizes(State& state) {
   auto error = padded_string::load(TWITTER_JSON).get(json);
   if (error) { std::cerr << error << std::endl; return; }
   ParsedJson pj = build_parsed_json(json);
-  for (UNUSED auto _ : state) {
+  for (SIMDJSON_UNUSED auto _ : state) {
     set<tuple<uint64_t, uint64_t>> image_sizes;
     ParsedJson::Iterator iter(pj);
 
@@ -541,7 +601,7 @@ static void print_json(State& state) noexcept {
 
   int code = json_parse(json, parser);
   if (code) { cerr << error_message(code) << endl; return; }
-  for (UNUSED auto _ : state) {
+  for (SIMDJSON_UNUSED auto _ : state) {
     std::stringstream s;
     if (!parser.print_json(s)) { cerr << "print_json failed" << endl; return; }
   }

benchmark/bench_parse_call.cpp

@@ -10,6 +10,36 @@ const char *GSOC_JSON = SIMDJSON_BENCHMARK_DATA_DIR "gsoc-2018.json";
 
 
 
+static void unicode_validate_twitter(State& state) {
+  dom::parser parser;
+  padded_string docdata;
+  auto error = padded_string::load(TWITTER_JSON).get(docdata);
+  if(error) {
+      cerr << "could not parse twitter.json" << error << endl;
+      return;
+  }
+  // we do not want mem. alloc. in the loop.
+  error = parser.allocate(docdata.size());
+  if(error) {
+      cout << error << endl;
+      return;
+  }
+  size_t bytes = 0;
+  for (SIMDJSON_UNUSED auto _ : state) {
+    bool is_ok = simdjson::validate_utf8(docdata.data(), docdata.size());
+    bytes += docdata.size();
+    benchmark::DoNotOptimize(is_ok);
+  }
+  // Gigabyte: https://en.wikipedia.org/wiki/Gigabyte
+  state.counters["Gigabytes"] = benchmark::Counter(
+	        double(bytes), benchmark::Counter::kIsRate,
+	        benchmark::Counter::OneK::kIs1000); // For GiB : kIs1024
+  state.counters["docs"] = Counter(double(state.iterations()), benchmark::Counter::kIsRate);
+}
+BENCHMARK(unicode_validate_twitter)->Repetitions(10)->ComputeStatistics("max", [](const std::vector<double>& v) -> double {
+    return *(std::max_element(std::begin(v), std::end(v)));
+  })->DisplayAggregatesOnly(true);
+
 static void parse_twitter(State& state) {
   dom::parser parser;
   padded_string docdata;
@@ -25,10 +55,9 @@ static void parse_twitter(State& state) {
       return;
   }
   size_t bytes = 0;
-  for (UNUSED auto _ : state) {
+  for (SIMDJSON_UNUSED auto _ : state) {
     dom::element doc;
     bytes += docdata.size();
-    ;
     if ((error = parser.parse(docdata).get(doc))) {
       cerr << "could not parse twitter.json" << error << endl;
       return;
@@ -61,7 +90,7 @@ static void parse_gsoc(State& state) {
       return;
   }
   size_t bytes = 0;
-  for (UNUSED auto _ : state) {
+  for (SIMDJSON_UNUSED auto _ : state) {
     bytes += docdata.size();
     dom::element doc;
     if ((error = parser.parse(docdata).get(doc))) {
@@ -87,7 +116,7 @@ SIMDJSON_DISABLE_DEPRECATED_WARNING
 static void json_parse(State& state) {
   ParsedJson pj;
   if (!pj.allocate_capacity(EMPTY_ARRAY.length())) { return; }
-  for (UNUSED auto _ : state) {
+  for (SIMDJSON_UNUSED auto _ : state) {
     auto error = json_parse(EMPTY_ARRAY, pj);
     if (error) { return; }
   }
@@ -97,7 +126,7 @@ BENCHMARK(json_parse);
 static void parser_parse_error_code(State& state) {
   dom::parser parser;
   if (parser.allocate(EMPTY_ARRAY.length())) { return; }
-  for (UNUSED auto _ : state) {
+  for (SIMDJSON_UNUSED auto _ : state) {
     auto error = parser.parse(EMPTY_ARRAY).error();
     if (error) { return; }
   }
@@ -109,9 +138,9 @@ BENCHMARK(parser_parse_error_code);
 static void parser_parse_exception(State& state) {
   dom::parser parser;
   if (parser.allocate(EMPTY_ARRAY.length())) { return; }
-  for (UNUSED auto _ : state) {
+  for (SIMDJSON_UNUSED auto _ : state) {
     try {
-      UNUSED dom::element doc = parser.parse(EMPTY_ARRAY);
+      SIMDJSON_UNUSED dom::element doc = parser.parse(EMPTY_ARRAY);
     } catch(simdjson_error &j) {
       cout << j.what() << endl;
       return;
@@ -125,7 +154,7 @@ BENCHMARK(parser_parse_exception);
 SIMDJSON_PUSH_DISABLE_WARNINGS
 SIMDJSON_DISABLE_DEPRECATED_WARNING
 static void build_parsed_json(State& state) {
-  for (UNUSED auto _ : state) {
+  for (SIMDJSON_UNUSED auto _ : state) {
     dom::parser parser = simdjson::build_parsed_json(EMPTY_ARRAY);
     if (!parser.valid) { return; }
   }
@@ -134,7 +163,7 @@ SIMDJSON_POP_DISABLE_WARNINGS
 
 BENCHMARK(build_parsed_json);
 static void document_parse_error_code(State& state) {
-  for (UNUSED auto _ : state) {
+  for (SIMDJSON_UNUSED auto _ : state) {
     dom::parser parser;
     auto error = parser.parse(EMPTY_ARRAY).error();
     if (error) { return; }
@@ -145,10 +174,10 @@ BENCHMARK(document_parse_error_code);
 #if SIMDJSON_EXCEPTIONS
 
 static void document_parse_exception(State& state) {
-  for (UNUSED auto _ : state) {
+  for (SIMDJSON_UNUSED auto _ : state) {
     try {
       dom::parser parser;
-      UNUSED dom::element doc = parser.parse(EMPTY_ARRAY);
+      SIMDJSON_UNUSED dom::element doc = parser.parse(EMPTY_ARRAY);
     } catch(simdjson_error &j) {
       cout << j.what() << endl;
       return;

benchmark/bench_sax.cpp

@@ -0,0 +1,359 @@
+#define SIMDJSON_IMPLEMENTATION_FALLBACK 0
+#define SIMDJSON_IMPLEMENTATION_WESTMERE 0
+#define SIMDJSON_IMPLEMENTATION_AMD64 0
+
+#include <iostream>
+#include <sstream>
+#include <random>
+
+#include "simdjson.h"
+
+SIMDJSON_PUSH_DISABLE_ALL_WARNINGS
+#include <benchmark/benchmark.h>
+SIMDJSON_POP_DISABLE_WARNINGS
+
+#include "simdjson.cpp"
+
+#if SIMDJSON_EXCEPTIONS
+
+using namespace benchmark;
+using namespace simdjson;
+using std::cerr;
+using std::endl;
+
+const char *TWITTER_JSON = SIMDJSON_BENCHMARK_DATA_DIR "twitter.json";
+const int REPETITIONS = 10;
+
+#if SIMDJSON_IMPLEMENTATION_HASWELL
+
+#include "twitter/sax_tweet_reader.h"
+
+static void sax_tweets(State &state) {
+  // Load twitter.json to a buffer
+  padded_string json;
+  if (auto error = padded_string::load(TWITTER_JSON).get(json)) { cerr << error << endl; return; }
+
+  // Allocate
+  twitter::sax_tweet_reader reader;
+  if (auto error = reader.set_capacity(json.size())) { cerr << error << endl; return; }
+
+  // Warm the vector
+  if (auto error = reader.read_tweets(json)) { throw error; }
+
+  // Read tweets
+  size_t bytes = 0;
+  size_t tweets = 0;
+  for (SIMDJSON_UNUSED auto _ : state) {
+    if (auto error = reader.read_tweets(json)) { throw error; }
+    bytes += json.size();
+    tweets += reader.tweets.size();
+  }
+  // Gigabyte: https://en.wikipedia.org/wiki/Gigabyte
+  state.counters["Gigabytes"] = benchmark::Counter(
+	        double(bytes), benchmark::Counter::kIsRate,
+	        benchmark::Counter::OneK::kIs1000); // For GiB : kIs1024
+  state.counters["docs"] = Counter(double(state.iterations()), benchmark::Counter::kIsRate);
+  state.counters["tweets"] = Counter(double(tweets), benchmark::Counter::kIsRate);
+}
+BENCHMARK(sax_tweets)->Repetitions(REPETITIONS)->ComputeStatistics("max", [](const std::vector<double>& v) -> double {
+    return *(std::max_element(std::begin(v), std::end(v)));
+  })->DisplayAggregatesOnly(true);
+
+#endif // SIMDJSON_IMPLEMENTATION_HASWELL
+
+#include "twitter/tweet.h"
+
+simdjson_really_inline uint64_t nullable_int(dom::element element) {
+  if (element.is_null()) { return 0; }
+  return element;
+}
+simdjson_really_inline void read_dom_tweets(dom::parser &parser, padded_string &json, std::vector<twitter::tweet> &tweets) {
+  for (dom::element tweet : parser.parse(json)["statuses"]) {
+    auto user = tweet["user"];
+    tweets.push_back(
+      {
+        tweet["id"],
+        tweet["text"],
+        tweet["created_at"],
+        nullable_int(tweet["in_reply_to_status_id"]),
+        tweet["retweet_count"],
+        tweet["favorite_count"],
+        { user["id"], user["screen_name"] }
+      }
+    );
+  }
+}
+
+static void dom_tweets(State &state) {
+  // Load twitter.json to a buffer
+  padded_string json;
+  if (auto error = padded_string::load(TWITTER_JSON).get(json)) { cerr << error << endl; return; }
+
+  // Allocate
+  dom::parser parser;
+  if (auto error = parser.allocate(json.size())) { cerr << error << endl; return; };
+
+  // Warm the vector
+  std::vector<twitter::tweet> tweets;
+  read_dom_tweets(parser, json, tweets);
+
+  // Read tweets
+  size_t bytes = 0;
+  size_t num_tweets = 0;
+  for (SIMDJSON_UNUSED auto _ : state) {
+    tweets.clear();
+    read_dom_tweets(parser, json, tweets);
+    bytes += json.size();
+    num_tweets += tweets.size();
+  }
+  // Gigabyte: https://en.wikipedia.org/wiki/Gigabyte
+  state.counters["Gigabytes"] = benchmark::Counter(
+	        double(bytes), benchmark::Counter::kIsRate,
+	        benchmark::Counter::OneK::kIs1000); // For GiB : kIs1024
+  state.counters["docs"] = Counter(double(state.iterations()), benchmark::Counter::kIsRate);
+  state.counters["tweets"] = Counter(double(num_tweets), benchmark::Counter::kIsRate);
+}
+BENCHMARK(dom_tweets)->Repetitions(REPETITIONS)->ComputeStatistics("max", [](const std::vector<double>& v) -> double {
+    return *(std::max_element(std::begin(v), std::end(v)));
+  })->DisplayAggregatesOnly(true);
+
+static void dom_parse(State &state) {
+  // Load twitter.json to a buffer
+  padded_string json;
+  if (auto error = padded_string::load(TWITTER_JSON).get(json)) { cerr << error << endl; return; }
+
+  // Allocate
+  dom::parser parser;
+  if (auto error = parser.allocate(json.size())) { cerr << error << endl; return; };
+
+  // Read tweets
+  size_t bytes = 0;
+  for (SIMDJSON_UNUSED auto _ : state) {
+    if (parser.parse(json).error()) { throw "Parsing failed"; };
+    bytes += json.size();
+  }
+  // Gigabyte: https://en.wikipedia.org/wiki/Gigabyte
+  state.counters["Gigabytes"] = benchmark::Counter(
+	        double(bytes), benchmark::Counter::kIsRate,
+	        benchmark::Counter::OneK::kIs1000); // For GiB : kIs1024
+  state.counters["docs"] = Counter(double(state.iterations()), benchmark::Counter::kIsRate);
+}
+BENCHMARK(dom_parse)->Repetitions(REPETITIONS)->ComputeStatistics("max", [](const std::vector<double>& v) -> double {
+    return *(std::max_element(std::begin(v), std::end(v)));
+  })->DisplayAggregatesOnly(true);
+
+
+/********************
+ * Large file parsing benchmarks:
+ ********************/
+
+static std::string build_json_array(size_t N) {
+  std::default_random_engine e;
+  std::uniform_real_distribution<> dis(0, 1);
+  std::stringstream myss;
+  myss << "[" << std::endl;
+  if(N > 0) {
+    myss << "{ \"x\":" << dis(e) << ",  \"y\":" << dis(e) << ", \"z\":" << dis(e) << "}" << std::endl;
+  }
+  for(size_t i = 1; i < N; i++) {
+    myss << "," << std::endl;
+    myss << "{ \"x\":" << dis(e) << ",  \"y\":" << dis(e) << ", \"z\":" << dis(e) << "}";
+  }
+  myss << std::endl;
+  myss << "]" << std::endl;
+  std::string answer = myss.str();
+  std::cout << "Creating a source file spanning " << (answer.size() + 512) / 1024 << " KB " << std::endl;  
+  return answer;
+}
+
+static const simdjson::padded_string& get_my_json_str() {
+  static simdjson::padded_string s = build_json_array(1000000);
+  return s;
+}
+
+struct my_point {
+  double x;
+  double y;
+  double z;
+};
+
+// ./benchmark/bench_sax --benchmark_filter=largerandom
+
+
+/*** 
+ * We start with the naive DOM-based approach.
+ **/
+static void dom_parse_largerandom(State &state) {
+  // Load twitter.json to a buffer
+  const padded_string& json = get_my_json_str();
+
+  // Allocate
+  dom::parser parser;
+  if (auto error = parser.allocate(json.size())) { cerr << error << endl; return; };
+
+  // Read
+  size_t bytes = 0;
+  simdjson::error_code error;
+  for (SIMDJSON_UNUSED auto _ : state) {
+    std::vector<my_point> container;
+    dom::element doc;
+    if ((error = parser.parse(json).get(doc))) { 
+      std::cerr << "failure: " << error << std::endl;
+      throw "Parsing failed"; 
+    };
+    for (auto p : doc) {
+      container.emplace_back(my_point{p["x"], p["y"], p["z"]});
+    }
+    bytes += json.size();
+    benchmark::DoNotOptimize(container.data());
+
+  }
+  // Gigabyte: https://en.wikipedia.org/wiki/Gigabyte
+  state.counters["Gigabytes"] = benchmark::Counter(
+	        double(bytes), benchmark::Counter::kIsRate,
+	        benchmark::Counter::OneK::kIs1000); // For GiB : kIs1024
+  state.counters["docs"] = Counter(double(state.iterations()), benchmark::Counter::kIsRate);
+}
+
+BENCHMARK(dom_parse_largerandom)->Repetitions(REPETITIONS)->ComputeStatistics("max", [](const std::vector<double>& v) -> double {
+    return *(std::max_element(std::begin(v), std::end(v)));
+  })->DisplayAggregatesOnly(true);
+
+#if SIMDJSON_IMPLEMENTATION_HASWELL
+
+/*** 
+ * Next we are going to code the SAX approach.
+ **/
+
+SIMDJSON_TARGET_HASWELL
+
+namespace largerandom {
+namespace {
+
+using namespace simdjson;
+using namespace haswell;
+using namespace haswell::stage2;
+struct sax_point_reader_visitor {
+public:
+  sax_point_reader_visitor(std::vector<my_point> &_points) : points(_points) {
+  }
+
+  simdjson_really_inline error_code visit_document_start(json_iterator &) { return SUCCESS; }
+  simdjson_really_inline error_code visit_object_start(json_iterator &) { return SUCCESS; }
+  simdjson_really_inline error_code visit_key(json_iterator &, const uint8_t *key) {
+    switch(key[0]) {
+      case 'x':
+        idx = 0;
+        break;
+      case 'y':
+        idx = 2;
+        break;
+      case 'z':
+        idx = 3;
+        break;  
+    }
+    return SUCCESS; 
+  }
+  simdjson_really_inline error_code visit_primitive(json_iterator &, const uint8_t *value) {
+    return numberparsing::parse_double(value).get(buffer[idx]);
+  }
+  simdjson_really_inline error_code visit_array_start(json_iterator &)  { return SUCCESS; }
+  simdjson_really_inline error_code visit_array_end(json_iterator &) { return SUCCESS; }
+  simdjson_really_inline error_code visit_object_end(json_iterator &)  { return SUCCESS; }
+  simdjson_really_inline error_code visit_document_end(json_iterator &)  { return SUCCESS; }
+  simdjson_really_inline error_code visit_empty_array(json_iterator &)  { return SUCCESS; }
+  simdjson_really_inline error_code visit_empty_object(json_iterator &)  { return SUCCESS; }
+  simdjson_really_inline error_code visit_root_primitive(json_iterator &, const uint8_t *)  { return SUCCESS; }
+  simdjson_really_inline error_code increment_count(json_iterator &) { return SUCCESS; }
+  std::vector<my_point> &points;
+  size_t idx{0};
+  double buffer[3];
+};
+
+struct sax_point_reader {
+  std::vector<my_point> points;
+  std::unique_ptr<uint8_t[]> string_buf;
+  size_t capacity;
+  dom_parser_implementation dom_parser;
+
+  sax_point_reader();
+  error_code set_capacity(size_t new_capacity);
+  error_code read_points(const padded_string &json);
+}; // struct sax_point_reader
+
+sax_point_reader::sax_point_reader() : points{}, string_buf{}, capacity{0}, dom_parser() {
+}
+
+error_code sax_point_reader::set_capacity(size_t new_capacity) {
+  // string_capacity copied from document::allocate
+  size_t string_capacity = SIMDJSON_ROUNDUP_N(5 * new_capacity / 3 + 32, 64);
+  string_buf.reset(new (std::nothrow) uint8_t[string_capacity]);
+  if (auto error = dom_parser.set_capacity(new_capacity)) { return error; }
+  if (capacity == 0) { // set max depth the first time only
+    if (auto error = dom_parser.set_max_depth(DEFAULT_MAX_DEPTH)) { return error; }
+  }
+  capacity = new_capacity;
+  return SUCCESS;
+}
+
+error_code sax_point_reader::read_points(const padded_string &json) {
+  // Allocate capacity if needed
+  points.clear();
+  if (capacity < json.size()) {
+    if (auto error = set_capacity(capacity)) { return error; }
+  }
+
+  // Run stage 1 first.
+  if (auto error = dom_parser.stage1((uint8_t *)json.data(), json.size(), false)) { return error; }
+
+  // Then walk the document, parsing the tweets as we go
+  json_iterator iter(dom_parser, 0);
+  sax_point_reader_visitor visitor(points);
+  if (auto error = iter.walk_document<false>(visitor)) { return error; }
+  return SUCCESS;
+}
+
+} // unnamed namespace
+} // namespace largerandom
+
+SIMDJSON_UNTARGET_REGION
+
+
+
+
+
+// ./benchmark/bench_sax --benchmark_filter=largerandom
+static void sax_parse_largerandom(State &state) {
+  // Load twitter.json to a buffer
+  const padded_string& json = get_my_json_str();
+
+  // Allocate
+  largerandom::sax_point_reader reader;
+  if (auto error = reader.set_capacity(json.size())) { throw error; }
+  // warming
+  for(size_t i = 0; i < 10; i++) {
+    if (auto error = reader.read_points(json)) { throw error; }
+  }
+
+  // Read
+  size_t bytes = 0;
+  for (SIMDJSON_UNUSED auto _ : state) {
+    if (auto error = reader.read_points(json)) { throw error; }
+    bytes += json.size();
+    benchmark::DoNotOptimize(reader.points.data());
+  }
+  // Gigabyte: https://en.wikipedia.org/wiki/Gigabyte
+  state.counters["Gigabytes"] = benchmark::Counter(
+	        double(bytes), benchmark::Counter::kIsRate,
+	        benchmark::Counter::OneK::kIs1000); // For GiB : kIs1024
+  state.counters["docs"] = Counter(double(state.iterations()), benchmark::Counter::kIsRate);
+}
+BENCHMARK(sax_parse_largerandom)->Repetitions(REPETITIONS)->ComputeStatistics("max", [](const std::vector<double>& v) -> double {
+    return *(std::max_element(std::begin(v), std::end(v)));
+  })->DisplayAggregatesOnly(true);
+
+#endif // SIMDJSON_IMPLEMENTATION_HASWELL
+
+#endif // SIMDJSON_EXCEPTIONS
+
+BENCHMARK_MAIN();

benchmark/benchfeatures.cpp

@@ -158,7 +158,7 @@ struct feature_benchmarker {
 
   }
 
-  really_inline void run_iterations(size_t iterations, bool stage1_only=false) {
+  simdjson_really_inline void run_iterations(size_t iterations, bool stage1_only=false) {
     struct7.run_iterations(iterations, stage1_only);
     struct7_miss.run_iterations(iterations, stage1_only);
     struct7_full.run_iterations(iterations, stage1_only);

benchmark/benchmark.h

@@ -30,7 +30,6 @@
       event_count allocate_count = collector.end();                            \
       aggregate << allocate_count;                                             \
     }                                                                          \
-    uint64_t S = size;                                                         \
     if (collector.has_events()) {                                              \
       printf("%7.3f", aggregate.best.cycles() / static_cast<double>(size));    \
       if (verbose) {                                                           \
@@ -76,7 +75,6 @@
       event_count allocate_count = collector.end();                            \
       aggregate << allocate_count;                                             \
     }                                                                          \
-    uint64_t S = size;                                                         \
     if (collector.has_events()) {                                              \
       printf("%7.3f", aggregate.best.cycles() / static_cast<double>(size));    \
       if (verbose) {                                                           \

benchmark/benchmarker.h

@@ -225,8 +225,19 @@ struct progress_bar {
   }
 };
 
+/**
+ * The speed at which we can allocate memory is strictly system specific.
+ * It depends on the OS and the runtime library. It is subject to various
+ * system-specific knobs. It is not something that we can reasonably 
+ * benchmark with crude timings.
+ * If someone wants to optimize how simdjson allocate memory, then it will
+ * almost surely require a distinct benchmarking tool. What is meant by
+ * "memory allocation" also requires a definition. Doing "new char[size]" can
+ * do many different things depending on the system.
+ */
+
 enum class BenchmarkStage {
-  ALL,
+  ALL, // This excludes allocation
   ALLOCATE,
   STAGE1,
   STAGE2
@@ -234,7 +245,7 @@ enum class BenchmarkStage {
 
 const char* benchmark_stage_name(BenchmarkStage stage) {
   switch (stage) {
-    case BenchmarkStage::ALL: return "All";
+    case BenchmarkStage::ALL: return "All (Without Allocation)";
     case BenchmarkStage::ALLOCATE: return "Allocate";
     case BenchmarkStage::STAGE1: return "Stage 1";
     case BenchmarkStage::STAGE2: return "Stage 2";
@@ -253,8 +264,8 @@ struct benchmarker {
   // Statistics about the JSON file independent of its speed (amount of utf-8, structurals, etc.).
   // Loaded on first parse.
   json_stats* stats;
-  // Speed and event summary for full parse (including allocation, stage 1 and stage 2)
-  event_aggregate all_stages{};
+  // Speed and event summary for full parse (stage 1 and stage 2, but *excluding* allocation)
+  event_aggregate all_stages_without_allocation{};
   // Speed and event summary for stage 1
   event_aggregate stage1{};
   // Speed and event summary for stage 2
@@ -285,23 +296,24 @@ struct benchmarker {
 
   const event_aggregate& operator[](BenchmarkStage stage) const {
     switch (stage) {
-      case BenchmarkStage::ALL: return this->all_stages;
+      case BenchmarkStage::ALL: return this->all_stages_without_allocation;
       case BenchmarkStage::STAGE1: return this->stage1;
       case BenchmarkStage::STAGE2: return this->stage2;
       case BenchmarkStage::ALLOCATE: return this->allocate_stage;
-      default: exit_error("Unknown stage"); return this->all_stages;
+      default: exit_error("Unknown stage"); return this->all_stages_without_allocation;
     }
   }
 
   int iterations() const {
-    return all_stages.iterations;
+    return all_stages_without_allocation.iterations;
   }
 
-  really_inline void run_iteration(bool stage1_only, bool hotbuffers=false) {
+  simdjson_really_inline void run_iteration(bool stage1_only, bool hotbuffers=false) {
     // Allocate dom::parser
     collector.start();
     dom::parser parser;
-    error_code error = parser.allocate(json.size());
+    // We always allocate at least 64KB. Smaller allocations may actually be slower under some systems.
+    error_code error = parser.allocate(json.size() < 65536 ? 65536 : json.size());
     if (error) {
       exit_error(string("Unable to allocate_stage ") + to_string(json.size()) + " bytes for the JSON result: " + error_message(error));
     }
@@ -329,7 +341,7 @@ struct benchmarker {
     // Stage 2 (unified machine) and the rest
 
     if (stage1_only) {
-      all_stages << stage1_count;
+      all_stages_without_allocation << stage1_count;
     } else {
       event_count stage2_count;
       collector.start();
@@ -339,7 +351,7 @@ struct benchmarker {
       }
       stage2_count = collector.end();
       stage2 << stage2_count;
-      all_stages << allocate_count + stage1_count + stage2_count;
+      all_stages_without_allocation << stage1_count + stage2_count;
     }
     // Calculate stats the first time we parse
     if (stats == NULL) {
@@ -372,7 +384,7 @@ struct benchmarker {
     loop << all_loop_count;
   }
 
-  really_inline void run_iterations(size_t iterations, bool stage1_only, bool hotbuffers=false) {
+  simdjson_really_inline void run_iterations(size_t iterations, bool stage1_only, bool hotbuffers=false) {
     for (size_t i = 0; i<iterations; i++) {
       run_iteration(stage1_only, hotbuffers);
     }
@@ -386,7 +398,7 @@ struct benchmarker {
       prefix,
       "Speed",
       stage.elapsed_ns() / static_cast<double>(stats->blocks), // per block
-      percent(stage.elapsed_sec(), all_stages.elapsed_sec()), // %
+      percent(stage.elapsed_sec(), all_stages_without_allocation.elapsed_sec()), // %
       stage.elapsed_ns() / static_cast<double>(stats->bytes), // per byte
       stage.elapsed_ns() / static_cast<double>(stats->structurals), // per structural
       (static_cast<double>(json.size()) / 1000000000.0) / stage.elapsed_sec() // GB/s
@@ -397,7 +409,7 @@ struct benchmarker {
         prefix,
         "Cycles",
         stage.cycles() / static_cast<double>(stats->blocks),
-        percent(stage.cycles(), all_stages.cycles()),
+        percent(stage.cycles(), all_stages_without_allocation.cycles()),
         stage.cycles() / static_cast<double>(stats->bytes),
         stage.cycles() / static_cast<double>(stats->structurals),
         (stage.cycles() / stage.elapsed_sec()) / 1000000000.0
@@ -406,7 +418,7 @@ struct benchmarker {
         prefix,
         "Instructions",
         stage.instructions() / static_cast<double>(stats->blocks),
-        percent(stage.instructions(), all_stages.instructions()),
+        percent(stage.instructions(), all_stages_without_allocation.instructions()),
         stage.instructions() / static_cast<double>(stats->bytes),
         stage.instructions() / static_cast<double>(stats->structurals),
         stage.instructions() / static_cast<double>(stage.cycles())
@@ -417,9 +429,9 @@ struct benchmarker {
         prefix,
         "Misses",
         stage.branch_misses(),
-        percent(stage.branch_misses(), all_stages.branch_misses()),
+        percent(stage.branch_misses(), all_stages_without_allocation.branch_misses()),
         stage.cache_misses(),
-        percent(stage.cache_misses(), all_stages.cache_misses()),
+        percent(stage.cache_misses(), all_stages_without_allocation.cache_misses()),
         stage.cache_references()
       );
     }
@@ -456,14 +468,14 @@ struct benchmarker {
                 allocate_stage.best.cycles() / static_cast<double>(json.size()),
                 stage1.best.cycles() / static_cast<double>(json.size()),
                 stage2.best.cycles() / static_cast<double>(json.size()),
-                all_stages.best.cycles() / static_cast<double>(json.size()),
-                gb / all_stages.best.elapsed_sec(),
+                all_stages_without_allocation.best.cycles() / static_cast<double>(json.size()),
+                gb / all_stages_without_allocation.best.elapsed_sec(),
                 gb / stage1.best.elapsed_sec(),
                 gb / stage2.best.elapsed_sec());
       } else {
         printf("\"%s\"\t\t\t\t\t%f\t%f\t%f\n",
                 base,
-                gb / all_stages.best.elapsed_sec(),
+                gb / all_stages_without_allocation.best.elapsed_sec(),
                 gb / stage1.best.elapsed_sec(),
                 gb / stage2.best.elapsed_sec());
       }
@@ -490,10 +502,10 @@ struct benchmarker {
           stats->blocks_with_16_structurals_flipped, percent(stats->blocks_with_16_structurals_flipped, stats->blocks));
       }
       printf("\n");
-      printf("All Stages\n");
-      print_aggregate("|    "   , all_stages.best);
+      printf("All Stages (excluding allocation)\n");
+      print_aggregate("|    "   , all_stages_without_allocation.best);
       // frequently, allocation is a tiny fraction of the running time so we omit it
-      if(allocate_stage.best.elapsed_sec() > 0.01 * all_stages.best.elapsed_sec()) {
+      if(allocate_stage.best.elapsed_sec() > 0.01 * all_stages_without_allocation.best.elapsed_sec()) {
         printf("|- Allocation\n");
         print_aggregate("|    ", allocate_stage.best);
       }
@@ -504,17 +516,16 @@ struct benchmarker {
       if (collector.has_events()) {
         double freq1 = (stage1.best.cycles() / stage1.best.elapsed_sec()) / 1000000000.0;
         double freq2 = (stage2.best.cycles() / stage2.best.elapsed_sec()) / 1000000000.0;
-        double freqall = (all_stages.best.cycles() / all_stages.best.elapsed_sec()) / 1000000000.0;
+        double freqall = (all_stages_without_allocation.best.cycles() / all_stages_without_allocation.best.elapsed_sec()) / 1000000000.0;
         double freqmin = min(freq1, freq2);
         double freqmax = max(freq1, freq2);
         if((freqall < 0.95 * freqmin) or (freqall > 1.05 * freqmax)) {
           printf("\nWarning: The processor frequency fluctuates in an expected way!!!\n"
-          "Expect the overall speed not to match stage 1 and stage 2 speeds.\n"
           "Range for stage 1 and stage 2 : [%.3f GHz, %.3f GHz], overall: %.3f GHz.\n",
           freqmin, freqmax, freqall);
         }
       }
-      printf("\n%.1f documents parsed per second (best)\n", 1.0/static_cast<double>(all_stages.best.elapsed_sec()));
+      printf("\n%.1f documents parsed per second (best)\n", 1.0/static_cast<double>(all_stages_without_allocation.best.elapsed_sec()));
     }
   }
 };

benchmark/checkperf.cmake

@@ -7,7 +7,7 @@
 
 # Clone the repository if it's not there
 find_package(Git QUIET)
-if (SIMDJSON_IS_UNDER_GIT AND Git_FOUND AND (GIT_VERSION_STRING VERSION_GREATER  "2.1.4") AND (NOT CMAKE_GENERATOR MATCHES Ninja) ) # We use "-C" which requires a recent git
+if (SIMDJSON_IS_UNDER_GIT AND SIMDJSON_GIT AND Git_FOUND AND (GIT_VERSION_STRING VERSION_GREATER  "2.1.4") AND (NOT CMAKE_GENERATOR MATCHES Ninja) ) # We use "-C" which requires a recent git
   message(STATUS "Git is available and it is recent. We are enabling checkperf targets.")
   # sync_git_repository(myrepo ...) creates two targets:
   # myrepo - if the repo does not exist, creates and syncs it against the origin branch

benchmark/distinctuseridcompetition.cpp

@@ -65,7 +65,7 @@ void print_vec(const std::vector<int64_t> &v) {
 // clang-format on
 
 
-really_inline void simdjson_recurse(std::vector<int64_t> & v, simdjson::dom::element element);
+simdjson_really_inline void simdjson_recurse(std::vector<int64_t> & v, simdjson::dom::element element);
 void simdjson_recurse(std::vector<int64_t> & v, simdjson::dom::array array) {
   for (auto child : array) {
     simdjson_recurse(v, child);
@@ -97,8 +97,8 @@ void simdjson_recurse(std::vector<int64_t> & v, simdjson::dom::object object) {
     }
   }
 }
-really_inline void simdjson_recurse(std::vector<int64_t> & v, simdjson::dom::element element) {
-  UNUSED simdjson::error_code error;
+simdjson_really_inline void simdjson_recurse(std::vector<int64_t> & v, simdjson::dom::element element) {
+  SIMDJSON_UNUSED simdjson::error_code error;
   simdjson::dom::array array;
   simdjson::dom::object object;
   if (not (error = element.get(array))) {
@@ -108,7 +108,7 @@ really_inline void simdjson_recurse(std::vector<int64_t> & v, simdjson::dom::ele
   }
 }
 
-really_inline std::vector<int64_t>
+simdjson_really_inline std::vector<int64_t>
 simdjson_just_dom(simdjson::dom::element doc) {
   std::vector<int64_t> answer;
   simdjson_recurse(answer, doc);
@@ -116,7 +116,7 @@ simdjson_just_dom(simdjson::dom::element doc) {
   return answer;
 }
 
-really_inline std::vector<int64_t>
+simdjson_really_inline std::vector<int64_t>
 simdjson_compute_stats(const simdjson::padded_string &p) {
   std::vector<int64_t> answer;
   simdjson::dom::parser parser;
@@ -129,7 +129,7 @@ simdjson_compute_stats(const simdjson::padded_string &p) {
   return answer;
 }
 
-really_inline simdjson::error_code
+simdjson_really_inline simdjson::error_code
 simdjson_just_parse(const simdjson::padded_string &p) {
   simdjson::dom::parser parser;
   return parser.parse(p).error();
@@ -187,7 +187,7 @@ void sajson_traverse(std::vector<int64_t> &answer, const sajson::value &node) {
   }
 }
 
-really_inline std::vector<int64_t>
+simdjson_really_inline std::vector<int64_t>
 sasjon_just_dom(sajson::document &d) {
   std::vector<int64_t> answer;
   sajson_traverse(answer, d.get_root());
@@ -195,7 +195,7 @@ sasjon_just_dom(sajson::document &d) {
   return answer;
 }
 
-really_inline std::vector<int64_t>
+simdjson_really_inline std::vector<int64_t>
 sasjon_compute_stats(const simdjson::padded_string &p) {
   std::vector<int64_t> answer;
   char *buffer = (char *)malloc(p.size());
@@ -212,7 +212,7 @@ sasjon_compute_stats(const simdjson::padded_string &p) {
   return answer;
 }
 
-really_inline bool
+simdjson_really_inline bool
 sasjon_just_parse(const simdjson::padded_string &p) {
   char *buffer = (char *)malloc(p.size());
   memcpy(buffer, p.data(), p.size());
@@ -263,7 +263,7 @@ void rapid_traverse(std::vector<int64_t> &answer, const rapidjson::Value &v) {
   }
 }
 
-really_inline std::vector<int64_t>
+simdjson_really_inline std::vector<int64_t>
 rapid_just_dom(rapidjson::Document &d) {
   std::vector<int64_t> answer;
   rapid_traverse(answer, d);
@@ -271,7 +271,7 @@ rapid_just_dom(rapidjson::Document &d) {
   return answer;
 }
 
-really_inline std::vector<int64_t>
+simdjson_really_inline std::vector<int64_t>
 rapid_compute_stats(const simdjson::padded_string &p) {
   std::vector<int64_t> answer;
   char *buffer = (char *)malloc(p.size() + 1);
@@ -289,7 +289,7 @@ rapid_compute_stats(const simdjson::padded_string &p) {
   return answer;
 }
 
-really_inline bool
+simdjson_really_inline bool
 rapid_just_parse(const simdjson::padded_string &p) {
   char *buffer = (char *)malloc(p.size() + 1);
   memcpy(buffer, p.data(), p.size());

benchmark/event_counter.h

@@ -133,13 +133,13 @@ struct event_collector {
   }
 #endif
 
-  really_inline void start() {
+  simdjson_really_inline void start() {
 #if defined(__linux)
     linux_events.start();
 #endif
     start_clock = steady_clock::now();
   }
-  really_inline event_count& end() {
+  simdjson_really_inline event_count& end() {
     time_point<steady_clock> end_clock = steady_clock::now();
 #if defined(__linux)
     linux_events.end(count.event_counts);

benchmark/get_corpus_benchmark.cpp

@@ -5,7 +5,7 @@
 #include <iostream>
 
 // Gigabyte: https://en.wikipedia.org/wiki/Gigabyte
-never_inline
+simdjson_never_inline
 double bench(std::string filename, simdjson::padded_string& p) {
   std::chrono::time_point<std::chrono::steady_clock> start_clock =
       std::chrono::steady_clock::now();

benchmark/minifiercompetition.cpp

@@ -93,6 +93,10 @@ int main(int argc, char *argv[]) {
     std::cout << std::endl;
   }
   char *buffer = simdjson::internal::allocate_padded_buffer(p.size() + 1);
+  if(buffer == nullptr) {
+    std::cerr << "Out of memory!" << std::endl;
+    abort();
+  }
   memcpy(buffer, p.data(), p.size());
   buffer[p.size()] = '\0';
 
@@ -139,6 +143,10 @@ int main(int argc, char *argv[]) {
             !just_data);
 
   char *mini_buffer = simdjson::internal::allocate_padded_buffer(p.size() + 1);
+  if(mini_buffer == nullptr) {
+    std::cerr << "Out of memory" << std::endl;
+    abort();
+  }
   size_t minisize;
   auto minierror = minify(p.data(), p.size(),mini_buffer, minisize);
   if (!minierror) { std::cerr << minierror << std::endl; exit(1); }

benchmark/parseandstatcompetition.cpp

@@ -48,7 +48,7 @@ void print_stat(const stat_t &s) {
          s.true_count, s.false_count);
 }
 
-really_inline void simdjson_process_atom(stat_t &s,
+simdjson_really_inline void simdjson_process_atom(stat_t &s,
                                          simdjson::dom::element element) {
   if (element.is<double>()) {
     s.number_count++;
@@ -102,7 +102,7 @@ void simdjson_recurse(stat_t &s, simdjson::dom::element element) {
   }
 }
 
-never_inline stat_t simdjson_compute_stats(const simdjson::padded_string &p) {
+simdjson_never_inline stat_t simdjson_compute_stats(const simdjson::padded_string &p) {
   stat_t s{};
   simdjson::dom::parser parser;
   simdjson::dom::element doc;
@@ -131,7 +131,7 @@ struct Stat {
   size_t stringLength; // Number of code units in all strings
 };
 
-static void GenStatPlus(Stat &stat, const dom::element &v) {
+static void GenStatPlus(Stat &stat, const dom::element v) {
   switch (v.type()) {
   case dom::element_type::ARRAY:
     for (dom::element child : dom::array(v)) {
@@ -142,7 +142,8 @@ static void GenStatPlus(Stat &stat, const dom::element &v) {
     break;
   case dom::element_type::OBJECT:
     for (dom::key_value_pair kv : dom::object(v)) {
-      GenStatPlus(stat, dom::element(kv.value));
+      GenStatPlus(stat, kv.value);
+      stat.stringLength += kv.key.size();
       stat.memberCount++;
       stat.stringCount++;
     }
@@ -211,13 +212,13 @@ static void RapidGenStat(Stat &stat, const rapidjson::Value &v) {
     break;
   }
 }
-never_inline Stat rapidjson_compute_stats_ref(const rapidjson::Value &doc) {
+simdjson_never_inline Stat rapidjson_compute_stats_ref(const rapidjson::Value &doc) {
   Stat s{};
   RapidGenStat(s, doc);
   return s;
 }
 
-never_inline Stat
+simdjson_never_inline Stat
 simdjson_compute_stats_refplus(const simdjson::dom::element &doc) {
   Stat s{};
   GenStatPlus(s, doc);
@@ -267,7 +268,7 @@ void sajson_traverse(stat_t &stats, const sajson::value &node) {
   }
 }
 
-never_inline stat_t sasjon_compute_stats(const simdjson::padded_string &p) {
+simdjson_never_inline stat_t sasjon_compute_stats(const simdjson::padded_string &p) {
   stat_t answer{};
   char *buffer = (char *)malloc(p.size());
   if (buffer == nullptr) {
@@ -328,7 +329,7 @@ void rapid_traverse(stat_t &stats, const rapidjson::Value &v) {
   }
 }
 
-never_inline stat_t rapid_compute_stats(const simdjson::padded_string &p) {
+simdjson_never_inline stat_t rapid_compute_stats(const simdjson::padded_string &p) {
   stat_t answer{};
   char *buffer = (char *)malloc(p.size() + 1);
   if (buffer == nullptr) {
@@ -354,7 +355,7 @@ never_inline stat_t rapid_compute_stats(const simdjson::padded_string &p) {
   return answer;
 }
 
-never_inline stat_t
+simdjson_never_inline stat_t
 rapid_accurate_compute_stats(const simdjson::padded_string &p) {
   stat_t answer{};
   char *buffer = (char *)malloc(p.size() + 1);

benchmark/parsingcompetition.cpp

@@ -53,7 +53,7 @@ using namespace rapidjson;
 
 #ifdef ALLPARSER
 // fastjson has a tricky interface
-void on_json_error(void *, UNUSED const fastjson::ErrorContext &ec) {
+void on_json_error(void *, SIMDJSON_UNUSED const fastjson::ErrorContext &ec) {
   // std::cerr<<"ERROR: "<<ec.mesg<<std::endl;
 }
 bool fastjson_parse(const char *input) {
@@ -65,7 +65,7 @@ bool fastjson_parse(const char *input) {
 // end of fastjson stuff
 #endif
 
-never_inline size_t sum_line_lengths(std::stringstream & is) {
+simdjson_never_inline size_t sum_line_lengths(std::stringstream & is) {
   std::string line;
   size_t sumofalllinelengths{0};
   while(std::getline(is, line)) {

benchmark/statisticalmodel.cpp

@@ -41,7 +41,7 @@ using stat_t = struct stat_s;
 
 
 
-really_inline void simdjson_process_atom(stat_t &s,
+simdjson_really_inline void simdjson_process_atom(stat_t &s,
                                          simdjson::dom::element element) {
   if (element.is<int64_t>()) {
     s.integer_count++;

benchmark/twitter/sax_tweet_reader.h

@@ -0,0 +1,67 @@
+#ifndef TWITTER_SAX_TWEET_READER_H
+#define TWITTER_SAX_TWEET_READER_H
+
+#include "simdjson.h"
+#include "sax_tweet_reader_visitor.h"
+#include "tweet.h"
+#include <vector>
+
+SIMDJSON_TARGET_HASWELL
+
+namespace twitter {
+namespace {
+
+using namespace simdjson;
+using namespace haswell;
+using namespace haswell::stage2;
+
+struct sax_tweet_reader {
+  std::vector<tweet> tweets;
+  std::unique_ptr<uint8_t[]> string_buf;
+  size_t capacity;
+  dom_parser_implementation dom_parser;
+
+  sax_tweet_reader();
+  error_code set_capacity(size_t new_capacity);
+  error_code read_tweets(padded_string &json);
+}; // struct tweet_reader
+
+sax_tweet_reader::sax_tweet_reader() : tweets{}, string_buf{}, capacity{0}, dom_parser() {
+}
+
+error_code sax_tweet_reader::set_capacity(size_t new_capacity) {
+  // string_capacity copied from document::allocate
+  size_t string_capacity = SIMDJSON_ROUNDUP_N(5 * new_capacity / 3 + 32, 64);
+  string_buf.reset(new (std::nothrow) uint8_t[string_capacity]);
+  if (auto error = dom_parser.set_capacity(new_capacity)) { return error; }
+  if (capacity == 0) { // set max depth the first time only
+    if (auto error = dom_parser.set_max_depth(DEFAULT_MAX_DEPTH)) { return error; }
+  }
+  capacity = new_capacity;
+  return SUCCESS;
+}
+
+// NOTE: this assumes the dom_parser is already allocated
+error_code sax_tweet_reader::read_tweets(padded_string &json) {
+  // Allocate capacity if needed
+  tweets.clear();
+  if (capacity < json.size()) {
+    if (auto error = set_capacity(capacity)) { return error; }
+  }
+
+  // Run stage 1 first.
+  if (auto error = dom_parser.stage1((uint8_t *)json.data(), json.size(), false)) { return error; }
+
+  // Then walk the document, parsing the tweets as we go
+  json_iterator iter(dom_parser, 0);
+  sax_tweet_reader_visitor visitor(tweets, string_buf.get());
+  if (auto error = iter.walk_document<false>(visitor)) { return error; }
+  return SUCCESS;
+}
+
+} // unnamed namespace
+} // namespace twitter
+
+SIMDJSON_UNTARGET_REGION
+
+#endif // TWITTER_SAX_TWEET_READER_H

benchmark/twitter/sax_tweet_reader_visitor.h

@@ -0,0 +1,519 @@
+#ifndef TWITTER_SAX_TWEET_READER_VISITOR_H
+#define TWITTER_SAX_TWEET_READER_VISITOR_H
+
+#include "simdjson.h"
+#include "tweet.h"
+#include <vector>
+
+SIMDJSON_TARGET_HASWELL
+
+namespace twitter {
+
+using namespace simdjson;
+using namespace haswell;
+using namespace haswell::stage2;
+
+struct sax_tweet_reader_visitor {
+public:
+  sax_tweet_reader_visitor(std::vector<tweet> &_tweets, uint8_t *string_buf);
+
+  simdjson_really_inline error_code visit_document_start(json_iterator &iter);
+  simdjson_really_inline error_code visit_object_start(json_iterator &iter);
+  simdjson_really_inline error_code visit_key(json_iterator &iter, const uint8_t *key);
+  simdjson_really_inline error_code visit_primitive(json_iterator &iter, const uint8_t *value);
+  simdjson_really_inline error_code visit_array_start(json_iterator &iter);
+  simdjson_really_inline error_code visit_array_end(json_iterator &iter);
+  simdjson_really_inline error_code visit_object_end(json_iterator &iter);
+  simdjson_really_inline error_code visit_document_end(json_iterator &iter);
+  simdjson_really_inline error_code visit_empty_array(json_iterator &iter);
+  simdjson_really_inline error_code visit_empty_object(json_iterator &iter);
+  simdjson_really_inline error_code visit_root_primitive(json_iterator &iter, const uint8_t *value);
+  simdjson_really_inline error_code increment_count(json_iterator &iter);
+
+private:
+  // Since we only care about one thing at each level, we just use depth as the marker for what
+  // object/array we're nested inside.
+  enum class containers {
+    document = 0,   //
+    top_object = 1, // {
+    statuses = 2,   // { "statuses": [
+    tweet = 3,      // { "statuses": [ {
+    user = 4        // { "statuses": [ { "user": {
+  };
+  /**
+   * The largest depth we care about.
+   * There can be things at lower depths.
+   */
+  static constexpr uint32_t MAX_SUPPORTED_DEPTH = uint32_t(containers::user);
+  static constexpr const char *STATE_NAMES[] = {
+    "document",
+    "top object",
+    "statuses",
+    "tweet",
+    "user"
+  };
+  enum class field_type {
+    any,
+    unsigned_integer,
+    string,
+    nullable_unsigned_integer,
+    object,
+    array
+  };
+  struct field {
+    const char * key{};
+    size_t len{0};
+    size_t offset;
+    containers container{containers::document};
+    field_type type{field_type::any};
+  };
+
+  containers container{containers::document};
+  std::vector<tweet> &tweets;
+  uint8_t *current_string_buf_loc;
+  const uint8_t *current_key{};
+
+  simdjson_really_inline bool in_container(json_iterator &iter);
+  simdjson_really_inline bool in_container_child(json_iterator &iter);
+  simdjson_really_inline void start_container(json_iterator &iter);
+  simdjson_really_inline void end_container(json_iterator &iter);
+  simdjson_really_inline error_code parse_nullable_unsigned(json_iterator &iter, const uint8_t *value, const field &f);
+  simdjson_really_inline error_code parse_unsigned(json_iterator &iter, const uint8_t *value, const field &f);
+  simdjson_really_inline error_code parse_string(json_iterator &iter, const uint8_t *value, const field &f);
+
+  struct field_lookup {
+    field entries[256]{};
+
+    field_lookup();
+    simdjson_really_inline field get(const uint8_t * key, containers container);
+  private:
+    simdjson_really_inline uint8_t hash(const char * key, uint32_t depth);
+    simdjson_really_inline void add(const char * key, size_t len, containers container, field_type type, size_t offset);
+    simdjson_really_inline void neg(const char * const key, uint32_t depth);
+  };
+  static field_lookup fields;
+}; // sax_tweet_reader_visitor
+
+sax_tweet_reader_visitor::sax_tweet_reader_visitor(std::vector<tweet> &_tweets, uint8_t *string_buf)
+  : tweets{_tweets},
+    current_string_buf_loc{string_buf} {
+}
+
+simdjson_really_inline error_code sax_tweet_reader_visitor::visit_document_start(json_iterator &iter) {
+  start_container(iter);
+  return SUCCESS;
+}
+simdjson_really_inline error_code sax_tweet_reader_visitor::visit_array_start(json_iterator &iter) {
+  // If we're not in a container we care about, don't bother with the rest
+  if (!in_container_child(iter)) { return SUCCESS; }
+
+  // Handle fields first
+  if (current_key) {
+    switch (fields.get(current_key, container).type) {
+      case field_type::array: // { "statuses": [
+        start_container(iter);
+        return SUCCESS;
+      case field_type::any:
+        return SUCCESS;
+      case field_type::object:
+      case field_type::unsigned_integer:
+      case field_type::nullable_unsigned_integer:
+      case field_type::string:
+        iter.log_error("unexpected array field");
+        return INCORRECT_TYPE;
+    }
+  }
+
+  // We're not in a field, so it must be a child of an array. We support any of those.
+  iter.log_error("unexpected array");
+  return INCORRECT_TYPE;
+}
+simdjson_really_inline error_code sax_tweet_reader_visitor::visit_object_start(json_iterator &iter) {
+  // If we're not in a container we care about, don't bother with the rest
+  if (!in_container_child(iter)) { return SUCCESS; }
+
+  // Handle known fields
+  if (current_key) {
+    auto f = fields.get(current_key, container);
+    switch (f.type) {
+      case field_type::object: // { "statuses": [ { "user": {
+        start_container(iter);
+        return SUCCESS;
+      case field_type::any:
+        return SUCCESS;
+      case field_type::array:
+      case field_type::unsigned_integer:
+      case field_type::nullable_unsigned_integer:
+      case field_type::string:
+        iter.log_error("unexpected object field");
+        return INCORRECT_TYPE;
+    }
+  }
+
+  // It's not a field, so it's a child of an array or document
+  switch (container) {
+    case containers::document: // top_object: {
+    case containers::statuses: // tweet:      { "statuses": [ {
+      start_container(iter);
+      return SUCCESS;
+    case containers::top_object:
+    case containers::tweet:
+    case containers::user:
+      iter.log_error("unexpected object");
+      return INCORRECT_TYPE;
+  }
+  SIMDJSON_UNREACHABLE();
+  return UNINITIALIZED;
+}
+simdjson_really_inline error_code sax_tweet_reader_visitor::visit_key(json_iterator &, const uint8_t *key) {
+  current_key = key;
+  return SUCCESS;
+}
+simdjson_really_inline error_code sax_tweet_reader_visitor::visit_primitive(json_iterator &iter, const uint8_t *value) {
+  // Don't bother unless we're in a container we care about
+  if (!in_container(iter)) { return SUCCESS; }
+
+  // Handle fields first
+  if (current_key) {
+    auto f = fields.get(current_key, container);
+    switch (f.type) {
+      case field_type::unsigned_integer:
+        return parse_unsigned(iter, value, f);
+      case field_type::nullable_unsigned_integer:
+        return parse_nullable_unsigned(iter, value, f);
+      case field_type::string:
+        return parse_string(iter, value, f);
+      case field_type::any:
+        return SUCCESS;
+      case field_type::array:
+      case field_type::object:
+        iter.log_error("unexpected primitive");
+        return INCORRECT_TYPE;
+    }
+  }
+
+  // If it's not a field, it's a child of an array.
+  // The only array we support is statuses, which must contain objects.
+  iter.log_error("unexpected primitive");
+  return INCORRECT_TYPE;
+}
+simdjson_really_inline error_code sax_tweet_reader_visitor::visit_array_end(json_iterator &iter) {
+  if (in_container(iter)) { end_container(iter); }
+  return SUCCESS;
+}
+simdjson_really_inline error_code sax_tweet_reader_visitor::visit_object_end(json_iterator &iter) {
+  if (in_container(iter)) { end_container(iter); }
+  return SUCCESS;
+}
+
+simdjson_really_inline error_code sax_tweet_reader_visitor::visit_document_end(json_iterator &iter) {
+  iter.log_end_value("document");
+  return SUCCESS;
+}
+
+simdjson_really_inline error_code sax_tweet_reader_visitor::visit_empty_array(json_iterator &) {
+  return SUCCESS;
+}
+simdjson_really_inline error_code sax_tweet_reader_visitor::visit_empty_object(json_iterator &) {
+  return SUCCESS;
+}
+simdjson_really_inline error_code sax_tweet_reader_visitor::visit_root_primitive(json_iterator &iter, const uint8_t *) {
+  iter.log_error("unexpected root primitive");
+  return INCORRECT_TYPE;
+}
+
+simdjson_really_inline error_code sax_tweet_reader_visitor::increment_count(json_iterator &) { return SUCCESS; }
+
+simdjson_really_inline bool sax_tweet_reader_visitor::in_container(json_iterator &iter) {
+  return iter.depth == uint32_t(container);
+}
+simdjson_really_inline bool sax_tweet_reader_visitor::in_container_child(json_iterator &iter) {
+  return iter.depth == uint32_t(container) + 1;
+}
+simdjson_really_inline void sax_tweet_reader_visitor::start_container(json_iterator &iter) {
+  SIMDJSON_ASSUME(iter.depth <= MAX_SUPPORTED_DEPTH); // Asserts in debug mode
+  container = containers(iter.depth);
+  if (logger::LOG_ENABLED) { iter.log_start_value(STATE_NAMES[iter.depth]); }
+}
+simdjson_really_inline void sax_tweet_reader_visitor::end_container(json_iterator &iter) {
+  if (logger::LOG_ENABLED) { iter.log_end_value(STATE_NAMES[int(container)]); }
+  container = containers(int(container) - 1);
+}
+simdjson_really_inline error_code sax_tweet_reader_visitor::parse_nullable_unsigned(json_iterator &iter, const uint8_t *value, const field &f) {
+  iter.log_value(f.key);
+  auto i = reinterpret_cast<uint64_t *>(reinterpret_cast<char *>(&tweets.back() + f.offset));
+  if (auto error = numberparsing::parse_unsigned(value).get(*i)) {
+    // If number parsing failed, check if it's null before returning the error
+    if (!atomparsing::is_valid_null_atom(value)) { iter.log_error("expected number or null"); return error; }
+    i = 0;
+  }
+  return SUCCESS;
+}
+simdjson_really_inline error_code sax_tweet_reader_visitor::parse_unsigned(json_iterator &iter, const uint8_t *value, const field &f) {
+  iter.log_value(f.key);
+  auto i = reinterpret_cast<uint64_t *>(reinterpret_cast<char *>(&tweets.back() + f.offset));
+  return numberparsing::parse_unsigned(value).get(*i);
+}
+simdjson_really_inline error_code sax_tweet_reader_visitor::parse_string(json_iterator &iter, const uint8_t *value, const field &f) {
+  iter.log_value(f.key);
+  auto s = reinterpret_cast<std::string_view *>(reinterpret_cast<char *>(&tweets.back() + f.offset));
+  return stringparsing::parse_string_to_buffer(value, current_string_buf_loc, *s);
+}
+
+sax_tweet_reader_visitor::field_lookup sax_tweet_reader_visitor::fields{};
+
+simdjson_really_inline uint8_t sax_tweet_reader_visitor::field_lookup::hash(const char * key, uint32_t depth) {
+  // These shift numbers were chosen specifically because this yields only 2 collisions between
+  // keys in twitter.json, leaves 0 as a distinct value, and has 0 collisions between keys we
+  // actually care about.
+  return uint8_t((key[0] << 0) ^ (key[1] << 3) ^ (key[2] << 3) ^ (key[3] << 1) ^ depth);
+}
+simdjson_really_inline sax_tweet_reader_visitor::field sax_tweet_reader_visitor::field_lookup::get(const uint8_t * key, containers c) {
+  auto index = hash((const char *)key, uint32_t(c));
+  auto entry = entries[index];
+  // TODO if any key is > SIMDJSON_PADDING, this will access inaccessible memory!
+  if (c != entry.container || memcmp(key, entry.key, entry.len)) { return entries[0]; }
+  return entry;
+}
+simdjson_really_inline void sax_tweet_reader_visitor::field_lookup::add(const char * key, size_t len, containers c, field_type type, size_t offset) {
+  auto index = hash(key, uint32_t(c));
+  if (index == 0) {
+    fprintf(stderr, "%s (depth %d) hashes to zero, which is used as 'missing value'\n", key, int(c));
+    assert(false);
+  }
+  if (entries[index].key) {
+    fprintf(stderr, "%s (depth %d) collides with %s (depth %d) !\n", key, int(c), entries[index].key, int(entries[index].container));
+    assert(false);
+  }
+  entries[index] = { key, len, offset, c, type };
+}
+simdjson_really_inline void sax_tweet_reader_visitor::field_lookup::neg(const char * const key, uint32_t depth) {
+  auto index = hash(key, depth);
+  if (entries[index].key) {
+    fprintf(stderr, "%s (depth %d) conflicts with %s (depth %d) !\n", key, depth, entries[index].key, int(entries[index].container));
+    assert(false);
+  }
+}
+
+sax_tweet_reader_visitor::field_lookup::field_lookup() {
+  add("\"statuses\"", strlen("\"statuses\""), containers::top_object, field_type::array, 0); // { "statuses": [...]
+  #define TWEET_FIELD(KEY, TYPE) add("\"" #KEY "\"", strlen("\"" #KEY "\""), containers::tweet, TYPE, offsetof(tweet, KEY));
+  TWEET_FIELD(id, field_type::unsigned_integer);
+  TWEET_FIELD(in_reply_to_status_id, field_type::nullable_unsigned_integer);
+  TWEET_FIELD(retweet_count, field_type::unsigned_integer);
+  TWEET_FIELD(favorite_count, field_type::unsigned_integer);
+  TWEET_FIELD(text, field_type::string);
+  TWEET_FIELD(created_at, field_type::string);
+  TWEET_FIELD(user, field_type::object)
+  #undef TWEET_FIELD
+  #define USER_FIELD(KEY, TYPE) add("\"" #KEY "\"", strlen("\"" #KEY "\""), containers::user, TYPE, offsetof(tweet, user)+offsetof(twitter_user, KEY));
+  USER_FIELD(id, field_type::unsigned_integer);
+  USER_FIELD(screen_name, field_type::string);
+  #undef USER_FIELD
+
+  // Check for collisions with other (unused) hash keys in typical twitter JSON
+  #define NEG(key, depth) neg("\"" #key "\"", depth);
+  NEG(display_url, 9);
+  NEG(expanded_url, 9);
+  neg("\"h\":", 9);
+  NEG(indices, 9);
+  NEG(resize, 9);
+  NEG(url, 9);
+  neg("\"w\":", 9);
+  NEG(display_url, 8);
+  NEG(expanded_url, 8);
+  neg("\"h\":", 8);
+  NEG(indices, 8);
+  NEG(large, 8);
+  NEG(medium, 8);
+  NEG(resize, 8);
+  NEG(small, 8);
+  NEG(thumb, 8);
+  NEG(url, 8);
+  neg("\"w\":", 8);
+  NEG(display_url, 7);
+  NEG(expanded_url, 7);
+  NEG(id_str, 7);
+  NEG(id, 7);
+  NEG(indices, 7);
+  NEG(large, 7);
+  NEG(media_url_https, 7);
+  NEG(media_url, 7);
+  NEG(medium, 7);
+  NEG(name, 7);
+  NEG(sizes, 7);
+  NEG(small, 7);
+  NEG(source_status_id_str, 7);
+  NEG(source_status_id, 7);
+  NEG(thumb, 7);
+  NEG(type, 7);
+  NEG(url, 7);
+  NEG(urls, 7);
+  NEG(description, 6);
+  NEG(display_url, 6);
+  NEG(expanded_url, 6);
+  NEG(id_str, 6);
+  NEG(id, 6);
+  NEG(indices, 6);
+  NEG(media_url_https, 6);
+  NEG(media_url, 6);
+  NEG(name, 6);
+  NEG(sizes, 6);
+  NEG(source_status_id_str, 6);
+  NEG(source_status_id, 6);
+  NEG(type, 6);
+  NEG(url, 6);
+  NEG(urls, 6);
+  NEG(contributors_enabled, 5);
+  NEG(default_profile_image, 5);
+  NEG(default_profile, 5);
+  NEG(description, 5);
+  NEG(entities, 5);
+  NEG(favourites_count, 5);
+  NEG(follow_request_sent, 5);
+  NEG(followers_count, 5);
+  NEG(following, 5);
+  NEG(friends_count, 5);
+  NEG(geo_enabled, 5);
+  NEG(hashtags, 5);
+  NEG(id_str, 5);
+  NEG(id, 5);
+  NEG(is_translation_enabled, 5);
+  NEG(is_translator, 5);
+  NEG(iso_language_code, 5);
+  NEG(lang, 5);
+  NEG(listed_count, 5);
+  NEG(location, 5);
+  NEG(media, 5);
+  NEG(name, 5);
+  NEG(notifications, 5);
+  NEG(profile_background_color, 5);
+  NEG(profile_background_image_url_https, 5);
+  NEG(profile_background_image_url, 5);
+  NEG(profile_background_tile, 5);
+  NEG(profile_banner_url, 5);
+  NEG(profile_image_url_https, 5);
+  NEG(profile_image_url, 5);
+  NEG(profile_link_color, 5);
+  NEG(profile_sidebar_border_color, 5);
+  NEG(profile_sidebar_fill_color, 5);
+  NEG(profile_text_color, 5);
+  NEG(profile_use_background_image, 5);
+  NEG(protected, 5);
+  NEG(result_type, 5);
+  NEG(statuses_count, 5);
+  NEG(symbols, 5);
+  NEG(time_zone, 5);
+  NEG(url, 5);
+  NEG(urls, 5);
+  NEG(user_mentions, 5);
+  NEG(utc_offset, 5);
+  NEG(verified, 5);
+  NEG(contributors_enabled, 4);
+  NEG(contributors, 4);
+  NEG(coordinates, 4);
+  NEG(default_profile_image, 4);
+  NEG(default_profile, 4);
+  NEG(description, 4);
+  NEG(entities, 4);
+  NEG(favorited, 4);
+  NEG(favourites_count, 4);
+  NEG(follow_request_sent, 4);
+  NEG(followers_count, 4);
+  NEG(following, 4);
+  NEG(friends_count, 4);
+  NEG(geo_enabled, 4);
+  NEG(geo, 4);
+  NEG(hashtags, 4);
+  NEG(id_str, 4);
+  NEG(in_reply_to_screen_name, 4);
+  NEG(in_reply_to_status_id_str, 4);
+  NEG(in_reply_to_user_id_str, 4);
+  NEG(in_reply_to_user_id, 4);
+  NEG(is_translation_enabled, 4);
+  NEG(is_translator, 4);
+  NEG(iso_language_code, 4);
+  NEG(lang, 4);
+  NEG(listed_count, 4);
+  NEG(location, 4);
+  NEG(media, 4);
+  NEG(metadata, 4);
+  NEG(name, 4);
+  NEG(notifications, 4);
+  NEG(place, 4);
+  NEG(possibly_sensitive, 4);
+  NEG(profile_background_color, 4);
+  NEG(profile_background_image_url_https, 4);
+  NEG(profile_background_image_url, 4);
+  NEG(profile_background_tile, 4);
+  NEG(profile_banner_url, 4);
+  NEG(profile_image_url_https, 4);
+  NEG(profile_image_url, 4);
+  NEG(profile_link_color, 4);
+  NEG(profile_sidebar_border_color, 4);
+  NEG(profile_sidebar_fill_color, 4);
+  NEG(profile_text_color, 4);
+  NEG(profile_use_background_image, 4);
+  NEG(protected, 4);
+  NEG(result_type, 4);
+  NEG(retweeted, 4);
+  NEG(source, 4);
+  NEG(statuses_count, 4);
+  NEG(symbols, 4);
+  NEG(time_zone, 4);
+  NEG(truncated, 4);
+  NEG(url, 4);
+  NEG(urls, 4);
+  NEG(user_mentions, 4);
+  NEG(utc_offset, 4);
+  NEG(verified, 4);
+  NEG(contributors, 3);
+  NEG(coordinates, 3);
+  NEG(entities, 3);
+  NEG(favorited, 3);
+  NEG(geo, 3);
+  NEG(id_str, 3);
+  NEG(in_reply_to_screen_name, 3);
+  NEG(in_reply_to_status_id_str, 3);
+  NEG(in_reply_to_user_id_str, 3);
+  NEG(in_reply_to_user_id, 3);
+  NEG(lang, 3);
+  NEG(metadata, 3);
+  NEG(place, 3);
+  NEG(possibly_sensitive, 3);
+  NEG(retweeted_status, 3);
+  NEG(retweeted, 3);
+  NEG(source, 3);
+  NEG(truncated, 3);
+  NEG(completed_in, 2);
+  NEG(count, 2);
+  NEG(max_id_str, 2);
+  NEG(max_id, 2);
+  NEG(next_results, 2);
+  NEG(query, 2);
+  NEG(refresh_url, 2);
+  NEG(since_id_str, 2);
+  NEG(since_id, 2);
+  NEG(search_metadata, 1);
+  #undef NEG
+}
+
+// sax_tweet_reader_visitor::field_lookup::find_min() {
+//   int min_count = 100000;
+//   for (int a=0;a<4;a++) {
+//     for (int b=0;b<4;b++) {
+//       for (int c=0;c<4;c++) {
+//         twitter::sax_tweet_reader_visitor::field_lookup fields(a,b,c);
+//         if (fields.collision_count) { continue; }
+//         if (fields.zero_emission) { continue; }
+//         if (fields.conflict_count < min_count) { printf("min=%d,%d,%d (%d)", a, b, c, fields.conflict_count); }
+//       }
+//     }
+//   }
+// }
+
+} // namespace twitter
+
+SIMDJSON_UNTARGET_REGION
+
+#endif // TWITTER_SAX_TWEET_READER_VISITOR_H

benchmark/twitter/tweet.h

@@ -0,0 +1,21 @@
+#ifndef TWEET_H
+#define TWEET_H
+
+#include "simdjson.h"
+#include "twitter_user.h"
+
+namespace twitter {
+
+struct tweet {
+  uint64_t id{};
+  std::string_view text{};
+  std::string_view created_at{};
+  uint64_t in_reply_to_status_id{};
+  uint64_t retweet_count{};
+  uint64_t favorite_count{};
+  twitter_user user{};
+};
+
+} // namespace twitter
+
+#endif // TWEET_H

benchmark/twitter/twitter_user.h

@@ -0,0 +1,15 @@
+#ifndef TWITTER_USER_H
+#define TWITTER_USER_H
+
+#include "simdjson.h"
+
+namespace twitter {
+
+struct twitter_user {
+  uint64_t id{};
+  std::string_view screen_name{};
+};
+
+} // namespace twitter
+
+#endif // TWITTER_USER_H

cmake/simdjson-flags.cmake

@@ -53,6 +53,9 @@ endif()
 option(SIMDJSON_COMPETITION "Compile competitive benchmarks" ON)
 
 option(SIMDJSON_GOOGLE_BENCHMARKS "compile the Google Benchmark benchmarks" ON)
+if(SIMDJSON_COMPETITION)
+  message(STATUS "Using SIMDJSON_GOOGLE_BENCHMARKS")
+endif()
 
 set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/tools/cmake")
 
@@ -72,9 +75,19 @@ set(THREADS_PREFER_PTHREAD_FLAG ON)
 #  set(CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE)
 #endif()
 
-
+option(SIMDJSON_VISUAL_STUDIO_BUILD_WITH_DEBUG_INFO_FOR_PROFILING "Under Visual Studio, add Zi to the compile flag and DEBUG to the link file to add debugging information to the release build for easier profiling inside tools like VTune" OFF)
 if(MSVC)
-  target_compile_options(simdjson-internal-flags INTERFACE /WX /W3 /sdl)
+if("${MSVC_TOOLSET_VERSION}" STREQUAL "140")
+  # Visual Studio 2015 issues warnings and we tolerate it,  cmake -G"Visual Studio 14" ..
+  target_compile_options(simdjson-internal-flags INTERFACE /W0 /sdl)
+else()
+  # Recent version of Visual Studio expected (2017, 2019...). Prior versions are unsupported.
+  target_compile_options(simdjson-internal-flags INTERFACE /WX /W3 /sdl /w34714) # https://docs.microsoft.com/en-us/cpp/error-messages/compiler-warnings/compiler-warning-level-4-c4714?view=vs-2019
+endif()
+if(SIMDJSON_VISUAL_STUDIO_BUILD_WITH_DEBUG_INFO_FOR_PROFILING)
+  target_link_options(simdjson-flags  INTERFACE    /DEBUG ) 
+  target_compile_options(simdjson-flags INTERFACE  /Zi)
+endif()
 else()
   target_compile_options(simdjson-internal-flags INTERFACE -fPIC)
   target_compile_options(simdjson-internal-flags INTERFACE -Werror -Wall -Wextra -Weffc++)
@@ -99,6 +112,10 @@ if(NOT SIMDJSON_IMPLEMENTATION_FALLBACK)
   target_compile_definitions(simdjson-internal-flags INTERFACE SIMDJSON_IMPLEMENTATION_FALLBACK=0)
 endif()
 
+option(SIMDJSON_BASH "Allow usage of bash within CMake" ON)
+
+option(SIMDJSON_GIT "Allow usage of git within CMake" ON)
+
 option(SIMDJSON_EXCEPTIONS "Enable simdjson's exception-throwing interface" ON)
 if(NOT SIMDJSON_EXCEPTIONS)
   message(STATUS "simdjson exception interface turned off. Code that does not check error codes will not compile.")
@@ -116,17 +133,6 @@ if(SIMDJSON_ENABLE_THREADS)
   target_compile_definitions(simdjson-flags INTERFACE SIMDJSON_THREADS_ENABLED=1) # This will be set in the code automatically.
 endif()
 
-# Some users compile simdjson with thread support but still do not want simdjson to use threads.
-#
-#  Important : Expect this option to disappear in the future.
-#
-option(SIMDJSON_DO_NOT_USE_THREADS_NO_MATTER_WHAT "Whether we enabled thread support or not (SIMDJSON_ENABLE_THREADS), do not use threads.\
- This option does nothing when thread support is not enabled. We reserve the right to remove this option in a future release in\
- favor of a runtime approach." OFF)
-if(SIMDJSON_DO_NOT_USE_THREADS_NO_MATTER_WHAT)
-  target_compile_definitions(simdjson-flags INTERFACE SIMDJSON_DO_NOT_USE_THREADS_NO_MATTER_WHAT=1)
-endif()
-
 if(SIMDJSON_USE_LIBCPP)
   target_link_libraries(simdjson-flags INTERFACE -stdlib=libc++ -lc++abi)
   # instead of the above line, we could have used

dependencies/CMakeLists.txt

@@ -2,7 +2,7 @@
 
 
 find_package(Git QUIET) # We want the library to build even if git is missing
-if ((Git_FOUND) AND (SIMDJSON_IS_UNDER_GIT))
+if ((Git_FOUND) AND SIMDJSON_GIT AND (SIMDJSON_IS_UNDER_GIT))
   message(STATUS "Git is available.")
   # Does NOT attempt to update or otherwise modify git submodules that are already initialized.
   function(initialize_submodule DIRECTORY)
@@ -18,6 +18,7 @@ if ((Git_FOUND) AND (SIMDJSON_IS_UNDER_GIT))
   endfunction(initialize_submodule)
 
   if (SIMDJSON_GOOGLE_BENCHMARKS)
+    message (STATUS "'SIMDJSON_GOOGLE_BENCHMARKS' is requested, configuring..." )
     option(BENCHMARK_ENABLE_TESTING OFF)
     set(BENCHMARK_ENABLE_TESTING OFF)
     option(BENCHMARK_ENABLE_INSTALL OFF)

doc/basics.md

@@ -8,8 +8,8 @@ An overview of what you need to know to use simdjson, with examples.
 * [Using simdjson as a CMake dependency](#using-simdjson-as-a-cmake-dependency)
 * [The Basics: Loading and Parsing JSON Documents](#the-basics-loading-and-parsing-json-documents)
 * [Using the Parsed JSON](#using-the-parsed-json)
-* [C++11 Support and string_view](#c++11-support-and-string_view)
-* [C++17 Support](#c++17-support)
+* [C++11 Support and string_view](#c11-support-and-string_view)
+* [C++17 Support](#c17-support)
 * [Minifying JSON strings without parsing](#minifying-json-strings-without-parsing)
 * [UTF-8 validation (alone)](#utf-8-validation-alone)
 * [JSON Pointer](#json-pointer)
@@ -19,13 +19,14 @@ An overview of what you need to know to use simdjson, with examples.
 * [Tree Walking and JSON Element Types](#tree-walking-and-json-element-types)
 * [Newline-Delimited JSON (ndjson) and JSON lines](#newline-delimited-json-ndjson-and-json-lines)
 * [Thread Safety](#thread-safety)
+* [Standard Compliance](#standard-compliance)
 
 
 Requirements
 ------------------
 
-- A recent compiler (LLVM clang6 or better, GNU GCC 7 or better) on a 64-bit (ARM or x64 Intel/AMD) POSIX systems such as macOS, freeBSD or Linux. We require that the compiler supports the C++11 standard or better.
-- Visual Studio 2017 or better under 64-bit Windows. Users should target a 64-bit build (x64) instead of a 32-bit build (x86). We support the LLVM clang compiler under Visual Studio (clangcl) as well as as the regular Visual Studio compiler.
+- A recent compiler (LLVM clang6 or better, GNU GCC 7.4 or better) on a 64-bit (ARM or x64 Intel/AMD) POSIX systems such as macOS, freeBSD or Linux. We require that the compiler supports the C++11 standard or better.
+- Visual Studio 2017 or better under 64-bit Windows. Users should target a 64-bit build (x64) instead of a 32-bit build (x86). We support the LLVM clang compiler under Visual Studio (clangcl) as well as as the regular Visual Studio compiler. We also support MinGW 64-bit under Windows.
 
 Including simdjson
 ------------------
@@ -45,28 +46,51 @@ c++ myproject.cpp simdjson.cpp
 ```
 
 Note:
-- Users on macOS and other platforms were default compilers do not provide C++11 compliant by default should request it with the appropriate flag (e.g., `c++ myproject.cpp simdjson.cpp`).
+- Users on macOS and other platforms were default compilers do not provide C++11 compliant by default should request it with the appropriate flag (e.g., `c++ -std=c++17 myproject.cpp simdjson.cpp`).
 - Visual Studio users should compile with the `_CRT_SECURE_NO_WARNINGS` flag to avoid warnings with respect to our use of standard C functions such as `fopen`.
 
+
+Using simdjson with package managers
+------------------
+
+You can install the simdjson library on your system or in your project using multiple package managers such as  MSYS2, the conan package manager, vcpkg, brew, the apt package manager (debian-based Linux systems), the FreeBSD package manager (FreeBSD), and so on. [Visit our wiki for more details](https://github.com/simdjson/simdjson/wiki/Installing-simdjson-with-a-package-manager).
+
 Using simdjson as a CMake dependency
 ------------------
 
-You can include the  simdjson repository as a folder in your CMake project. In the parent
-`CMakeLists.txt`, include the following lines:
+You can include the  simdjson as a CMake dependency by including the following lines in your `CMakeLists.txt`:
 
+```cmake
+include(FetchContent)
+
+FetchContent_Declare(
+  simdjson
+  GIT_REPOSITORY https://github.com/simdjson/simdjson.git
+  GIT_TAG  v0.5.0
+  GIT_SHALLOW TRUE)
+
+set(SIMDJSON_JUST_LIBRARY ON CACHE INTERNAL "")
+set(SIMDJSON_BUILD_STATIC ON CACHE INTERNAL "")
+
+FetchContent_MakeAvailable(simdjson)
 ```
-set(SIMDJSON_JUST_LIBRARY ON CACHE STRING "Build just the library, nothing else." FORCE)
-add_subdirectory(simdjson EXCLUDE_FROM_ALL)
-```
+
+You should replace `GIT_TAG  v0.5.0` by the version you need. If you omit `GIT_TAG  v0.5.0`, you will work from the main branch of simdjson: we recommend that if you are working on production code, 
 
 Elsewhere in your project, you can  declare dependencies on simdjson with lines such as these:
 
-```
+```cmake
 add_executable(myprogram myprogram.cpp)
 target_link_libraries(myprogram simdjson)
 ```
 
-See [our CMake demonstration](https://github.com/simdjson/cmakedemo).
+We recommend CMake version 3.15 or better.
+
+See [our CMake demonstration](https://github.com/simdjson/cmake_demo_single_file). It works under Linux, FreeBSD, macOS and Windows (including Visual Studio).
+
+
+
+The CMake build in simdjson can be taylored with a few variables. You can see the available variables and their default values by entering the `cmake -LA` command.
 
 The Basics: Loading and Parsing JSON Documents
 ----------------------------------------------
@@ -238,7 +262,7 @@ padded_string json = R"(  { "foo": 1, "bar": 2 }  )"_padded;
 dom::parser parser;
 dom::object object;
 auto error = parser.parse(json).get(object);
-if (!error) { cerr << error << endl; return; }
+if (error) { cerr << error << endl; return; }
 for (dom::key_value_pair field : object) {
   cout << field.key << " = " << field.value << endl;
 }
@@ -284,7 +308,7 @@ JSON Pointer
 ------------
 
 The simdjson library also supports [JSON pointer](https://tools.ietf.org/html/rfc6901) through the
-at() method, letting you reach further down into the document in a single call:
+`at_pointer()` method, letting you reach further down into the document in a single call:
 
 ```c++
 auto cars_json = R"( [
@@ -294,9 +318,39 @@ auto cars_json = R"( [
 ] )"_padded;
 dom::parser parser;
 dom::element cars = parser.parse(cars_json);
-cout << cars.at("0/tire_pressure/1") << endl; // Prints 39.9
+cout << cars.at_pointer("/0/tire_pressure/1") << endl; // Prints 39.9
 ```
 
+A JSON Path is a sequence of segments each starting with the '/' character. Within arrays, an integer
+index allows you to select the indexed node. Within objects, the string value of the key allows you to
+select the value. If your keys contain the characters '/' or '~', they must be escaped as '~1' and
+'~0' respectively. An empty JSON Path refers to the whole document.
+
+We also extend the JSON Pointer support to include *relative* paths.  
+You can apply a JSON path to any node and the path gets interpreted relatively, as if the currrent node were a whole JSON document.
+
+Consider the following example: 
+
+```c++
+auto cars_json = R"( [
+  { "make": "Toyota", "model": "Camry",  "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
+  { "make": "Kia",    "model": "Soul",   "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
+  { "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
+] )"_padded;
+dom::parser parser;
+dom::element cars = parser.parse(cars_json);
+cout << cars.at_pointer("/0/tire_pressure/1") << endl; // Prints 39.9
+for (dom::element car_element : cars) {
+    dom::object car;
+    simdjson::error_code error;
+    if ((error = car_element.get(car))) { std::cerr << error << std::endl; return; }
+    double x = car.at_pointer("/tire_pressure/1");
+    cout << x << endl; // Prints 39.9, 31 and 30
+}
+```
+
+
+
 Error Handling
 --------------
 
@@ -566,6 +620,25 @@ The parsed results (`dom::document`, `dom::element`, `array`, `object`) depend o
 The CPU detection, which runs the first time parsing is attempted and switches to the fastest
 parser for your CPU, is transparent and thread-safe.
 
+
+Standard Compliance
+--------------------
+
+The simdjson library is fully compliant with  the [RFC 8259](https://www.tbray.org/ongoing/When/201x/2017/12/14/rfc8259.html) JSON specification.
+
+- The only insignificant whitespace characters allowed are the space, the horizontal tab, the line feed and the carriage return. In particular, a JSON document may not contain an unespaced null character.
+- A single string or a single number is considered to be a valid JSON document.
+- We fully validate the numbers according to the JSON specification. For example,  the string `01` is not valid JSON document since the specification states that *leading zeros are not allowed*.
+- The specification allows implementations to set limits on the range and precision of numbers accepted.  We support 64-bit floating-point numbers as well as integer values. 
+  - We parse integers and floating-point numbers as separate types which allows us to support all signed (two complement's) 64-bit integers, like a Java `long` or a C/C++ `long long` and all 64-bit unsigned integers. When we cannot represent exactly an integer as a signed or unsigned 64-bit value, we reject the JSON document. 
+  - We support the full range of 64-bit floating-point numbers (binary64). The values range from `std::numeric_limits<double>::lowest()`  to `std::numeric_limits<double>::max()`, so from -1.7976e308 all the way to 1.7975e308. Extreme values (less or equal to -1e308, greater or equal to 1e308) are rejected: we refuse to parse the input document. Numbers are parsed with with a perfect accuracy (ULP 0): the nearest floating-point value is chosen, rounding to even when needed. If you serialized your floating-point numbers with 17 significant digits in a standard compliant manner, the simdjson library is guaranteed to recovere the example same numbers, exactly.
+- The specification states that JSON text exchanged between systems that are not part of a closed ecosystem MUST be encoded using UTF-8. The simdjson library does full UTF-8 validation as part of the parsing. The specification states that implementations MUST NOT add a byte order mark: the simdjson library rejects documents starting with a  byte order mark.
+- The simdjson library validates string content for unescaped characters. Unescaped line breaks and tabs in strings are not allowed.
+- The simdjson library accepts objects with repeated keys: all of the name/value pairs, including duplicates, are reported. We do not enforce key uniqueness.
+- The specification states that an implementation may set limits on the size of texts that it accepts. The simdjson library limits single JSON documents to 4 GiB. It will refuse to parse a JSON document larger than 4294967295 bytes. (This limitation does not apply to streams of JSON documents, only to single JSON documents.)
+- The specification states that an implementation may set limits on the maximum depth of nesting. By default, the simdjson will refuse to parse documents with a depth exceeding 1024.
+
+
 Backwards Compatibility
 -----------------------
 

doc/basics_doxygen.md

@@ -28,20 +28,36 @@ c++ myproject.cpp simdjson.cpp
 ```
 
 Note:
-- Users on macOS and other platforms were default compilers do not provide C++11 compliant by default should request it with the appropriate flag (e.g., `c++ myproject.cpp simdjson.cpp`).
+- Users on macOS and other platforms were default compilers do not provide C++11 compliant by default should request it with the appropriate flag (e.g., `c++  -std=c++17 myproject.cpp simdjson.cpp`).
 - Visual Studio users should compile with the `_CRT_SECURE_NO_WARNINGS` flag to avoid warnings with respect to our use of standard C functions such as `fopen`.
 
+Using simdjson with package managers
+------------------
+
+You can install the simdjson library on your system or in your project using multiple package managers such as  MSYS2, the conan package manager, vcpkg, brew, the apt package manager (debian-based Linux systems), the FreeBSD package manager (FreeBSD), and so on. [Visit our wiki for more details](https://github.com/simdjson/simdjson/wiki/Installing-simdjson-with-a-package-manager).
+
 Using simdjson as a CMake dependency
 ------------------
 
-You can include the  simdjson repository as a folder in your CMake project. In the parent
-`CMakeLists.txt`, include the following lines:
+You can include the  simdjson as a CMake dependency by including the following lines in your `CMakeLists.txt`:
 
 ```
-set(SIMDJSON_JUST_LIBRARY ON CACHE STRING "Build just the library, nothing else." FORCE)
-add_subdirectory(simdjson EXCLUDE_FROM_ALL)
+include(FetchContent)
+
+FetchContent_Declare(
+  simdjson
+  GIT_REPOSITORY https://github.com/simdjson/simdjson.git
+  GIT_TAG  v0.4.7
+  GIT_SHALLOW TRUE)
+
+set(SIMDJSON_JUST_LIBRARY ON CACHE INTERNAL "")
+set(SIMDJSON_BUILD_STATIC ON CACHE INTERNAL "")
+
+FetchContent_MakeAvailable(simdjson)
 ```
 
+You should replace `GIT_TAG  v0.5.0` by the version you need. If you omit `GIT_TAG  v0.5.0`, you will work from the main branch of simdjson: we recommend that if you are working on production code, 
+
 Elsewhere in your project, you can  declare dependencies on simdjson with lines such as these:
 
 ```
@@ -49,7 +65,13 @@ add_executable(myprogram myprogram.cpp)
 target_link_libraries(myprogram simdjson)
 ```
 
-See [our CMake demonstration](https://github.com/simdjson/cmakedemo).
+We recommend CMake version 3.15 or better.
+
+See [our CMake demonstration](https://github.com/simdjson/cmake_demo_single_file). It works under Linux, FreeBSD, macOS and Windows (including Visual Studio).
+
+The CMake build in simdjson can be taylored with a few variables. You can see the available variables and their default values by entering the `cmake -LA` command.
+
+
 
 The Basics: Loading and Parsing JSON Documents
 ----------------------------------------------
@@ -267,9 +289,9 @@ JSON Pointer
 ------------
 
 The simdjson library also supports [JSON pointer](https://tools.ietf.org/html/rfc6901) through the
-at() method, letting you reach further down into the document in a single call:
+`at_pointer()` method, letting you reach further down into the document in a single call:
 
-```
+```c++
 auto cars_json = R"( [
   { "make": "Toyota", "model": "Camry",  "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
   { "make": "Kia",    "model": "Soul",   "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
@@ -277,9 +299,38 @@ auto cars_json = R"( [
 ] )"_padded;
 dom::parser parser;
 dom::element cars = parser.parse(cars_json);
-cout << cars.at("0/tire_pressure/1") << endl; // Prints 39.9
+cout << cars.at_pointer("/0/tire_pressure/1") << endl; // Prints 39.9
 ```
 
+A JSON Path is a sequence of segments each starting with the '/' character. Within arrays, an integer
+index allows you to select the indexed node. Within objects, the string value of the key allows you to
+select the value. If your keys contain the characters '/' or '~', they must be escaped as '~1' and
+'~0' respectively. An empty JSON Path refers to the whole document.
+
+We also extend the JSON Pointer support to include *relative* paths.  
+You can apply a JSON path to any node and the path gets interpreted relatively, as if the currrent node were a whole JSON document.
+
+Consider the following example: 
+
+```c++
+auto cars_json = R"( [
+  { "make": "Toyota", "model": "Camry",  "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
+  { "make": "Kia",    "model": "Soul",   "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
+  { "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
+] )"_padded;
+dom::parser parser;
+dom::element cars = parser.parse(cars_json);
+cout << cars.at_pointer("/0/tire_pressure/1") << endl; // Prints 39.9
+for (dom::element car_element : cars) {
+    dom::object car;
+    simdjson::error_code error;
+    if ((error = car_element.get(car))) { std::cerr << error << std::endl; return; }
+    double x = car.at_pointer("/tire_pressure/1");
+    cout << x << endl; // Prints 39.9, 31 and 30
+}
+```
+
+
 Error Handling
 --------------
 

doc/implementation-selection.md

@@ -47,7 +47,7 @@ Inspecting the Detected Implementation
 You can check what implementation is running with `active_implementation`:
 
 ```c++
-cout << "simdjson v" << #SIMDJSON_VERSION << endl;
+cout << "simdjson v" << STRINGIFY(SIMDJSON_VERSION) << endl;
 cout << "Detected the best implementation for your machine: " << simdjson::active_implementation->name();
 cout << "(" << simdjson::active_implementation->description() << ")" << endl;
 ```

doc/performance.md

@@ -8,11 +8,10 @@ are still some scenarios where tuning can enhance performance.
   * [Keeping documents around for longer](#keeping-documents-around-for-longer)
 * [Server Loops: Long-Running Processes and Memory Capacity](#server-loops-long-running-processes-and-memory-capacity)
 * [Large files and huge page support](#large-files-and-huge-page-support)
-* [Computed GOTOs](#computed-gotos)
 * [Number parsing](#number-parsing)
 * [Visual Studio](#visual-studio)
 * [Downclocking](#downclocking)
-
+* [Best Use of the DOM API](#best-use-of-the-dom-api)
 
 Reusing the parser for maximum efficiency
 -----------------------------------------
@@ -122,18 +121,6 @@ use the `-H` flag to omit the memory allocation cost from the benchmark results.
 ./parse -H largefile # without memory allocation
 ```
 
-Computed GOTOs
---------------
-
-For best performance, we use a technique called "computed goto" when the compiler supports it, it is
-also sometimes described as "Labels as Values". Though it is not part of the C++ standard, it is
-supported by many major compilers and it brings measurable performance benefits that are difficult
-to achieve otherwise. The computed gotos are  automatically disabled under Visual Studio.
-
-If you wish to forcefully disable computed gotos, you can do so by compiling the code with
-`-DSIMDJSON_NO_COMPUTED_GOTO=1`. It is not recommended to disable computed gotos if your compiler
-supports it. In fact, you should almost never need to be concerned with computed gotos.
-
 Number parsing
 --------------
 
@@ -154,19 +141,36 @@ Visual Studio
 
 On Intel and AMD Windows platforms, Microsoft Visual Studio enables programmers to build either 32-bit (x86) or 64-bit (x64) binaries. We urge you to always use 64-bit mode. Visual Studio 2019 should default on 64-bit builds when you have a 64-bit version of Windows, which we recommend.
 
-We do not recommend that you compile simdjson with architecture-specific flags such as  `arch:AVX2`. The simdjson library automatically selects the best execution kernel at runtime.
+When compiling with Visual Studio, we recommend the flags `/Ob2 /O2` or better. We do not recommend that you compile simdjson with architecture-specific flags such as  `arch:AVX2`. The simdjson library automatically selects the best execution kernel at runtime.
 
-Recent versions of Microsoft Visual Studio on Windows provides support for the LLVM Clang compiler. You  only need to install the "Clang compiler" optional component. You may also get a copy of the 64-bit LLVM CLang compiler for [Windows directly from LLVM](https://releases.llvm.org/download.html). The simdjson library fully supports the LLVM Clang compiler under Windows. In fact, you may get better performance out of simdjson with the LLVM Clang compiler than with the regular Visual Studio compiler.
+Recent versions of Microsoft Visual Studio on Windows provides support for the LLVM Clang compiler. You  only need to install the "Clang compiler" optional component (ClangCL). You may also get a copy of the 64-bit LLVM CLang compiler for [Windows directly from LLVM](https://releases.llvm.org/download.html). The simdjson library fully supports the LLVM Clang compiler under Windows. In fact, you may get better performance out of simdjson with the LLVM Clang compiler than with the regular Visual Studio compiler. Meanwhile the [LLVM CLang compiler is binary compatible with Visual Studio](https://clang.llvm.org/docs/MSVCCompatibility.html) which means that you can combine their binaries (executables and libraries).
+
+Under Windows, we also support the GNU GCC compiler via MSYS2. The performance of 64-bit MSYS2 under Windows excellent (on par with Linux).
 
 
 Downclocking
 --------------
 
-You should not expect the simdjson library to cause downclocking of your recent Intel CPU cores.
+
+
+SIMD instructions are the public transportation of computing. Instead of using 4 distinct instructions to add numbers, you can replace them with a single instruction that does the same work. Though the one instruction is slightly more expensive, the energy used per unit of work is much less with SIMD. If you can increase your speed using SIMD instructions (NEON, SSE, AVX), you should expect to reduce your power usage.
+
+The SIMD instructions that simdjson relies upon (SSE and AVX under x64, NEON under ARM) are routinely part of runtime libraries (e.g., [Go](https://golang.org/src/runtime/memmove_amd64.s), [Glibc](https://github.com/ihtsae/glibc/commit/5f3d0b78e011d2a72f9e88b0e9ef5bc081d18f97), [LLVM](https://github.com/llvm/llvm-project/blob/96f3ea0d21b48ca088355db10d4d1a2e9bc9f884/lldb/tools/debugserver/source/MacOSX/i386/DNBArchImplI386.cpp), [Rust](https://github.com/rust-lang/rust/commit/070fad1701fb36b112853b0a6a9787a7bb7ff34c), [Java](http://hg.openjdk.java.net/jdk8u/jdk8u/hotspot/file/c1374141598c/src/cpu/x86/vm/stubGenerator_x86_64.cpp#l1297), [PHP](https://github.com/php/php-src/blob/e5cb53ec68603d4dbdd780fd3ecfca943b4fd383/ext/standard/string.c)). What distinguishes the simdjson library is that it is built from the ground up to benefit from these instructions. 
+
+
+You should not expect the simdjson library to cause *downclocking* of your recent Intel CPU cores.
 
 On some Intel processors, using SIMD instructions in a sustained manner on the same CPU core may result in a phenomenon called downclocking whereas the processor initially runs these instructions at a slow speed before reducing the frequency of the core for a short time (milliseconds). Intel refers to these states as licenses. On some current Intel processors, it occurs under two scenarios:
 
 - [Whenever 512-bit AVX-512 instructions are used](https://lemire.me/blog/2018/09/07/avx-512-when-and-how-to-use-these-new-instructions/).
 - Whenever heavy 256-bit or wider instructions are used. Heavy instructions are those involving floating point operations or integer multiplications (since these execute on the floating point unit).
 
-The simdjson library does not currently support AVX-512 instructions and it does not make use of heavy 256-bit instructions. Thus there should be no downclocking due to simdjson on recent processors. You may still be worried about which SIMD instruction set is used by simdjson.  Thankfully,  [you can always determine and change which architecture-specific implementation is used](implementation-selection.md). Thus even if your CPU supports AVX2, you do not need to use AVX2. You are in control.
+The simdjson library does not currently support AVX-512 instructions and it does not make use of heavy 256-bit instructions. We do use vectorized multiplications, but only using 128-bit registers. Thus there should be no downclocking due to simdjson on recent processors. 
+
+You may still be worried about which SIMD instruction set is used by simdjson.  Thankfully,  [you can always determine and change which architecture-specific implementation is used](implementation-selection.md) by simdjson. Thus even if your CPU supports AVX2, you do not need to use AVX2. You are in control.
+
+Best Use of the DOM API
+-------------------------
+
+The simdjson API provides access to the JSON DOM (document-object-model) content as a tree of `dom::element` instances, each representing an object, an array or an atomic type (null, true, false, number). These `dom::element` instances are lightweight objects (e.g., spanning 16 bytes) and it might be advantageous to pass them by value, as opposed to passing them by reference or by pointer.
+

fuzz/fuzz_dump_raw_tape.cpp

@@ -13,6 +13,6 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
     if (error) { return 1; }
 
     NulOStream os;
-    UNUSED auto dumpstatus = elem.dump_raw_tape(os);
+    SIMDJSON_UNUSED auto dumpstatus = elem.dump_raw_tape(os);
     return 0;
 }

fuzz/fuzz_parser.cpp

@@ -4,7 +4,7 @@
 #include <string>
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
   simdjson::dom::parser parser;
-  UNUSED simdjson::dom::element elem;
-  UNUSED auto error = parser.parse(Data, Size).get(elem);
+  SIMDJSON_UNUSED simdjson::dom::element elem;
+  SIMDJSON_UNUSED auto error = parser.parse(Data, Size).get(elem);
   return 0;
 }

include/simdjson.h

@@ -5,6 +5,33 @@
  * @mainpage
  *
  * Check the [README.md](https://github.com/lemire/simdjson/blob/master/README.md#simdjson--parsing-gigabytes-of-json-per-second).
+ * 
+ * Sample code. See https://github.com/simdjson/simdjson/blob/master/doc/basics.md for more examples.
+
+    #include "simdjson.h"
+
+    int main(void) {
+      // load from `twitter.json` file:
+      simdjson::dom::parser parser;
+      simdjson::dom::element tweets = parser.load("twitter.json");
+      std::cout << tweets["search_metadata"]["count"] << " results." << std::endl;
+
+      // Parse and iterate through an array of objects
+      auto abstract_json = R"( [
+        {  "12345" : {"a":12.34, "b":56.78, "c": 9998877}   },
+        {  "12545" : {"a":11.44, "b":12.78, "c": 11111111}  }
+        ] )"_padded;
+  
+      for (simdjson::dom::object obj : parser.parse(abstract_json)) {
+        for(const auto& key_value : obj) {
+          cout << "key: " << key_value.key << " : ";
+          simdjson::dom::object innerobj = key_value.value;
+          cout << "a: " << double(innerobj["a"]) << ", ";
+          cout << "b: " << double(innerobj["b"]) << ", ";
+          cout << "c: " << int64_t(innerobj["c"]) << endl;
+        }
+      }
+    }
  */
 
 #include "simdjson/compiler_check.h"
@@ -31,16 +58,16 @@ SIMDJSON_DISABLE_UNDESIRED_WARNINGS
 #include "simdjson/dom/parsedjson_iterator.h"
 
 // Inline functions
-#include "simdjson/inline/array.h"
-#include "simdjson/inline/document_stream.h"
-#include "simdjson/inline/document.h"
-#include "simdjson/inline/element.h"
-#include "simdjson/inline/error.h"
-#include "simdjson/inline/object.h"
-#include "simdjson/inline/padded_string.h"
-#include "simdjson/inline/parsedjson_iterator.h"
-#include "simdjson/inline/parser.h"
-#include "simdjson/inline/tape_ref.h"
+#include "simdjson/dom/array-inl.h"
+#include "simdjson/dom/document_stream-inl.h"
+#include "simdjson/dom/document-inl.h"
+#include "simdjson/dom/element-inl.h"
+#include "simdjson/error-inl.h"
+#include "simdjson/dom/object-inl.h"
+#include "simdjson/padded_string-inl.h"
+#include "simdjson/dom/parsedjson_iterator-inl.h"
+#include "simdjson/dom/parser-inl.h"
+#include "simdjson/internal/tape_ref-inl.h"
 
 SIMDJSON_POP_DISABLE_WARNINGS
 

include/simdjson/common_defs.h

@@ -38,55 +38,49 @@ constexpr size_t DEFAULT_MAX_DEPTH = 1024;
 
 #if defined(__GNUC__)
   // Marks a block with a name so that MCA analysis can see it.
-  #define BEGIN_DEBUG_BLOCK(name) __asm volatile("# LLVM-MCA-BEGIN " #name);
-  #define END_DEBUG_BLOCK(name) __asm volatile("# LLVM-MCA-END " #name);
-  #define DEBUG_BLOCK(name, block) BEGIN_DEBUG_BLOCK(name); block; END_DEBUG_BLOCK(name);
+  #define SIMDJSON_BEGIN_DEBUG_BLOCK(name) __asm volatile("# LLVM-MCA-BEGIN " #name);
+  #define SIMDJSON_END_DEBUG_BLOCK(name) __asm volatile("# LLVM-MCA-END " #name);
+  #define SIMDJSON_DEBUG_BLOCK(name, block) BEGIN_DEBUG_BLOCK(name); block; END_DEBUG_BLOCK(name);
 #else
-  #define BEGIN_DEBUG_BLOCK(name)
-  #define END_DEBUG_BLOCK(name)
-  #define DEBUG_BLOCK(name, block)
-#endif
-
-#if !defined(SIMDJSON_REGULAR_VISUAL_STUDIO) && !defined(SIMDJSON_NO_COMPUTED_GOTO)
-  // We assume here that *only* regular visual studio
-  // does not support computed gotos.
-  // Implemented using Labels as Values which works in GCC and CLANG (and maybe
-  // also in Intel's compiler), but won't work in MSVC.
-  // Compute gotos are good for performance, enable them if you can.
-  #define SIMDJSON_USE_COMPUTED_GOTO
+  #define SIMDJSON_BEGIN_DEBUG_BLOCK(name)
+  #define SIMDJSON_END_DEBUG_BLOCK(name)
+  #define SIMDJSON_DEBUG_BLOCK(name, block)
 #endif
 
 // Align to N-byte boundary
-#define ROUNDUP_N(a, n) (((a) + ((n)-1)) & ~((n)-1))
-#define ROUNDDOWN_N(a, n) ((a) & ~((n)-1))
+#define SIMDJSON_ROUNDUP_N(a, n) (((a) + ((n)-1)) & ~((n)-1))
+#define SIMDJSON_ROUNDDOWN_N(a, n) ((a) & ~((n)-1))
 
-#define ISALIGNED_N(ptr, n) (((uintptr_t)(ptr) & ((n)-1)) == 0)
+#define SIMDJSON_ISALIGNED_N(ptr, n) (((uintptr_t)(ptr) & ((n)-1)) == 0)
 
 #if defined(SIMDJSON_REGULAR_VISUAL_STUDIO)
 
-  #define really_inline __forceinline
-  #define never_inline __declspec(noinline)
+  #define simdjson_really_inline __forceinline
+  #define simdjson_never_inline __declspec(noinline)
 
-  #define UNUSED
-  #define WARN_UNUSED
+  #define SIMDJSON_UNUSED
+  #define SIMDJSON_WARN_UNUSED
 
-  #ifndef likely
-  #define likely(x) x
+  #ifndef simdjson_likely
+  #define simdjson_likely(x) x
   #endif
-  #ifndef unlikely
-  #define unlikely(x) x
+  #ifndef simdjson_unlikely
+  #define simdjson_unlikely(x) x
   #endif
 
   #define SIMDJSON_PUSH_DISABLE_WARNINGS __pragma(warning( push ))
   #define SIMDJSON_PUSH_DISABLE_ALL_WARNINGS __pragma(warning( push, 0 ))
   #define SIMDJSON_DISABLE_VS_WARNING(WARNING_NUMBER) __pragma(warning( disable : WARNING_NUMBER ))
   // Get rid of Intellisense-only warnings (Code Analysis)
-  // Though __has_include is C++17, it looks like it is supported in Visual Studio 2017 or better.
-  // We are probably not supporting earlier version of Visual Studio in any case.
+  // Though __has_include is C++17, it is supported in Visual Studio 2017 or better (_MSC_VER>=1910).
+  #if defined(_MSC_VER) && (_MSC_VER>=1910) 
   #if __has_include(<CppCoreCheck\Warnings.h>)
   #include <CppCoreCheck\Warnings.h>
   #define SIMDJSON_DISABLE_UNDESIRED_WARNINGS SIMDJSON_DISABLE_VS_WARNING(ALL_CPPCORECHECK_WARNINGS)
-  #else
+  #endif
+  #endif
+
+  #ifndef SIMDJSON_DISABLE_UNDESIRED_WARNINGS
   #define SIMDJSON_DISABLE_UNDESIRED_WARNINGS
   #endif
 
@@ -95,17 +89,17 @@ constexpr size_t DEFAULT_MAX_DEPTH = 1024;
 
 #else // SIMDJSON_REGULAR_VISUAL_STUDIO
 
-  #define really_inline inline __attribute__((always_inline, unused))
-  #define never_inline inline __attribute__((noinline, unused))
+  #define simdjson_really_inline inline __attribute__((always_inline))
+  #define simdjson_never_inline inline __attribute__((noinline))
 
-  #define UNUSED __attribute__((unused))
-  #define WARN_UNUSED __attribute__((warn_unused_result))
+  #define SIMDJSON_UNUSED __attribute__((unused))
+  #define SIMDJSON_WARN_UNUSED __attribute__((warn_unused_result))
 
-  #ifndef likely
-  #define likely(x) __builtin_expect(!!(x), 1)
+  #ifndef simdjson_likely
+  #define simdjson_likely(x) __builtin_expect(!!(x), 1)
   #endif
-  #ifndef unlikely
-  #define unlikely(x) __builtin_expect(!!(x), 0)
+  #ifndef simdjson_unlikely
+  #define simdjson_unlikely(x) __builtin_expect(!!(x), 0)
   #endif
 
   #define SIMDJSON_PUSH_DISABLE_WARNINGS _Pragma("GCC diagnostic push")

include/simdjson/dom/array-inl.h

@@ -12,11 +12,11 @@ namespace simdjson {
 //
 // simdjson_result<dom::array> inline implementation
 //
-really_inline simdjson_result<dom::array>::simdjson_result() noexcept
+simdjson_really_inline simdjson_result<dom::array>::simdjson_result() noexcept
     : internal::simdjson_result_base<dom::array>() {}
-really_inline simdjson_result<dom::array>::simdjson_result(dom::array value) noexcept
+simdjson_really_inline simdjson_result<dom::array>::simdjson_result(dom::array value) noexcept
     : internal::simdjson_result_base<dom::array>(std::forward<dom::array>(value)) {}
-really_inline simdjson_result<dom::array>::simdjson_result(error_code error) noexcept
+simdjson_really_inline simdjson_result<dom::array>::simdjson_result(error_code error) noexcept
     : internal::simdjson_result_base<dom::array>(error) {}
 
 #if SIMDJSON_EXCEPTIONS
@@ -36,9 +36,9 @@ inline size_t simdjson_result<dom::array>::size() const noexcept(false) {
 
 #endif // SIMDJSON_EXCEPTIONS
 
-inline simdjson_result<dom::element> simdjson_result<dom::array>::at(const std::string_view &json_pointer) const noexcept {
+inline simdjson_result<dom::element> simdjson_result<dom::array>::at_pointer(std::string_view json_pointer) const noexcept {
   if (error()) { return error(); }
-  return first.at(json_pointer);
+  return first.at_pointer(json_pointer);
 }
 inline simdjson_result<dom::element> simdjson_result<dom::array>::at(size_t index) const noexcept {
   if (error()) { return error(); }
@@ -50,8 +50,8 @@ namespace dom {
 //
 // array inline implementation
 //
-really_inline array::array() noexcept : tape{} {}
-really_inline array::array(const internal::tape_ref &_tape) noexcept : tape{_tape} {}
+simdjson_really_inline array::array() noexcept : tape{} {}
+simdjson_really_inline array::array(const internal::tape_ref &_tape) noexcept : tape{_tape} {}
 inline array::iterator array::begin() const noexcept {
   return internal::tape_ref(tape.doc, tape.json_index + 1);
 }
@@ -61,7 +61,15 @@ inline array::iterator array::end() const noexcept {
 inline size_t array::size() const noexcept {
   return tape.scope_count();
 }
-inline simdjson_result<element> array::at(const std::string_view &json_pointer) const noexcept {
+inline simdjson_result<element> array::at_pointer(std::string_view json_pointer) const noexcept {
+  if(json_pointer[0] != '/') {
+    if(json_pointer.size() == 0) { // an empty string means that we return the current node
+      return element(this->tape); // copy the current node
+    } else { // otherwise there is an error
+      return INVALID_JSON_POINTER;
+    }
+  }
+  json_pointer = json_pointer.substr(1);
   // - means "the append position" or "the element after the end of the array"
   // We don't support this, because we're returning a real element, not a position.
   if (json_pointer == "-") { return INDEX_OUT_OF_BOUNDS; }
@@ -84,9 +92,13 @@ inline simdjson_result<element> array::at(const std::string_view &json_pointer)
 
   // Get the child
   auto child = array(tape).at(array_index);
+  // If there is an error, it ends here
+  if(child.error()) {
+    return child;
+  }
   // If there is a /, we're not done yet, call recursively.
   if (i < json_pointer.length()) {
-    child = child.at(json_pointer.substr(i+1));
+    child = child.at_pointer(json_pointer.substr(i));
   }
   return child;
 }
@@ -102,18 +114,37 @@ inline simdjson_result<element> array::at(size_t index) const noexcept {
 //
 // array::iterator inline implementation
 //
-really_inline array::iterator::iterator(const internal::tape_ref &_tape) noexcept : tape{_tape} { }
+simdjson_really_inline array::iterator::iterator(const internal::tape_ref &_tape) noexcept : tape{_tape} { }
 inline element array::iterator::operator*() const noexcept {
   return element(tape);
 }
-inline bool array::iterator::operator!=(const array::iterator& other) const noexcept {
-  return tape.json_index != other.tape.json_index;
-}
 inline array::iterator& array::iterator::operator++() noexcept {
   tape.json_index = tape.after_element();
   return *this;
 }
-
+inline array::iterator array::iterator::operator++(int) noexcept {
+  array::iterator out = *this;
+  ++*this;
+  return out;
+}
+inline bool array::iterator::operator!=(const array::iterator& other) const noexcept {
+  return tape.json_index != other.tape.json_index;
+}
+inline bool array::iterator::operator==(const array::iterator& other) const noexcept {
+  return tape.json_index == other.tape.json_index;
+}
+inline bool array::iterator::operator<(const array::iterator& other) const noexcept {
+  return tape.json_index < other.tape.json_index;
+}
+inline bool array::iterator::operator<=(const array::iterator& other) const noexcept {
+  return tape.json_index <= other.tape.json_index;
+}
+inline bool array::iterator::operator>=(const array::iterator& other) const noexcept {
+  return tape.json_index >= other.tape.json_index;
+}
+inline bool array::iterator::operator>(const array::iterator& other) const noexcept {
+  return tape.json_index > other.tape.json_index;
+}
 inline std::ostream& operator<<(std::ostream& out, const array &value) {
   return out << minify<array>(value);
 }
@@ -150,4 +181,15 @@ inline std::ostream& operator<<(std::ostream& out, const simdjson_result<dom::ar
 
 } // namespace simdjson
 
+#include "simdjson/dom/element-inl.h"
+
+#if defined(__cpp_lib_ranges)
+static_assert(std::ranges::view<simdjson::dom::array>);
+static_assert(std::ranges::sized_range<simdjson::dom::array>);
+#if SIMDJSON_EXCEPTIONS
+static_assert(std::ranges::view<simdjson::simdjson_result<simdjson::dom::array>>);
+static_assert(std::ranges::sized_range<simdjson::simdjson_result<simdjson::dom::array>>);
+#endif // SIMDJSON_EXCEPTIONS
+#endif // defined(__cpp_lib_ranges)
+
 #endif // SIMDJSON_INLINE_ARRAY_H

include/simdjson/dom/array.h

@@ -19,14 +19,17 @@ class element;
 class array {
 public:
   /** Create a new, invalid array */
-  really_inline array() noexcept;
+  simdjson_really_inline array() noexcept;
 
   class iterator {
   public:
+    using value_type = element;
+    using difference_type = std::ptrdiff_t;
+
     /**
      * Get the actual value
      */
-    inline element operator*() const noexcept;
+    inline value_type operator*() const noexcept;
     /**
      * Get the next value.
      *
@@ -34,14 +37,30 @@ public:
      *
      */
     inline iterator& operator++() noexcept;
+    /**
+     * Get the next value.
+     *
+     * Part of the  std::iterator interface.
+     */
+    inline iterator operator++(int) noexcept;
     /**
      * Check if these values come from the same place in the JSON.
      *
      * Part of the std::iterator interface.
      */
     inline bool operator!=(const iterator& other) const noexcept;
+    inline bool operator==(const iterator& other) const noexcept;
+
+    inline bool operator<(const iterator& other) const noexcept;
+    inline bool operator<=(const iterator& other) const noexcept;
+    inline bool operator>=(const iterator& other) const noexcept;
+    inline bool operator>(const iterator& other) const noexcept;
+
+    iterator() noexcept = default;
+    iterator(const iterator&) noexcept = default;
+    iterator& operator=(const iterator&) noexcept = default;
   private:
-    really_inline iterator(const internal::tape_ref &tape) noexcept;
+    simdjson_really_inline iterator(const internal::tape_ref &tape) noexcept;
     internal::tape_ref tape;
     friend class array;
   };
@@ -65,12 +84,14 @@ public:
    */
   inline size_t size() const noexcept;
   /**
-   * Get the value associated with the given JSON pointer.
+   * Get the value associated with the given JSON pointer.  We use the RFC 6901
+   * https://tools.ietf.org/html/rfc6901 standard, interpreting the current node
+   * as the root of its own JSON document.
    *
    *   dom::parser parser;
    *   array a = parser.parse(R"([ { "foo": { "a": [ 10, 20, 30 ] }} ])"_padded);
-   *   a.at("0/foo/a/1") == 20
-   *   a.at("0")["foo"]["a"].at(1) == 20
+   *   a.at_pointer("/0/foo/a/1") == 20
+   *   a.at_pointer("0")["foo"]["a"].at(1) == 20
    *
    * @return The value associated with the given JSON pointer, or:
    *         - NO_SUCH_FIELD if a field does not exist in an object
@@ -78,7 +99,7 @@ public:
    *         - INCORRECT_TYPE if a non-integer is used to access an array
    *         - INVALID_JSON_POINTER if the JSON pointer is invalid and cannot be parsed
    */
-  inline simdjson_result<element> at(const std::string_view &json_pointer) const noexcept;
+  inline simdjson_result<element> at_pointer(std::string_view json_pointer) const noexcept;
 
   /**
    * Get the value at the given index. This function has linear-time complexity and
@@ -99,7 +120,7 @@ public:
   inline simdjson_result<element> at(size_t index) const noexcept;
 
 private:
-  really_inline array(const internal::tape_ref &tape) noexcept;
+  simdjson_really_inline array(const internal::tape_ref &tape) noexcept;
   internal::tape_ref tape;
   friend class element;
   friend struct simdjson_result<element>;
@@ -124,11 +145,11 @@ inline std::ostream& operator<<(std::ostream& out, const array &value);
 template<>
 struct simdjson_result<dom::array> : public internal::simdjson_result_base<dom::array> {
 public:
-  really_inline simdjson_result() noexcept; ///< @private
-  really_inline simdjson_result(dom::array value) noexcept; ///< @private
-  really_inline simdjson_result(error_code error) noexcept; ///< @private
+  simdjson_really_inline simdjson_result() noexcept; ///< @private
+  simdjson_really_inline simdjson_result(dom::array value) noexcept; ///< @private
+  simdjson_really_inline simdjson_result(error_code error) noexcept; ///< @private
 
-  inline simdjson_result<dom::element> at(const std::string_view &json_pointer) const noexcept;
+  inline simdjson_result<dom::element> at_pointer(std::string_view json_pointer) const noexcept;
   inline simdjson_result<dom::element> at(size_t index) const noexcept;
 
 #if SIMDJSON_EXCEPTIONS
@@ -155,4 +176,19 @@ inline std::ostream& operator<<(std::ostream& out, const simdjson_result<dom::ar
 
 } // namespace simdjson
 
+#if defined(__cpp_lib_ranges)
+#include <ranges>
+
+namespace std {
+namespace ranges {
+template<>
+inline constexpr bool enable_view<simdjson::dom::array> = true;
+#if SIMDJSON_EXCEPTIONS
+template<>
+inline constexpr bool enable_view<simdjson::simdjson_result<simdjson::dom::array>> = true;
+#endif // SIMDJSON_EXCEPTIONS
+} // namespace ranges
+} // namespace std
+#endif // defined(__cpp_lib_ranges)
+
 #endif // SIMDJSON_DOM_ARRAY_H

include/simdjson/dom/document-inl.h

@@ -20,7 +20,7 @@ inline element document::root() const noexcept {
   return element(internal::tape_ref(this, 1));
 }
 
-WARN_UNUSED
+SIMDJSON_WARN_UNUSED
 inline error_code document::allocate(size_t capacity) noexcept {
   if (capacity == 0) {
     string_buf.reset();
@@ -33,10 +33,10 @@ inline error_code document::allocate(size_t capacity) noexcept {
   // worse with "[7,7,7,7,6,7,7,7,6,7,7,6,[7,7,7,7,6,7,7,7,6,7,7,6,7,7,7,7,7,7,6"
   //where len + 1 tape elements are
   // generated, see issue https://github.com/lemire/simdjson/issues/345
-  size_t tape_capacity = ROUNDUP_N(capacity + 3, 64);
+  size_t tape_capacity = SIMDJSON_ROUNDUP_N(capacity + 3, 64);
   // a document with only zero-length strings... could have len/3 string
   // and we would need len/3 * 5 bytes on the string buffer
-  size_t string_capacity = ROUNDUP_N(5 * capacity / 3 + 32, 64);
+  size_t string_capacity = SIMDJSON_ROUNDUP_N(5 * capacity / 3 + 32, 64);
   string_buf.reset( new (std::nothrow) uint8_t[string_capacity]);
   tape.reset(new (std::nothrow) uint64_t[tape_capacity]);
   return string_buf && tape ? SUCCESS : MEMALLOC;

include/simdjson/dom/document_stream-inl.h

@@ -64,7 +64,7 @@ inline void stage1_worker::run(document_stream * ds, dom::parser * stage1, size_
 }
 #endif
 
-really_inline document_stream::document_stream(
+simdjson_really_inline document_stream::document_stream(
   dom::parser &_parser,
   const uint8_t *_buf,
   size_t _len,
@@ -75,6 +75,9 @@ really_inline document_stream::document_stream(
     len{_len},
     batch_size{_batch_size},
     error{SUCCESS}
+#ifdef SIMDJSON_THREADS_ENABLED
+    , use_thread(_parser.threaded) // we need to make a copy because _parser.threaded can change
+#endif
 {
 #ifdef SIMDJSON_THREADS_ENABLED
   if(worker.get() == nullptr) {
@@ -83,45 +86,49 @@ really_inline document_stream::document_stream(
 #endif
 }
 
-really_inline document_stream::document_stream() noexcept
+simdjson_really_inline document_stream::document_stream() noexcept
   : parser{nullptr},
     buf{nullptr},
     len{0},
     batch_size{0},
-    error{UNINITIALIZED} {
+    error{UNINITIALIZED}
+#ifdef SIMDJSON_THREADS_ENABLED
+    , use_thread(false)
+#endif 
+{
 }
 
-really_inline document_stream::~document_stream() noexcept {
+simdjson_really_inline document_stream::~document_stream() noexcept {
 }
 
-really_inline document_stream::iterator document_stream::begin() noexcept {
+simdjson_really_inline document_stream::iterator document_stream::begin() noexcept {
   start();
   // If there are no documents, we're finished.
   return iterator(*this, error == EMPTY);
 }
 
-really_inline document_stream::iterator document_stream::end() noexcept {
+simdjson_really_inline document_stream::iterator document_stream::end() noexcept {
   return iterator(*this, true);
 }
 
-really_inline document_stream::iterator::iterator(document_stream& _stream, bool is_end) noexcept
+simdjson_really_inline document_stream::iterator::iterator(document_stream& _stream, bool is_end) noexcept
   : stream{_stream}, finished{is_end} {
 }
 
-really_inline simdjson_result<element> document_stream::iterator::operator*() noexcept {
+simdjson_really_inline simdjson_result<element> document_stream::iterator::operator*() noexcept {
   // Once we have yielded any errors, we're finished.
   if (stream.error) { finished = true; return stream.error; }
   return stream.parser->doc.root();
 }
 
-really_inline document_stream::iterator& document_stream::iterator::operator++() noexcept {
+simdjson_really_inline document_stream::iterator& document_stream::iterator::operator++() noexcept {
   stream.next();
   // If that was the last document, we're finished.
   if (stream.error == EMPTY) { finished = true; }
   return *this;
 }
 
-really_inline bool document_stream::iterator::operator!=(const document_stream::iterator &other) const noexcept {
+simdjson_really_inline bool document_stream::iterator::operator!=(const document_stream::iterator &other) const noexcept {
   return finished != other.finished;
 }
 
@@ -137,7 +144,7 @@ inline void document_stream::start() noexcept {
   if (error) { return; }
 
 #ifdef SIMDJSON_THREADS_ENABLED
-  if (next_batch_start() < len) {
+  if (use_thread && next_batch_start() < len) {
     // Kick off the first thread if needed
     error = stage1_thread_parser.ensure_capacity(batch_size);
     if (error) { return; }
@@ -150,9 +157,16 @@ inline void document_stream::start() noexcept {
   next();
 }
 
-really_inline size_t document_stream::iterator::current_index() noexcept {
+simdjson_really_inline size_t document_stream::iterator::current_index() const noexcept {
   return stream.doc_index;
 }
+
+simdjson_really_inline std::string_view document_stream::iterator::source() const noexcept {
+  size_t next_doc_index = stream.batch_start + stream.parser->implementation->structural_indexes[stream.parser->implementation->next_structural_index];
+  return std::string_view(reinterpret_cast<const char*>(stream.buf) + current_index(), next_doc_index - current_index() - 1);
+}
+
+
 inline void document_stream::next() noexcept {
   if (error) { return; }
 
@@ -165,7 +179,11 @@ inline void document_stream::next() noexcept {
     if (batch_start >= len) { break; }
 
 #ifdef SIMDJSON_THREADS_ENABLED
-    load_from_stage1_thread();
+    if(use_thread) {
+      load_from_stage1_thread();
+    } else {
+      error = run_stage1(*parser, batch_start);
+    }
 #else
     error = run_stage1(*parser, batch_start);
 #endif
@@ -221,31 +239,31 @@ inline void document_stream::start_stage1_thread() noexcept {
 
 } // namespace dom
 
-really_inline simdjson_result<dom::document_stream>::simdjson_result() noexcept
+simdjson_really_inline simdjson_result<dom::document_stream>::simdjson_result() noexcept
   : simdjson_result_base() {
 }
-really_inline simdjson_result<dom::document_stream>::simdjson_result(error_code error) noexcept
+simdjson_really_inline simdjson_result<dom::document_stream>::simdjson_result(error_code error) noexcept
   : simdjson_result_base(error) {
 }
-really_inline simdjson_result<dom::document_stream>::simdjson_result(dom::document_stream &&value) noexcept
+simdjson_really_inline simdjson_result<dom::document_stream>::simdjson_result(dom::document_stream &&value) noexcept
   : simdjson_result_base(std::forward<dom::document_stream>(value)) {
 }
 
 #if SIMDJSON_EXCEPTIONS
-really_inline dom::document_stream::iterator simdjson_result<dom::document_stream>::begin() noexcept(false) {
+simdjson_really_inline dom::document_stream::iterator simdjson_result<dom::document_stream>::begin() noexcept(false) {
   if (error()) { throw simdjson_error(error()); }
   return first.begin();
 }
-really_inline dom::document_stream::iterator simdjson_result<dom::document_stream>::end() noexcept(false) {
+simdjson_really_inline dom::document_stream::iterator simdjson_result<dom::document_stream>::end() noexcept(false) {
   if (error()) { throw simdjson_error(error()); }
   return first.end();
 }
 #else // SIMDJSON_EXCEPTIONS
-really_inline dom::document_stream::iterator simdjson_result<dom::document_stream>::begin() noexcept {
+simdjson_really_inline dom::document_stream::iterator simdjson_result<dom::document_stream>::begin() noexcept {
   first.error = error();
   return first.begin();
 }
-really_inline dom::document_stream::iterator simdjson_result<dom::document_stream>::end() noexcept {
+simdjson_really_inline dom::document_stream::iterator simdjson_result<dom::document_stream>::end() noexcept {
   first.error = error();
   return first.end();
 }

include/simdjson/dom/document_stream.h

@@ -80,13 +80,13 @@ public:
    *  error = parser.parse_many(json).get(docs);
    *  ```
    */
-  really_inline document_stream() noexcept;
+  simdjson_really_inline document_stream() noexcept;
   /** Move one document_stream to another. */
-  really_inline document_stream(document_stream &&other) noexcept = default;
+  simdjson_really_inline document_stream(document_stream &&other) noexcept = default;
   /** Move one document_stream to another. */
-  really_inline document_stream &operator=(document_stream &&other) noexcept = default;
+  simdjson_really_inline document_stream &operator=(document_stream &&other) noexcept = default;
 
-  really_inline ~document_stream() noexcept;
+  simdjson_really_inline ~document_stream() noexcept;
 
   /**
    * An iterator through a forward-only stream of documents.
@@ -96,7 +96,7 @@ public:
     /**
      * Get the current document (or error).
      */
-    really_inline simdjson_result<element> operator*() noexcept;
+    simdjson_really_inline simdjson_result<element> operator*() noexcept;
     /**
      * Advance to the next document.
      */
@@ -105,7 +105,7 @@ public:
      * Check if we're at the end yet.
      * @param other the end iterator to compare to.
      */
-    really_inline bool operator!=(const iterator &other) const noexcept;
+    simdjson_really_inline bool operator!=(const iterator &other) const noexcept;
     /**
      * @private
      * 
@@ -121,9 +121,30 @@ public:
      * may change in future versions of simdjson: we find the API somewhat
      * awkward and we would like to offer something friendlier.  
      */
-     really_inline size_t current_index() noexcept;
+     simdjson_really_inline size_t current_index() const noexcept;
+    /**
+     * @private
+     * 
+     * Gives a view of the current document.
+     *
+     *   document_stream stream = parser.parse_many(json,window);
+     *   for(auto i = stream.begin(); i != stream.end(); ++i) {
+     *      auto doc = *i;
+     *      std::string_view v = i->source();
+     *   }
+     * 
+     * The returned string_view instance is simply a map to the (unparsed)
+     * source string: it may thus include white-space characters and all manner
+     * of padding.
+     * 
+     * This function (source()) is experimental and the usage
+     * may change in future versions of simdjson: we find the API somewhat
+     * awkward and we would like to offer something friendlier.  
+     */
+     simdjson_really_inline std::string_view source() const noexcept;
+
   private:
-    really_inline iterator(document_stream &s, bool finished) noexcept;
+    simdjson_really_inline iterator(document_stream &s, bool finished) noexcept;
     /** The document_stream we're iterating through. */
     document_stream& stream;
     /** Whether we're finished or not. */
@@ -134,11 +155,11 @@ public:
   /**
    * Start iterating the documents in the stream.
    */
-  really_inline iterator begin() noexcept;
+  simdjson_really_inline iterator begin() noexcept;
   /**
    * The end of the stream, for iterator comparison purposes.
    */
-  really_inline iterator end() noexcept;
+  simdjson_really_inline iterator end() noexcept;
 
 private:
 
@@ -148,8 +169,13 @@ private:
   /**
    * Construct a document_stream. Does not allocate or parse anything until the iterator is
    * used.
+   * 
+   * @param parser is a reference to the parser instance used to generate this document_stream
+   * @param buf is the raw byte buffer we need to process
+   * @param len is the length of the raw byte buffer in bytes
+   * @param batch_size is the size of the windows (must be strictly greater or equal to the largest JSON document)
    */
-  really_inline document_stream(
+  simdjson_really_inline document_stream(
     dom::parser &parser,
     const uint8_t *buf,
     size_t len,
@@ -210,6 +236,9 @@ private:
   size_t doc_index{};
 
 #ifdef SIMDJSON_THREADS_ENABLED
+  /** Indicates whether we use threads. Note that this needs to be a constant during the execution of the parsing. */
+  bool use_thread; 
+
   inline void load_from_stage1_thread() noexcept;
 
   /** Start a thread to run stage 1 on the next batch. */
@@ -241,18 +270,18 @@ private:
 template<>
 struct simdjson_result<dom::document_stream> : public internal::simdjson_result_base<dom::document_stream> {
 public:
-  really_inline simdjson_result() noexcept; ///< @private
-  really_inline simdjson_result(error_code error) noexcept; ///< @private
-  really_inline simdjson_result(dom::document_stream &&value) noexcept; ///< @private
+  simdjson_really_inline simdjson_result() noexcept; ///< @private
+  simdjson_really_inline simdjson_result(error_code error) noexcept; ///< @private
+  simdjson_really_inline simdjson_result(dom::document_stream &&value) noexcept; ///< @private
 
 #if SIMDJSON_EXCEPTIONS
-  really_inline dom::document_stream::iterator begin() noexcept(false);
-  really_inline dom::document_stream::iterator end() noexcept(false);
+  simdjson_really_inline dom::document_stream::iterator begin() noexcept(false);
+  simdjson_really_inline dom::document_stream::iterator end() noexcept(false);
 #else // SIMDJSON_EXCEPTIONS
   [[deprecated("parse_many() and load_many() may return errors. Use document_stream stream; error = parser.parse_many().get(doc); instead.")]]
-  really_inline dom::document_stream::iterator begin() noexcept;
+  simdjson_really_inline dom::document_stream::iterator begin() noexcept;
   [[deprecated("parse_many() and load_many() may return errors. Use document_stream stream; error = parser.parse_many().get(doc); instead.")]]
-  really_inline dom::document_stream::iterator end() noexcept;
+  simdjson_really_inline dom::document_stream::iterator end() noexcept;
 #endif // SIMDJSON_EXCEPTIONS
 }; // struct simdjson_result<dom::document_stream>
 

include/simdjson/dom/element-inl.h

@@ -12,11 +12,11 @@ namespace simdjson {
 //
 // simdjson_result<dom::element> inline implementation
 //
-really_inline simdjson_result<dom::element>::simdjson_result() noexcept
+simdjson_really_inline simdjson_result<dom::element>::simdjson_result() noexcept
     : internal::simdjson_result_base<dom::element>() {}
-really_inline simdjson_result<dom::element>::simdjson_result(dom::element &&value) noexcept
+simdjson_really_inline simdjson_result<dom::element>::simdjson_result(dom::element &&value) noexcept
     : internal::simdjson_result_base<dom::element>(std::forward<dom::element>(value)) {}
-really_inline simdjson_result<dom::element>::simdjson_result(error_code error) noexcept
+simdjson_really_inline simdjson_result<dom::element>::simdjson_result(error_code error) noexcept
     : internal::simdjson_result_base<dom::element>(error) {}
 inline simdjson_result<dom::element_type> simdjson_result<dom::element>::type() const noexcept {
   if (error()) { return error(); }
@@ -24,140 +24,148 @@ inline simdjson_result<dom::element_type> simdjson_result<dom::element>::type()
 }
 
 template<typename T>
-really_inline bool simdjson_result<dom::element>::is() const noexcept {
+simdjson_really_inline bool simdjson_result<dom::element>::is() const noexcept {
   return !error() && first.is<T>();
 }
 template<typename T>
-really_inline simdjson_result<T> simdjson_result<dom::element>::get() const noexcept {
+simdjson_really_inline simdjson_result<T> simdjson_result<dom::element>::get() const noexcept {
   if (error()) { return error(); }
   return first.get<T>();
 }
 template<typename T>
-WARN_UNUSED really_inline error_code simdjson_result<dom::element>::get(T &value) const noexcept {
+SIMDJSON_WARN_UNUSED simdjson_really_inline error_code simdjson_result<dom::element>::get(T &value) const noexcept {
   if (error()) { return error(); }
   return first.get<T>(value);
 }
 
-really_inline simdjson_result<dom::array> simdjson_result<dom::element>::get_array() const noexcept {
+simdjson_really_inline simdjson_result<dom::array> simdjson_result<dom::element>::get_array() const noexcept {
   if (error()) { return error(); }
   return first.get_array();
 }
-really_inline simdjson_result<dom::object> simdjson_result<dom::element>::get_object() const noexcept {
+simdjson_really_inline simdjson_result<dom::object> simdjson_result<dom::element>::get_object() const noexcept {
   if (error()) { return error(); }
   return first.get_object();
 }
-really_inline simdjson_result<const char *> simdjson_result<dom::element>::get_c_str() const noexcept {
+simdjson_really_inline simdjson_result<const char *> simdjson_result<dom::element>::get_c_str() const noexcept {
   if (error()) { return error(); }
   return first.get_c_str();
 }
-really_inline simdjson_result<size_t> simdjson_result<dom::element>::get_string_length() const noexcept {
+simdjson_really_inline simdjson_result<size_t> simdjson_result<dom::element>::get_string_length() const noexcept {
   if (error()) { return error(); }
   return first.get_string_length();
 }
-really_inline simdjson_result<std::string_view> simdjson_result<dom::element>::get_string() const noexcept {
+simdjson_really_inline simdjson_result<std::string_view> simdjson_result<dom::element>::get_string() const noexcept {
   if (error()) { return error(); }
   return first.get_string();
 }
-really_inline simdjson_result<int64_t> simdjson_result<dom::element>::get_int64() const noexcept {
+simdjson_really_inline simdjson_result<int64_t> simdjson_result<dom::element>::get_int64() const noexcept {
   if (error()) { return error(); }
   return first.get_int64();
 }
-really_inline simdjson_result<uint64_t> simdjson_result<dom::element>::get_uint64() const noexcept {
+simdjson_really_inline simdjson_result<uint64_t> simdjson_result<dom::element>::get_uint64() const noexcept {
   if (error()) { return error(); }
   return first.get_uint64();
 }
-really_inline simdjson_result<double> simdjson_result<dom::element>::get_double() const noexcept {
+simdjson_really_inline simdjson_result<double> simdjson_result<dom::element>::get_double() const noexcept {
   if (error()) { return error(); }
   return first.get_double();
 }
-really_inline simdjson_result<bool> simdjson_result<dom::element>::get_bool() const noexcept {
+simdjson_really_inline simdjson_result<bool> simdjson_result<dom::element>::get_bool() const noexcept {
   if (error()) { return error(); }
   return first.get_bool();
 }
 
-really_inline bool simdjson_result<dom::element>::is_array() const noexcept {
+simdjson_really_inline bool simdjson_result<dom::element>::is_array() const noexcept {
   return !error() && first.is_array();
 }
-really_inline bool simdjson_result<dom::element>::is_object() const noexcept {
+simdjson_really_inline bool simdjson_result<dom::element>::is_object() const noexcept {
   return !error() && first.is_object();
 }
-really_inline bool simdjson_result<dom::element>::is_string() const noexcept {
+simdjson_really_inline bool simdjson_result<dom::element>::is_string() const noexcept {
   return !error() && first.is_string();
 }
-really_inline bool simdjson_result<dom::element>::is_int64() const noexcept {
+simdjson_really_inline bool simdjson_result<dom::element>::is_int64() const noexcept {
   return !error() && first.is_int64();
 }
-really_inline bool simdjson_result<dom::element>::is_uint64() const noexcept {
+simdjson_really_inline bool simdjson_result<dom::element>::is_uint64() const noexcept {
   return !error() && first.is_uint64();
 }
-really_inline bool simdjson_result<dom::element>::is_double() const noexcept {
+simdjson_really_inline bool simdjson_result<dom::element>::is_double() const noexcept {
   return !error() && first.is_double();
 }
-really_inline bool simdjson_result<dom::element>::is_bool() const noexcept {
+simdjson_really_inline bool simdjson_result<dom::element>::is_bool() const noexcept {
   return !error() && first.is_bool();
 }
 
-really_inline bool simdjson_result<dom::element>::is_null() const noexcept {
+simdjson_really_inline bool simdjson_result<dom::element>::is_null() const noexcept {
   return !error() && first.is_null();
 }
 
-really_inline simdjson_result<dom::element> simdjson_result<dom::element>::operator[](const std::string_view &key) const noexcept {
+simdjson_really_inline simdjson_result<dom::element> simdjson_result<dom::element>::operator[](std::string_view key) const noexcept {
   if (error()) { return error(); }
   return first[key];
 }
-really_inline simdjson_result<dom::element> simdjson_result<dom::element>::operator[](const char *key) const noexcept {
+simdjson_really_inline simdjson_result<dom::element> simdjson_result<dom::element>::operator[](const char *key) const noexcept {
   if (error()) { return error(); }
   return first[key];
 }
-really_inline simdjson_result<dom::element> simdjson_result<dom::element>::at(const std::string_view &json_pointer) const noexcept {
+simdjson_really_inline simdjson_result<dom::element> simdjson_result<dom::element>::at_pointer(const std::string_view json_pointer) const noexcept {
+  if (error()) { return error(); }
+  return first.at_pointer(json_pointer);
+}
+[[deprecated("For standard compliance, use at_pointer instead, and prefix your pointers with a slash '/', see RFC6901 ")]]
+simdjson_really_inline simdjson_result<dom::element> simdjson_result<dom::element>::at(const std::string_view json_pointer) const noexcept {
+SIMDJSON_PUSH_DISABLE_WARNINGS
+SIMDJSON_DISABLE_DEPRECATED_WARNING
   if (error()) { return error(); }
   return first.at(json_pointer);
+SIMDJSON_POP_DISABLE_WARNINGS
 }
-really_inline simdjson_result<dom::element> simdjson_result<dom::element>::at(size_t index) const noexcept {
+simdjson_really_inline simdjson_result<dom::element> simdjson_result<dom::element>::at(size_t index) const noexcept {
   if (error()) { return error(); }
   return first.at(index);
 }
-really_inline simdjson_result<dom::element> simdjson_result<dom::element>::at_key(const std::string_view &key) const noexcept {
+simdjson_really_inline simdjson_result<dom::element> simdjson_result<dom::element>::at_key(std::string_view key) const noexcept {
   if (error()) { return error(); }
   return first.at_key(key);
 }
-really_inline simdjson_result<dom::element> simdjson_result<dom::element>::at_key_case_insensitive(const std::string_view &key) const noexcept {
+simdjson_really_inline simdjson_result<dom::element> simdjson_result<dom::element>::at_key_case_insensitive(std::string_view key) const noexcept {
   if (error()) { return error(); }
   return first.at_key_case_insensitive(key);
 }
 
 #if SIMDJSON_EXCEPTIONS
 
-really_inline simdjson_result<dom::element>::operator bool() const noexcept(false) {
+simdjson_really_inline simdjson_result<dom::element>::operator bool() const noexcept(false) {
   return get<bool>();
 }
-really_inline simdjson_result<dom::element>::operator const char *() const noexcept(false) {
+simdjson_really_inline simdjson_result<dom::element>::operator const char *() const noexcept(false) {
   return get<const char *>();
 }
-really_inline simdjson_result<dom::element>::operator std::string_view() const noexcept(false) {
+simdjson_really_inline simdjson_result<dom::element>::operator std::string_view() const noexcept(false) {
   return get<std::string_view>();
 }
-really_inline simdjson_result<dom::element>::operator uint64_t() const noexcept(false) {
+simdjson_really_inline simdjson_result<dom::element>::operator uint64_t() const noexcept(false) {
   return get<uint64_t>();
 }
-really_inline simdjson_result<dom::element>::operator int64_t() const noexcept(false) {
+simdjson_really_inline simdjson_result<dom::element>::operator int64_t() const noexcept(false) {
   return get<int64_t>();
 }
-really_inline simdjson_result<dom::element>::operator double() const noexcept(false) {
+simdjson_really_inline simdjson_result<dom::element>::operator double() const noexcept(false) {
   return get<double>();
 }
-really_inline simdjson_result<dom::element>::operator dom::array() const noexcept(false) {
+simdjson_really_inline simdjson_result<dom::element>::operator dom::array() const noexcept(false) {
   return get<dom::array>();
 }
-really_inline simdjson_result<dom::element>::operator dom::object() const noexcept(false) {
+simdjson_really_inline simdjson_result<dom::element>::operator dom::object() const noexcept(false) {
   return get<dom::object>();
 }
 
-really_inline dom::array::iterator simdjson_result<dom::element>::begin() const noexcept(false) {
+simdjson_really_inline dom::array::iterator simdjson_result<dom::element>::begin() const noexcept(false) {
   if (error()) { throw simdjson_error(error()); }
   return first.begin();
 }
-really_inline dom::array::iterator simdjson_result<dom::element>::end() const noexcept(false) {
+simdjson_really_inline dom::array::iterator simdjson_result<dom::element>::end() const noexcept(false) {
   if (error()) { throw simdjson_error(error()); }
   return first.end();
 }
@@ -169,8 +177,8 @@ namespace dom {
 //
 // element inline implementation
 //
-really_inline element::element() noexcept : tape{} {}
-really_inline element::element(const internal::tape_ref &_tape) noexcept : tape{_tape} { }
+simdjson_really_inline element::element() noexcept : tape{} {}
+simdjson_really_inline element::element(const internal::tape_ref &_tape) noexcept : tape{_tape} { }
 
 inline element_type element::type() const noexcept {
   auto tape_type = tape.tape_ref_type();
@@ -212,7 +220,7 @@ inline simdjson_result<std::string_view> element::get_string() const noexcept {
   }
 }
 inline simdjson_result<uint64_t> element::get_uint64() const noexcept {
-  if(unlikely(!tape.is_uint64())) { // branch rarely taken
+  if(simdjson_unlikely(!tape.is_uint64())) { // branch rarely taken
     if(tape.is_int64()) {
       int64_t result = tape.next_tape_value<int64_t>();
       if (result < 0) {
@@ -225,7 +233,7 @@ inline simdjson_result<uint64_t> element::get_uint64() const noexcept {
   return tape.next_tape_value<int64_t>();
 }
 inline simdjson_result<int64_t> element::get_int64() const noexcept {
-  if(unlikely(!tape.is_int64())) { // branch rarely taken
+  if(simdjson_unlikely(!tape.is_int64())) { // branch rarely taken
     if(tape.is_uint64()) {
       uint64_t result = tape.next_tape_value<uint64_t>();
       // Wrapping max in parens to handle Windows issue: https://stackoverflow.com/questions/11544073/how-do-i-deal-with-the-max-macro-in-windows-h-colliding-with-max-in-std
@@ -248,7 +256,7 @@ inline simdjson_result<double> element::get_double() const noexcept {
   // We can expect get<double> to refer to a double type almost all the time.
   // It is important to craft the code accordingly so that the compiler can use this
   // information. (This could also be solved with profile-guided optimization.)
-  if(unlikely(!tape.is_double())) { // branch rarely taken
+  if(simdjson_unlikely(!tape.is_double())) { // branch rarely taken
     if(tape.is_uint64()) {
       return double(tape.next_tape_value<uint64_t>());
     } else if(tape.is_int64()) {
@@ -277,18 +285,18 @@ inline simdjson_result<object> element::get_object() const noexcept {
 }
 
 template<typename T>
-WARN_UNUSED really_inline error_code element::get(T &value) const noexcept {
+SIMDJSON_WARN_UNUSED simdjson_really_inline error_code element::get(T &value) const noexcept {
   return get<T>().get(value);
 }
 // An element-specific version prevents recursion with simdjson_result::get<element>(value)
 template<>
-WARN_UNUSED really_inline error_code element::get<element>(element &value) const noexcept {
+SIMDJSON_WARN_UNUSED simdjson_really_inline error_code element::get<element>(element &value) const noexcept {
   value = element(tape);
   return SUCCESS;
 }
 
 template<typename T>
-really_inline bool element::is() const noexcept {
+simdjson_really_inline bool element::is() const noexcept {
   auto result = get<T>();
   return !result.error();
 }
@@ -334,29 +342,43 @@ inline array::iterator element::end() const noexcept(false) {
 
 #endif // SIMDJSON_EXCEPTIONS
 
-inline simdjson_result<element> element::operator[](const std::string_view &key) const noexcept {
+inline simdjson_result<element> element::operator[](std::string_view key) const noexcept {
   return at_key(key);
 }
 inline simdjson_result<element> element::operator[](const char *key) const noexcept {
   return at_key(key);
 }
-inline simdjson_result<element> element::at(const std::string_view &json_pointer) const noexcept {
+
+inline simdjson_result<element> element::at_pointer(std::string_view json_pointer) const noexcept {
   switch (tape.tape_ref_type()) {
     case internal::tape_type::START_OBJECT:
-      return object(tape).at(json_pointer);
+      return object(tape).at_pointer(json_pointer);
     case internal::tape_type::START_ARRAY:
-      return array(tape).at(json_pointer);
-    default:
-      return INCORRECT_TYPE;
+      return array(tape).at_pointer(json_pointer);
+    default: {
+      if(json_pointer.empty()) { // an empty string means that we return the current node
+        return INVALID_JSON_POINTER;
+      }
+      dom::element copy(*this);
+      return simdjson_result<element>(std::move(copy));
+    }
   }
 }
+
+[[deprecated("For standard compliance, use at_pointer instead, and prefix your pointers with a slash '/', see RFC6901 ")]]
+inline simdjson_result<element> element::at(std::string_view json_pointer) const noexcept {
+  // version 0.4 of simdjson allowed non-compliant pointers
+  auto std_pointer = (json_pointer.empty() ? "" : "/") + std::string(json_pointer.begin(), json_pointer.end());
+  return at_pointer(std_pointer);
+}
+
 inline simdjson_result<element> element::at(size_t index) const noexcept {
   return get<array>().at(index);
 }
-inline simdjson_result<element> element::at_key(const std::string_view &key) const noexcept {
+inline simdjson_result<element> element::at_key(std::string_view key) const noexcept {
   return get<object>().at_key(key);
 }
-inline simdjson_result<element> element::at_key_case_insensitive(const std::string_view &key) const noexcept {
+inline simdjson_result<element> element::at_key_case_insensitive(std::string_view key) const noexcept {
   return get<object>().at_key_case_insensitive(key);
 }
 
@@ -419,7 +441,7 @@ inline std::ostream& minifier<dom::element>::print(std::ostream& out) {
     case tape_type::START_ARRAY: {
       // If we're too deep, we need to recurse to go deeper.
       depth++;
-      if (unlikely(depth >= MAX_DEPTH)) {
+      if (simdjson_unlikely(depth >= MAX_DEPTH)) {
         out << minify<dom::array>(dom::array(iter));
         iter.json_index = iter.matching_brace_index() - 1; // Jump to the ]
         depth--;
@@ -446,7 +468,7 @@ inline std::ostream& minifier<dom::element>::print(std::ostream& out) {
     case tape_type::START_OBJECT: {
       // If we're too deep, we need to recurse to go deeper.
       depth++;
-      if (unlikely(depth >= MAX_DEPTH)) {
+      if (simdjson_unlikely(depth >= MAX_DEPTH)) {
         out << minify<dom::object>(dom::object(iter));
         iter.json_index = iter.matching_brace_index() - 1; // Jump to the }
         depth--;
@@ -520,12 +542,12 @@ inline std::ostream& minifier<dom::element>::print(std::ostream& out) {
 #if SIMDJSON_EXCEPTIONS
 
 template<>
-really_inline std::ostream& minifier<simdjson_result<dom::element>>::print(std::ostream& out) {
+simdjson_really_inline std::ostream& minifier<simdjson_result<dom::element>>::print(std::ostream& out) {
   if (value.error()) { throw simdjson_error(value.error()); }
   return out << minify<dom::element>(value.first);
 }
 
-really_inline std::ostream& operator<<(std::ostream& out, const simdjson_result<dom::element> &value) noexcept(false) {
+simdjson_really_inline std::ostream& operator<<(std::ostream& out, const simdjson_result<dom::element> &value) noexcept(false) {
   return out << minify<simdjson_result<dom::element>>(value);
 }
 #endif

include/simdjson/dom/element.h

@@ -38,10 +38,10 @@ enum class element_type {
 class element {
 public:
   /** Create a new, invalid element. */
-  really_inline element() noexcept;
+  simdjson_really_inline element() noexcept;
 
   /** The type of this element. */
-  really_inline element_type type() const noexcept;
+  simdjson_really_inline element_type type() const noexcept;
 
   /**
    * Cast this element to an array.
@@ -207,7 +207,7 @@ public:
    * @tparam T bool, double, uint64_t, int64_t, std::string_view, const char *, dom::array, dom::object
    */
   template<typename T>
-  really_inline bool is() const noexcept;
+  simdjson_really_inline bool is() const noexcept;
 
   /**
    * Get the value as the provided type (T).
@@ -244,7 +244,7 @@ public:
    * @returns The error that occurred, or SUCCESS if there was no error.
    */
   template<typename T>
-  WARN_UNUSED really_inline error_code get(T &value) const noexcept;
+  SIMDJSON_WARN_UNUSED simdjson_really_inline error_code get(T &value) const noexcept;
 
   /**
    * Get the value as the provided type (T), setting error if it's not the given type.
@@ -366,7 +366,7 @@ public:
    *         - NO_SUCH_FIELD if the field does not exist in the object
    *         - INCORRECT_TYPE if this is not an object
    */
-  inline simdjson_result<element> operator[](const std::string_view &key) const noexcept;
+  inline simdjson_result<element> operator[](std::string_view key) const noexcept;
 
   /**
    * Get the value associated with the given key.
@@ -384,21 +384,51 @@ public:
   inline simdjson_result<element> operator[](const char *key) const noexcept;
 
   /**
-   * Get the value associated with the given JSON pointer.
+   * Get the value associated with the given JSON pointer.  We use the RFC 6901
+   * https://tools.ietf.org/html/rfc6901 standard.
    *
    *   dom::parser parser;
    *   element doc = parser.parse(R"({ "foo": { "a": [ 10, 20, 30 ] }})"_padded);
-   *   doc.at("/foo/a/1") == 20
-   *   doc.at("/")["foo"]["a"].at(1) == 20
-   *   doc.at("")["foo"]["a"].at(1) == 20
+   *   doc.at_pointer("/foo/a/1") == 20
+   *   doc.at_pointer("/foo")["a"].at(1) == 20
+   *   doc.at_pointer("")["foo"]["a"].at(1) == 20
    *
+   * It is allowed for a key to be the empty string:
+   *
+   *   dom::parser parser;
+   *   object obj = parser.parse(R"({ "": { "a": [ 10, 20, 30 ] }})"_padded);
+   *   obj.at_pointer("//a/1") == 20
+   * 
    * @return The value associated with the given JSON pointer, or:
    *         - NO_SUCH_FIELD if a field does not exist in an object
    *         - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length
    *         - INCORRECT_TYPE if a non-integer is used to access an array
    *         - INVALID_JSON_POINTER if the JSON pointer is invalid and cannot be parsed
    */
-  inline simdjson_result<element> at(const std::string_view &json_pointer) const noexcept;
+  inline simdjson_result<element> at_pointer(const std::string_view json_pointer) const noexcept;
+  
+  /**
+   * 
+   * Version 0.4 of simdjson used an incorrect interpretation of the JSON Pointer standard
+   * and allowed the following :
+   * 
+   *   dom::parser parser;
+   *   element doc = parser.parse(R"({ "foo": { "a": [ 10, 20, 30 ] }})"_padded);
+   *   doc.at("foo/a/1") == 20
+   * 
+   * Though it is intuitive, it is not compliant with RFC 6901
+   * https://tools.ietf.org/html/rfc6901 
+   * 
+   * For standard compliance, use the at_pointer function instead.
+   * 
+   * @return The value associated with the given JSON pointer, or:
+   *         - NO_SUCH_FIELD if a field does not exist in an object
+   *         - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length
+   *         - INCORRECT_TYPE if a non-integer is used to access an array
+   *         - INVALID_JSON_POINTER if the JSON pointer is invalid and cannot be parsed
+   */
+  [[deprecated("For standard compliance, use at_pointer instead, and prefix your pointers with a slash '/', see RFC6901 ")]]
+  inline simdjson_result<element> at(const std::string_view json_pointer) const noexcept;
 
   /**
    * Get the value at the given index.
@@ -420,7 +450,7 @@ public:
    * @return The value associated with this field, or:
    *         - NO_SUCH_FIELD if the field does not exist in the object
    */
-  inline simdjson_result<element> at_key(const std::string_view &key) const noexcept;
+  inline simdjson_result<element> at_key(std::string_view key) const noexcept;
 
   /**
    * Get the value associated with the given key in a case-insensitive manner.
@@ -430,13 +460,13 @@ public:
    * @return The value associated with this field, or:
    *         - NO_SUCH_FIELD if the field does not exist in the object
    */
-  inline simdjson_result<element> at_key_case_insensitive(const std::string_view &key) const noexcept;
+  inline simdjson_result<element> at_key_case_insensitive(std::string_view key) const noexcept;
 
   /** @private for debugging. Prints out the root element. */
   inline bool dump_raw_tape(std::ostream &out) const noexcept;
 
 private:
-  really_inline element(const internal::tape_ref &tape) noexcept;
+  simdjson_really_inline element(const internal::tape_ref &tape) noexcept;
   internal::tape_ref tape;
   friend class document;
   friend class object;
@@ -472,56 +502,58 @@ inline std::ostream& operator<<(std::ostream& out, element_type type);
 template<>
 struct simdjson_result<dom::element> : public internal::simdjson_result_base<dom::element> {
 public:
-  really_inline simdjson_result() noexcept; ///< @private
-  really_inline simdjson_result(dom::element &&value) noexcept; ///< @private
-  really_inline simdjson_result(error_code error) noexcept; ///< @private
+  simdjson_really_inline simdjson_result() noexcept; ///< @private
+  simdjson_really_inline simdjson_result(dom::element &&value) noexcept; ///< @private
+  simdjson_really_inline simdjson_result(error_code error) noexcept; ///< @private
 
-  really_inline simdjson_result<dom::element_type> type() const noexcept;
+  simdjson_really_inline simdjson_result<dom::element_type> type() const noexcept;
   template<typename T>
-  really_inline bool is() const noexcept;
+  simdjson_really_inline bool is() const noexcept;
   template<typename T>
-  really_inline simdjson_result<T> get() const noexcept;
+  simdjson_really_inline simdjson_result<T> get() const noexcept;
   template<typename T>
-  WARN_UNUSED really_inline error_code get(T &value) const noexcept;
+  SIMDJSON_WARN_UNUSED simdjson_really_inline error_code get(T &value) const noexcept;
 
-  really_inline simdjson_result<dom::array> get_array() const noexcept;
-  really_inline simdjson_result<dom::object> get_object() const noexcept;
-  really_inline simdjson_result<const char *> get_c_str() const noexcept;
-  really_inline simdjson_result<size_t> get_string_length() const noexcept;
-  really_inline simdjson_result<std::string_view> get_string() const noexcept;
-  really_inline simdjson_result<int64_t> get_int64() const noexcept;
-  really_inline simdjson_result<uint64_t> get_uint64() const noexcept;
-  really_inline simdjson_result<double> get_double() const noexcept;
-  really_inline simdjson_result<bool> get_bool() const noexcept;
+  simdjson_really_inline simdjson_result<dom::array> get_array() const noexcept;
+  simdjson_really_inline simdjson_result<dom::object> get_object() const noexcept;
+  simdjson_really_inline simdjson_result<const char *> get_c_str() const noexcept;
+  simdjson_really_inline simdjson_result<size_t> get_string_length() const noexcept;
+  simdjson_really_inline simdjson_result<std::string_view> get_string() const noexcept;
+  simdjson_really_inline simdjson_result<int64_t> get_int64() const noexcept;
+  simdjson_really_inline simdjson_result<uint64_t> get_uint64() const noexcept;
+  simdjson_really_inline simdjson_result<double> get_double() const noexcept;
+  simdjson_really_inline simdjson_result<bool> get_bool() const noexcept;
 
-  really_inline bool is_array() const noexcept;
-  really_inline bool is_object() const noexcept;
-  really_inline bool is_string() const noexcept;
-  really_inline bool is_int64() const noexcept;
-  really_inline bool is_uint64() const noexcept;
-  really_inline bool is_double() const noexcept;
-  really_inline bool is_bool() const noexcept;
-  really_inline bool is_null() const noexcept;
+  simdjson_really_inline bool is_array() const noexcept;
+  simdjson_really_inline bool is_object() const noexcept;
+  simdjson_really_inline bool is_string() const noexcept;
+  simdjson_really_inline bool is_int64() const noexcept;
+  simdjson_really_inline bool is_uint64() const noexcept;
+  simdjson_really_inline bool is_double() const noexcept;
+  simdjson_really_inline bool is_bool() const noexcept;
+  simdjson_really_inline bool is_null() const noexcept;
 
-  really_inline simdjson_result<dom::element> operator[](const std::string_view &key) const noexcept;
-  really_inline simdjson_result<dom::element> operator[](const char *key) const noexcept;
-  really_inline simdjson_result<dom::element> at(const std::string_view &json_pointer) const noexcept;
-  really_inline simdjson_result<dom::element> at(size_t index) const noexcept;
-  really_inline simdjson_result<dom::element> at_key(const std::string_view &key) const noexcept;
-  really_inline simdjson_result<dom::element> at_key_case_insensitive(const std::string_view &key) const noexcept;
+  simdjson_really_inline simdjson_result<dom::element> operator[](std::string_view key) const noexcept;
+  simdjson_really_inline simdjson_result<dom::element> operator[](const char *key) const noexcept;
+  simdjson_really_inline simdjson_result<dom::element> at_pointer(const std::string_view json_pointer) const noexcept;
+  [[deprecated("For standard compliance, use at_pointer instead, and prefix your pointers with a slash '/', see RFC6901 ")]]
+  simdjson_really_inline simdjson_result<dom::element> at(const std::string_view json_pointer) const noexcept;
+  simdjson_really_inline simdjson_result<dom::element> at(size_t index) const noexcept;
+  simdjson_really_inline simdjson_result<dom::element> at_key(std::string_view key) const noexcept;
+  simdjson_really_inline simdjson_result<dom::element> at_key_case_insensitive(std::string_view key) const noexcept;
 
 #if SIMDJSON_EXCEPTIONS
-  really_inline operator bool() const noexcept(false);
-  really_inline explicit operator const char*() const noexcept(false);
-  really_inline operator std::string_view() const noexcept(false);
-  really_inline operator uint64_t() const noexcept(false);
-  really_inline operator int64_t() const noexcept(false);
-  really_inline operator double() const noexcept(false);
-  really_inline operator dom::array() const noexcept(false);
-  really_inline operator dom::object() const noexcept(false);
+  simdjson_really_inline operator bool() const noexcept(false);
+  simdjson_really_inline explicit operator const char*() const noexcept(false);
+  simdjson_really_inline operator std::string_view() const noexcept(false);
+  simdjson_really_inline operator uint64_t() const noexcept(false);
+  simdjson_really_inline operator int64_t() const noexcept(false);
+  simdjson_really_inline operator double() const noexcept(false);
+  simdjson_really_inline operator dom::array() const noexcept(false);
+  simdjson_really_inline operator dom::object() const noexcept(false);
 
-  really_inline dom::array::iterator begin() const noexcept(false);
-  really_inline dom::array::iterator end() const noexcept(false);
+  simdjson_really_inline dom::array::iterator begin() const noexcept(false);
+  simdjson_really_inline dom::array::iterator end() const noexcept(false);
 #endif // SIMDJSON_EXCEPTIONS
 };
 
@@ -537,7 +569,7 @@ public:
  *        underlying output stream, that error will be propagated (simdjson_error will not be
  *        thrown).
  */
-really_inline std::ostream& operator<<(std::ostream& out, const simdjson_result<dom::element> &value) noexcept(false);
+simdjson_really_inline std::ostream& operator<<(std::ostream& out, const simdjson_result<dom::element> &value) noexcept(false);
 #endif
 
 } // namespace simdjson

include/simdjson/dom/jsonparser.h

@@ -59,7 +59,7 @@ inline int json_parse(const padded_string &s, dom::parser &parser) noexcept {
 }
 
 [[deprecated("Use parser.parse() instead")]]
-WARN_UNUSED inline dom::parser build_parsed_json(const uint8_t *buf, size_t len, bool realloc_if_needed = true) noexcept {
+SIMDJSON_WARN_UNUSED inline dom::parser build_parsed_json(const uint8_t *buf, size_t len, bool realloc_if_needed = true) noexcept {
   dom::parser parser;
   error_code code = parser.parse(buf, len, realloc_if_needed).error();
   // The deprecated json_parse API is a signal that the user plans to *use* the error code / valid
@@ -71,7 +71,7 @@ WARN_UNUSED inline dom::parser build_parsed_json(const uint8_t *buf, size_t len,
   return parser;
 }
 [[deprecated("Use parser.parse() instead")]]
-WARN_UNUSED inline dom::parser build_parsed_json(const char *buf, size_t len, bool realloc_if_needed = true) noexcept {
+SIMDJSON_WARN_UNUSED inline dom::parser build_parsed_json(const char *buf, size_t len, bool realloc_if_needed = true) noexcept {
   dom::parser parser;
   error_code code = parser.parse(buf, len, realloc_if_needed).error();
   // The deprecated json_parse API is a signal that the user plans to *use* the error code / valid
@@ -83,7 +83,7 @@ WARN_UNUSED inline dom::parser build_parsed_json(const char *buf, size_t len, bo
   return parser;
 }
 [[deprecated("Use parser.parse() instead")]]
-WARN_UNUSED inline dom::parser build_parsed_json(const std::string &s, bool realloc_if_needed = true) noexcept {
+SIMDJSON_WARN_UNUSED inline dom::parser build_parsed_json(const std::string &s, bool realloc_if_needed = true) noexcept {
   dom::parser parser;
   error_code code = parser.parse(s.data(), s.length(), realloc_if_needed).error();
   // The deprecated json_parse API is a signal that the user plans to *use* the error code / valid
@@ -95,7 +95,7 @@ WARN_UNUSED inline dom::parser build_parsed_json(const std::string &s, bool real
   return parser;
 }
 [[deprecated("Use parser.parse() instead")]]
-WARN_UNUSED inline dom::parser build_parsed_json(const padded_string &s) noexcept {
+SIMDJSON_WARN_UNUSED inline dom::parser build_parsed_json(const padded_string &s) noexcept {
   dom::parser parser;
   error_code code = parser.parse(s).error();
   // The deprecated json_parse API is a signal that the user plans to *use* the error code / valid

include/simdjson/dom/object-inl.h

@@ -12,14 +12,14 @@ namespace simdjson {
 //
 // simdjson_result<dom::object> inline implementation
 //
-really_inline simdjson_result<dom::object>::simdjson_result() noexcept
+simdjson_really_inline simdjson_result<dom::object>::simdjson_result() noexcept
     : internal::simdjson_result_base<dom::object>() {}
-really_inline simdjson_result<dom::object>::simdjson_result(dom::object value) noexcept
+simdjson_really_inline simdjson_result<dom::object>::simdjson_result(dom::object value) noexcept
     : internal::simdjson_result_base<dom::object>(std::forward<dom::object>(value)) {}
-really_inline simdjson_result<dom::object>::simdjson_result(error_code error) noexcept
+simdjson_really_inline simdjson_result<dom::object>::simdjson_result(error_code error) noexcept
     : internal::simdjson_result_base<dom::object>(error) {}
 
-inline simdjson_result<dom::element> simdjson_result<dom::object>::operator[](const std::string_view &key) const noexcept {
+inline simdjson_result<dom::element> simdjson_result<dom::object>::operator[](std::string_view key) const noexcept {
   if (error()) { return error(); }
   return first[key];
 }
@@ -27,15 +27,15 @@ inline simdjson_result<dom::element> simdjson_result<dom::object>::operator[](co
   if (error()) { return error(); }
   return first[key];
 }
-inline simdjson_result<dom::element> simdjson_result<dom::object>::at(const std::string_view &json_pointer) const noexcept {
+inline simdjson_result<dom::element> simdjson_result<dom::object>::at_pointer(std::string_view json_pointer) const noexcept {
   if (error()) { return error(); }
-  return first.at(json_pointer);
+  return first.at_pointer(json_pointer);
 }
-inline simdjson_result<dom::element> simdjson_result<dom::object>::at_key(const std::string_view &key) const noexcept {
+inline simdjson_result<dom::element> simdjson_result<dom::object>::at_key(std::string_view key) const noexcept {
   if (error()) { return error(); }
   return first.at_key(key);
 }
-inline simdjson_result<dom::element> simdjson_result<dom::object>::at_key_case_insensitive(const std::string_view &key) const noexcept {
+inline simdjson_result<dom::element> simdjson_result<dom::object>::at_key_case_insensitive(std::string_view key) const noexcept {
   if (error()) { return error(); }
   return first.at_key_case_insensitive(key);
 }
@@ -62,8 +62,8 @@ namespace dom {
 //
 // object inline implementation
 //
-really_inline object::object() noexcept : tape{} {}
-really_inline object::object(const internal::tape_ref &_tape) noexcept : tape{_tape} { }
+simdjson_really_inline object::object() noexcept : tape{} {}
+simdjson_really_inline object::object(const internal::tape_ref &_tape) noexcept : tape{_tape} { }
 inline object::iterator object::begin() const noexcept {
   return internal::tape_ref(tape.doc, tape.json_index + 1);
 }
@@ -74,16 +74,23 @@ inline size_t object::size() const noexcept {
   return tape.scope_count();
 }
 
-inline simdjson_result<element> object::operator[](const std::string_view &key) const noexcept {
+inline simdjson_result<element> object::operator[](std::string_view key) const noexcept {
   return at_key(key);
 }
 inline simdjson_result<element> object::operator[](const char *key) const noexcept {
   return at_key(key);
 }
-inline simdjson_result<element> object::at(const std::string_view &json_pointer) const noexcept {
+inline simdjson_result<element> object::at_pointer(std::string_view json_pointer) const noexcept {
+  if(json_pointer[0] != '/') {
+    if(json_pointer.size() == 0) { // an empty string means that we return the current node
+      return element(this->tape); // copy the current node
+    } else { // otherwise there is an error
+      return INVALID_JSON_POINTER;
+    }
+  }
+  json_pointer = json_pointer.substr(1);
   size_t slash = json_pointer.find('/');
   std::string_view key = json_pointer.substr(0, slash);
-
   // Grab the child with the given key
   simdjson_result<element> child;
 
@@ -109,15 +116,17 @@ inline simdjson_result<element> object::at(const std::string_view &json_pointer)
   } else {
     child = at_key(key);
   }
-
+  if(child.error()) {
+    return child; // we do not continue if there was an error
+  }
   // If there is a /, we have to recurse and look up more of the path
   if (slash != std::string_view::npos) {
-    child = child.at(json_pointer.substr(slash+1));
+    child = child.at_pointer(json_pointer.substr(slash));
   }
-
   return child;
 }
-inline simdjson_result<element> object::at_key(const std::string_view &key) const noexcept {
+
+inline simdjson_result<element> object::at_key(std::string_view key) const noexcept {
   iterator end_field = end();
   for (iterator field = begin(); field != end_field; ++field) {
     if (field.key_equals(key)) {
@@ -129,7 +138,7 @@ inline simdjson_result<element> object::at_key(const std::string_view &key) cons
 // In case you wonder why we need this, please see
 // https://github.com/simdjson/simdjson/issues/323
 // People do seek keys in a case-insensitive manner.
-inline simdjson_result<element> object::at_key_case_insensitive(const std::string_view &key) const noexcept {
+inline simdjson_result<element> object::at_key_case_insensitive(std::string_view key) const noexcept {
   iterator end_field = end();
   for (iterator field = begin(); field != end_field; ++field) {
     if (field.key_equals_case_insensitive(key)) {
@@ -142,18 +151,38 @@ inline simdjson_result<element> object::at_key_case_insensitive(const std::strin
 //
 // object::iterator inline implementation
 //
-really_inline object::iterator::iterator(const internal::tape_ref &_tape) noexcept : tape{_tape} { }
+simdjson_really_inline object::iterator::iterator(const internal::tape_ref &_tape) noexcept : tape{_tape} { }
 inline const key_value_pair object::iterator::operator*() const noexcept {
   return key_value_pair(key(), value());
 }
 inline bool object::iterator::operator!=(const object::iterator& other) const noexcept {
   return tape.json_index != other.tape.json_index;
 }
+inline bool object::iterator::operator==(const object::iterator& other) const noexcept {
+  return tape.json_index == other.tape.json_index;
+}
+inline bool object::iterator::operator<(const object::iterator& other) const noexcept {
+  return tape.json_index < other.tape.json_index;
+}
+inline bool object::iterator::operator<=(const object::iterator& other) const noexcept {
+  return tape.json_index <= other.tape.json_index;
+}
+inline bool object::iterator::operator>=(const object::iterator& other) const noexcept {
+  return tape.json_index >= other.tape.json_index;
+}
+inline bool object::iterator::operator>(const object::iterator& other) const noexcept {
+  return tape.json_index > other.tape.json_index;
+}
 inline object::iterator& object::iterator::operator++() noexcept {
   tape.json_index++;
   tape.json_index = tape.after_element();
   return *this;
 }
+inline object::iterator object::iterator::operator++(int) noexcept {
+  object::iterator out = *this;
+  ++*this;
+  return out;
+}
 inline std::string_view object::iterator::key() const noexcept {
   return tape.get_string_view();
 }
@@ -180,7 +209,7 @@ inline element object::iterator::value() const noexcept {
  * on the long run.
  */
 
-inline bool object::iterator::key_equals(const std::string_view & o) const noexcept {
+inline bool object::iterator::key_equals(std::string_view o) const noexcept {
   // We use the fact that the key length can be computed quickly
   // without access to the string buffer.
   const uint32_t len = key_length();
@@ -191,7 +220,7 @@ inline bool object::iterator::key_equals(const std::string_view & o) const noexc
   return false;
 }
 
-inline bool object::iterator::key_equals_case_insensitive(const std::string_view & o) const noexcept {
+inline bool object::iterator::key_equals_case_insensitive(std::string_view o) const noexcept {
   // We use the fact that the key length can be computed quickly
   // without access to the string buffer.
   const uint32_t len = key_length();
@@ -206,7 +235,7 @@ inline bool object::iterator::key_equals_case_insensitive(const std::string_view
 //
 // key_value_pair inline implementation
 //
-inline key_value_pair::key_value_pair(const std::string_view &_key, element _value) noexcept :
+inline key_value_pair::key_value_pair(std::string_view _key, element _value) noexcept :
   key(_key), value(_value) {}
 
 inline std::ostream& operator<<(std::ostream& out, const object &value) {
@@ -252,4 +281,13 @@ inline std::ostream& operator<<(std::ostream& out, const simdjson_result<dom::ob
 
 } // namespace simdjson
 
+#if defined(__cpp_lib_ranges)
+static_assert(std::ranges::view<simdjson::dom::object>);
+static_assert(std::ranges::sized_range<simdjson::dom::object>);
+#if SIMDJSON_EXCEPTIONS
+static_assert(std::ranges::view<simdjson::simdjson_result<simdjson::dom::object>>);
+static_assert(std::ranges::sized_range<simdjson::simdjson_result<simdjson::dom::object>>);
+#endif // SIMDJSON_EXCEPTIONS
+#endif // defined(__cpp_lib_ranges)
+
 #endif // SIMDJSON_INLINE_OBJECT_H

include/simdjson/dom/object.h

@@ -20,14 +20,17 @@ class key_value_pair;
 class object {
 public:
   /** Create a new, invalid object */
-  really_inline object() noexcept;
+  simdjson_really_inline object() noexcept;
 
   class iterator {
   public:
+    using value_type = key_value_pair;
+    using difference_type = std::ptrdiff_t;
+
     /**
      * Get the actual key/value pair
      */
-    inline const key_value_pair operator*() const noexcept;
+    inline const value_type operator*() const noexcept;
     /**
      * Get the next key/value pair.
      *
@@ -36,11 +39,24 @@ public:
      */
     inline iterator& operator++() noexcept;
     /**
-     * Check if these key value pairs come from the same place in the JSON.
+     * Get the next key/value pair.
+     *
+     * Part of the std::iterator interface.
+     *
+     */
+    inline iterator operator++(int) noexcept;
+    /**
+     * Check if these values come from the same place in the JSON.
      *
      * Part of the std::iterator interface.
      */
     inline bool operator!=(const iterator& other) const noexcept;
+    inline bool operator==(const iterator& other) const noexcept;
+
+    inline bool operator<(const iterator& other) const noexcept;
+    inline bool operator<=(const iterator& other) const noexcept;
+    inline bool operator>=(const iterator& other) const noexcept;
+    inline bool operator>(const iterator& other) const noexcept;
     /**
      * Get the key of this key/value pair.
      */
@@ -54,13 +70,13 @@ public:
      * Returns true if the key in this key/value pair is equal
      * to the provided string_view.
      */
-    inline bool key_equals(const std::string_view & o) const noexcept;
+    inline bool key_equals(std::string_view o) const noexcept;
     /**
      * Returns true if the key in this key/value pair is equal
      * to the provided string_view in a case-insensitive manner.
      * Case comparisons may only be handled correctly for ASCII strings.
      */
-    inline bool key_equals_case_insensitive(const std::string_view & o) const noexcept;
+    inline bool key_equals_case_insensitive(std::string_view o) const noexcept;
     /**
      * Get the key of this key/value pair.
      */
@@ -69,8 +85,12 @@ public:
      * Get the value of this key/value pair.
      */
     inline element value() const noexcept;
+
+    iterator() noexcept = default;
+    iterator(const iterator&) noexcept = default;
+    iterator& operator=(const iterator&) noexcept = default;
   private:
-    really_inline iterator(const internal::tape_ref &tape) noexcept;
+    simdjson_really_inline iterator(const internal::tape_ref &tape) noexcept;
 
     internal::tape_ref tape;
 
@@ -110,7 +130,7 @@ public:
    *         - NO_SUCH_FIELD if the field does not exist in the object
    *         - INCORRECT_TYPE if this is not an object
    */
-  inline simdjson_result<element> operator[](const std::string_view &key) const noexcept;
+  inline simdjson_result<element> operator[](std::string_view key) const noexcept;
 
   /**
    * Get the value associated with the given key.
@@ -130,12 +150,21 @@ public:
   inline simdjson_result<element> operator[](const char *key) const noexcept;
 
   /**
-   * Get the value associated with the given JSON pointer.
+   * Get the value associated with the given JSON pointer. We use the RFC 6901
+   * https://tools.ietf.org/html/rfc6901 standard, interpreting the current node
+   * as the root of its own JSON document.
    *
    *   dom::parser parser;
    *   object obj = parser.parse(R"({ "foo": { "a": [ 10, 20, 30 ] }})"_padded);
-   *   obj.at("foo/a/1") == 20
-   *   obj.at("foo")["a"].at(1) == 20
+   *   obj.at_pointer("/foo/a/1") == 20
+   *   obj.at_pointer("/foo")["a"].at(1) == 20
+   *
+   * It is allowed for a key to be the empty string:
+   *
+   *   dom::parser parser;
+   *   object obj = parser.parse(R"({ "": { "a": [ 10, 20, 30 ] }})"_padded);
+   *   obj.at_pointer("//a/1") == 20
+   *   obj.at_pointer("/")["a"].at(1) == 20
    *
    * @return The value associated with the given JSON pointer, or:
    *         - NO_SUCH_FIELD if a field does not exist in an object
@@ -143,7 +172,7 @@ public:
    *         - INCORRECT_TYPE if a non-integer is used to access an array
    *         - INVALID_JSON_POINTER if the JSON pointer is invalid and cannot be parsed
    */
-  inline simdjson_result<element> at(const std::string_view &json_pointer) const noexcept;
+  inline simdjson_result<element> at_pointer(std::string_view json_pointer) const noexcept;
 
   /**
    * Get the value associated with the given key.
@@ -159,7 +188,7 @@ public:
    * @return The value associated with this field, or:
    *         - NO_SUCH_FIELD if the field does not exist in the object
    */
-  inline simdjson_result<element> at_key(const std::string_view &key) const noexcept;
+  inline simdjson_result<element> at_key(std::string_view key) const noexcept;
 
   /**
    * Get the value associated with the given key in a case-insensitive manner.
@@ -172,10 +201,10 @@ public:
    * @return The value associated with this field, or:
    *         - NO_SUCH_FIELD if the field does not exist in the object
    */
-  inline simdjson_result<element> at_key_case_insensitive(const std::string_view &key) const noexcept;
+  inline simdjson_result<element> at_key_case_insensitive(std::string_view key) const noexcept;
 
 private:
-  really_inline object(const internal::tape_ref &tape) noexcept;
+  simdjson_really_inline object(const internal::tape_ref &tape) noexcept;
 
   internal::tape_ref tape;
 
@@ -196,7 +225,7 @@ public:
   element value;
 
 private:
-  really_inline key_value_pair(const std::string_view &_key, element _value) noexcept;
+  simdjson_really_inline key_value_pair(std::string_view _key, element _value) noexcept;
   friend class object;
 };
 
@@ -227,15 +256,15 @@ inline std::ostream& operator<<(std::ostream& out, const key_value_pair &value);
 template<>
 struct simdjson_result<dom::object> : public internal::simdjson_result_base<dom::object> {
 public:
-  really_inline simdjson_result() noexcept; ///< @private
-  really_inline simdjson_result(dom::object value) noexcept; ///< @private
-  really_inline simdjson_result(error_code error) noexcept; ///< @private
+  simdjson_really_inline simdjson_result() noexcept; ///< @private
+  simdjson_really_inline simdjson_result(dom::object value) noexcept; ///< @private
+  simdjson_really_inline simdjson_result(error_code error) noexcept; ///< @private
 
-  inline simdjson_result<dom::element> operator[](const std::string_view &key) const noexcept;
+  inline simdjson_result<dom::element> operator[](std::string_view key) const noexcept;
   inline simdjson_result<dom::element> operator[](const char *key) const noexcept;
-  inline simdjson_result<dom::element> at(const std::string_view &json_pointer) const noexcept;
-  inline simdjson_result<dom::element> at_key(const std::string_view &key) const noexcept;
-  inline simdjson_result<dom::element> at_key_case_insensitive(const std::string_view &key) const noexcept;
+  inline simdjson_result<dom::element> at_pointer(std::string_view json_pointer) const noexcept;
+  inline simdjson_result<dom::element> at_key(std::string_view key) const noexcept;
+  inline simdjson_result<dom::element> at_key_case_insensitive(std::string_view key) const noexcept;
 
 #if SIMDJSON_EXCEPTIONS
   inline dom::object::iterator begin() const noexcept(false);
@@ -261,4 +290,19 @@ inline std::ostream& operator<<(std::ostream& out, const simdjson_result<dom::ob
 
 } // namespace simdjson
 
+#if defined(__cpp_lib_ranges)
+#include <ranges>
+
+namespace std {
+namespace ranges {
+template<>
+inline constexpr bool enable_view<simdjson::dom::object> = true;
+#if SIMDJSON_EXCEPTIONS
+template<>
+inline constexpr bool enable_view<simdjson::simdjson_result<simdjson::dom::object>> = true;
+#endif // SIMDJSON_EXCEPTIONS
+} // namespace ranges
+} // namespace std
+#endif // defined(__cpp_lib_ranges)
+
 #endif // SIMDJSON_DOM_OBJECT_H

include/simdjson/dom/parsedjson_iterator-inl.h

@@ -13,7 +13,7 @@ SIMDJSON_DISABLE_DEPRECATED_WARNING
 
 // Because of template weirdness, the actual class definition is inline in the document class
 
-WARN_UNUSED bool dom::parser::Iterator::is_ok() const {
+SIMDJSON_WARN_UNUSED bool dom::parser::Iterator::is_ok() const {
   return location < tape_length;
 }
 

include/simdjson/dom/parser-inl.h

@@ -15,12 +15,12 @@ namespace dom {
 //
 // parser inline implementation
 //
-really_inline parser::parser(size_t max_capacity) noexcept
+simdjson_really_inline parser::parser(size_t max_capacity) noexcept
   : _max_capacity{max_capacity},
-    loaded_bytes(nullptr, &aligned_free_char) {
+    loaded_bytes(nullptr) {
 }
-really_inline parser::parser(parser &&other) noexcept = default;
-really_inline parser &parser::operator=(parser &&other) noexcept = default;
+simdjson_really_inline parser::parser(parser &&other) noexcept = default;
+simdjson_really_inline parser &parser::operator=(parser &&other) noexcept = default;
 
 inline bool parser::is_valid() const noexcept { return valid; }
 inline int parser::get_error_code() const noexcept { return error; }
@@ -93,30 +93,25 @@ inline simdjson_result<document_stream> parser::load_many(const std::string &pat
 inline simdjson_result<element> parser::parse(const uint8_t *buf, size_t len, bool realloc_if_needed) & noexcept {
   error_code _error = ensure_capacity(len);
   if (_error) { return _error; }
+  std::unique_ptr<uint8_t[]> tmp_buf;
 
   if (realloc_if_needed) {
-    const uint8_t *tmp_buf = buf;
-    buf = (uint8_t *)internal::allocate_padded_buffer(len);
-    if (buf == nullptr)
-      return MEMALLOC;
-    memcpy((void *)buf, tmp_buf, len);
-  }
-
-  _error = implementation->parse(buf, len, doc);
-  if (realloc_if_needed) {
-    aligned_free((void *)buf); // must free before we exit
+    tmp_buf.reset((uint8_t *)internal::allocate_padded_buffer(len));
+    if (tmp_buf.get() == nullptr) { return MEMALLOC; }
+    memcpy((void *)tmp_buf.get(), buf, len);
   }
+  _error = implementation->parse(realloc_if_needed ? tmp_buf.get() : buf, len, doc);
   if (_error) { return _error; }
 
   return doc.root();
 }
-really_inline simdjson_result<element> parser::parse(const char *buf, size_t len, bool realloc_if_needed) & noexcept {
+simdjson_really_inline simdjson_result<element> parser::parse(const char *buf, size_t len, bool realloc_if_needed) & noexcept {
   return parse((const uint8_t *)buf, len, realloc_if_needed);
 }
-really_inline simdjson_result<element> parser::parse(const std::string &s) & noexcept {
+simdjson_really_inline simdjson_result<element> parser::parse(const std::string &s) & noexcept {
   return parse(s.data(), s.length(), s.capacity() - s.length() < SIMDJSON_PADDING);
 }
-really_inline simdjson_result<element> parser::parse(const padded_string &s) & noexcept {
+simdjson_really_inline simdjson_result<element> parser::parse(const padded_string &s) & noexcept {
   return parse(s.data(), s.length(), false);
 }
 
@@ -133,17 +128,17 @@ inline simdjson_result<document_stream> parser::parse_many(const padded_string &
   return parse_many(s.data(), s.length(), batch_size);
 }
 
-really_inline size_t parser::capacity() const noexcept {
+simdjson_really_inline size_t parser::capacity() const noexcept {
   return implementation ? implementation->capacity() : 0;
 }
-really_inline size_t parser::max_capacity() const noexcept {
+simdjson_really_inline size_t parser::max_capacity() const noexcept {
   return _max_capacity;
 }
-really_inline size_t parser::max_depth() const noexcept {
+simdjson_really_inline size_t parser::max_depth() const noexcept {
   return implementation ? implementation->max_depth() : DEFAULT_MAX_DEPTH;
 }
 
-WARN_UNUSED
+SIMDJSON_WARN_UNUSED
 inline error_code parser::allocate(size_t capacity, size_t max_depth) noexcept {
   //
   // Reallocate implementation and document if needed
@@ -169,7 +164,7 @@ inline error_code parser::allocate(size_t capacity, size_t max_depth) noexcept {
   return SUCCESS;
 }
 
-WARN_UNUSED
+SIMDJSON_WARN_UNUSED
 inline bool parser::allocate_capacity(size_t capacity, size_t max_depth) noexcept {
   return !allocate(capacity, max_depth);
 }
@@ -178,7 +173,7 @@ inline error_code parser::ensure_capacity(size_t desired_capacity) noexcept {
   // If we don't have enough capacity, (try to) automatically bump it.
   // If the document was taken, reallocate that too.
   // Both in one if statement to minimize unlikely branching.
-  if (unlikely(capacity() < desired_capacity || !doc.tape)) {
+  if (simdjson_unlikely(capacity() < desired_capacity || !doc.tape)) {
     if (desired_capacity > max_capacity()) {
       return error = CAPACITY;
     }
@@ -188,7 +183,7 @@ inline error_code parser::ensure_capacity(size_t desired_capacity) noexcept {
   return SUCCESS;
 }
 
-really_inline void parser::set_max_capacity(size_t max_capacity) noexcept {
+simdjson_really_inline void parser::set_max_capacity(size_t max_capacity) noexcept {
   _max_capacity = max_capacity;
 }
 

include/simdjson/dom/parser.h

@@ -46,20 +46,20 @@ public:
   *    to allocate an initial capacity, call allocate() after constructing the parser.
   *    Defaults to SIMDJSON_MAXSIZE_BYTES (the largest single document simdjson can process).
   */
-  really_inline explicit parser(size_t max_capacity = SIMDJSON_MAXSIZE_BYTES) noexcept;
+  simdjson_really_inline explicit parser(size_t max_capacity = SIMDJSON_MAXSIZE_BYTES) noexcept;
   /**
    * Take another parser's buffers and state.
    *
    * @param other The parser to take. Its capacity is zeroed.
    */
-  really_inline parser(parser &&other) noexcept;
+  simdjson_really_inline parser(parser &&other) noexcept;
   parser(const parser &) = delete; ///< @private Disallow copying
   /**
    * Take another parser's buffers and state.
    *
    * @param other The parser to take. Its capacity is zeroed.
    */
-  really_inline parser &operator=(parser &&other) noexcept;
+  simdjson_really_inline parser &operator=(parser &&other) noexcept;
   parser &operator=(const parser &) = delete; ///< @private Disallow copying
 
   /** Deallocate the JSON parser. */
@@ -129,17 +129,17 @@ public:
   inline simdjson_result<element> parse(const uint8_t *buf, size_t len, bool realloc_if_needed = true) & noexcept;
   inline simdjson_result<element> parse(const uint8_t *buf, size_t len, bool realloc_if_needed = true) && =delete;
   /** @overload parse(const uint8_t *buf, size_t len, bool realloc_if_needed) */
-  really_inline simdjson_result<element> parse(const char *buf, size_t len, bool realloc_if_needed = true) & noexcept;
-  really_inline simdjson_result<element> parse(const char *buf, size_t len, bool realloc_if_needed = true) && =delete;
+  simdjson_really_inline simdjson_result<element> parse(const char *buf, size_t len, bool realloc_if_needed = true) & noexcept;
+  simdjson_really_inline simdjson_result<element> parse(const char *buf, size_t len, bool realloc_if_needed = true) && =delete;
   /** @overload parse(const uint8_t *buf, size_t len, bool realloc_if_needed) */
-  really_inline simdjson_result<element> parse(const std::string &s) & noexcept;
-  really_inline simdjson_result<element> parse(const std::string &s) && =delete;
+  simdjson_really_inline simdjson_result<element> parse(const std::string &s) & noexcept;
+  simdjson_really_inline simdjson_result<element> parse(const std::string &s) && =delete;
   /** @overload parse(const uint8_t *buf, size_t len, bool realloc_if_needed) */
-  really_inline simdjson_result<element> parse(const padded_string &s) & noexcept;
-  really_inline simdjson_result<element> parse(const padded_string &s) && =delete;
+  simdjson_really_inline simdjson_result<element> parse(const padded_string &s) & noexcept;
+  simdjson_really_inline simdjson_result<element> parse(const padded_string &s) && =delete;
 
   /** @private We do not want to allow implicit conversion from C string to std::string. */
-  really_inline simdjson_result<element> parse(const char *buf) noexcept = delete;
+  simdjson_really_inline simdjson_result<element> parse(const char *buf) noexcept = delete;
 
   /**
    * Load a file containing many JSON documents.
@@ -196,7 +196,7 @@ public:
    * @param batch_size The batch size to use. MUST be larger than the largest document. The sweet
    *                   spot is cache-related: small enough to fit in cache, yet big enough to
    *                   parse as many documents as possible in one tight loop.
-   *                   Defaults to 10MB, which has been a reasonable sweet spot in our tests.
+   *                   Defaults to 1MB (as simdjson::dom::DEFAULT_BATCH_SIZE), which has been a reasonable sweet spot in our tests.
    * @return The stream, or an error. An empty input will yield 0 documents rather than an EMPTY error. Errors:
    *         - IO_ERROR if there was an error opening or reading the file.
    *         - MEMALLOC if the parser does not have enough capacity and memory allocation fails.
@@ -291,7 +291,7 @@ public:
    * @param max_depth The new max_depth. Defaults to DEFAULT_MAX_DEPTH.
    * @return The error, if there is one.
    */
-  WARN_UNUSED inline error_code allocate(size_t capacity, size_t max_depth = DEFAULT_MAX_DEPTH) noexcept;
+  SIMDJSON_WARN_UNUSED inline error_code allocate(size_t capacity, size_t max_depth = DEFAULT_MAX_DEPTH) noexcept;
 
   /**
    * @private deprecated because it returns bool instead of error_code, which is our standard for
@@ -305,14 +305,14 @@ public:
    * @return true if successful, false if allocation failed.
    */
   [[deprecated("Use allocate() instead.")]]
-  WARN_UNUSED inline bool allocate_capacity(size_t capacity, size_t max_depth = DEFAULT_MAX_DEPTH) noexcept;
+  SIMDJSON_WARN_UNUSED inline bool allocate_capacity(size_t capacity, size_t max_depth = DEFAULT_MAX_DEPTH) noexcept;
 
   /**
    * The largest document this parser can support without reallocating.
    *
    * @return Current capacity, in bytes.
    */
-  really_inline size_t capacity() const noexcept;
+  simdjson_really_inline size_t capacity() const noexcept;
 
   /**
    * The largest document this parser can automatically support.
@@ -321,14 +321,14 @@ public:
    *
    * @return Maximum capacity, in bytes.
    */
-  really_inline size_t max_capacity() const noexcept;
+  simdjson_really_inline size_t max_capacity() const noexcept;
 
   /**
    * The maximum level of nested object and arrays supported by this parser.
    *
    * @return Maximum depth, in bytes.
    */
-  really_inline size_t max_depth() const noexcept;
+  simdjson_really_inline size_t max_depth() const noexcept;
 
   /**
    * Set max_capacity. This is the largest document this parser can automatically support.
@@ -340,8 +340,16 @@ public:
    *
    * @param max_capacity The new maximum capacity, in bytes.
    */
-  really_inline void set_max_capacity(size_t max_capacity) noexcept;
+  simdjson_really_inline void set_max_capacity(size_t max_capacity) noexcept;
 
+#ifdef SIMDJSON_THREADS_ENABLED
+  /**
+   * The parser instance can use threads when they are available to speed up some
+   * operations. It is enabled by default. Changing this attribute will change the
+   * behavior of the parser for future operations.
+   */
+  bool threaded{true};
+#endif
   /** @private Use the new DOM API instead */
   class Iterator;
   /** @private Use simdjson_error instead */
@@ -380,6 +388,7 @@ public:
   /** @private Private and deprecated: use `parser.parse(...).doc.dump_raw_tape()` instead */
   inline bool dump_raw_tape(std::ostream &os) const noexcept;
 
+
 private:
   /**
    * The maximum document length this parser will automatically support.
@@ -391,7 +400,7 @@ private:
   /**
    * The loaded buffer (reused each time load() is called)
    */
-  std::unique_ptr<char[], decltype(&aligned_free_char)> loaded_bytes;
+  std::unique_ptr<char[]> loaded_bytes;
 
   /** Capacity of loaded_bytes buffer. */
   size_t _loaded_bytes_capacity{0};
@@ -421,6 +430,8 @@ private:
 
   friend class parser::Iterator;
   friend class document_stream;
+
+
 }; // class parser
 
 } // namespace dom

include/simdjson/error-inl.h

@@ -11,7 +11,7 @@ namespace internal {
   // We store the error code so we can validate the error message is associated with the right code
   struct error_code_info {
     error_code code;
-    std::string message;
+    const char* message; // do not use a fancy std::string where a simple C string will do (no alloc, no destructor)
   };
   // These MUST match the codes in error_code. We check this constraint in basictests.
   extern SIMDJSON_DLLIMPORTEXPORT const error_code_info error_codes[];
@@ -20,10 +20,11 @@ namespace internal {
 
 inline const char *error_message(error_code error) noexcept {
   // If you're using error_code, we're trusting you got it from the enum.
-  return internal::error_codes[int(error)].message.c_str();
+  return internal::error_codes[int(error)].message;
 }
 
-inline const std::string &error_message(int error) noexcept {
+// deprecated function
+inline const std::string error_message(int error) noexcept {
   if (error < 0 || error >= error_code::NUM_ERROR_CODES) {
     return internal::error_codes[UNEXPECTED_ERROR].message;
   }
@@ -41,7 +42,7 @@ namespace internal {
 //
 
 template<typename T>
-really_inline void simdjson_result_base<T>::tie(T &value, error_code &error) && noexcept {
+simdjson_really_inline void simdjson_result_base<T>::tie(T &value, error_code &error) && noexcept {
   // on the clang compiler that comes with current macOS (Apple clang version 11.0.0),
   // tie(width, error) = size["w"].get<uint64_t>();
   // fails with "error: no viable overloaded '='""
@@ -52,49 +53,49 @@ really_inline void simdjson_result_base<T>::tie(T &value, error_code &error) &&
 }
 
 template<typename T>
-WARN_UNUSED really_inline error_code simdjson_result_base<T>::get(T &value) && noexcept {
+SIMDJSON_WARN_UNUSED simdjson_really_inline error_code simdjson_result_base<T>::get(T &value) && noexcept {
   error_code error;
   std::forward<simdjson_result_base<T>>(*this).tie(value, error);
   return error;
 }
 
 template<typename T>
-really_inline error_code simdjson_result_base<T>::error() const noexcept {
+simdjson_really_inline error_code simdjson_result_base<T>::error() const noexcept {
   return this->second;
 }
 
 #if SIMDJSON_EXCEPTIONS
 
 template<typename T>
-really_inline T& simdjson_result_base<T>::value() noexcept(false) {
+simdjson_really_inline T& simdjson_result_base<T>::value() noexcept(false) {
   if (error()) { throw simdjson_error(error()); }
   return this->first;
 }
 
 template<typename T>
-really_inline T&& simdjson_result_base<T>::take_value() && noexcept(false) {
+simdjson_really_inline T&& simdjson_result_base<T>::take_value() && noexcept(false) {
   if (error()) { throw simdjson_error(error()); }
   return std::forward<T>(this->first);
 }
 
 template<typename T>
-really_inline simdjson_result_base<T>::operator T&&() && noexcept(false) {
+simdjson_really_inline simdjson_result_base<T>::operator T&&() && noexcept(false) {
   return std::forward<simdjson_result_base<T>>(*this).take_value();
 }
 
 #endif // SIMDJSON_EXCEPTIONS
 
 template<typename T>
-really_inline simdjson_result_base<T>::simdjson_result_base(T &&value, error_code error) noexcept
+simdjson_really_inline simdjson_result_base<T>::simdjson_result_base(T &&value, error_code error) noexcept
     : std::pair<T, error_code>(std::forward<T>(value), error) {}
 template<typename T>
-really_inline simdjson_result_base<T>::simdjson_result_base(error_code error) noexcept
+simdjson_really_inline simdjson_result_base<T>::simdjson_result_base(error_code error) noexcept
     : simdjson_result_base(T{}, error) {}
 template<typename T>
-really_inline simdjson_result_base<T>::simdjson_result_base(T &&value) noexcept
+simdjson_really_inline simdjson_result_base<T>::simdjson_result_base(T &&value) noexcept
     : simdjson_result_base(std::forward<T>(value), SUCCESS) {}
 template<typename T>
-really_inline simdjson_result_base<T>::simdjson_result_base() noexcept
+simdjson_really_inline simdjson_result_base<T>::simdjson_result_base() noexcept
     : simdjson_result_base(T{}, UNINITIALIZED) {}
 
 } // namespace internal
@@ -104,50 +105,50 @@ really_inline simdjson_result_base<T>::simdjson_result_base() noexcept
 ///
 
 template<typename T>
-really_inline void simdjson_result<T>::tie(T &value, error_code &error) && noexcept {
+simdjson_really_inline void simdjson_result<T>::tie(T &value, error_code &error) && noexcept {
   std::forward<internal::simdjson_result_base<T>>(*this).tie(value, error);
 }
 
 template<typename T>
-WARN_UNUSED really_inline error_code simdjson_result<T>::get(T &value) && noexcept {
+SIMDJSON_WARN_UNUSED simdjson_really_inline error_code simdjson_result<T>::get(T &value) && noexcept {
   return std::forward<internal::simdjson_result_base<T>>(*this).get(value);
 }
 
 template<typename T>
-really_inline error_code simdjson_result<T>::error() const noexcept {
+simdjson_really_inline error_code simdjson_result<T>::error() const noexcept {
   return internal::simdjson_result_base<T>::error();
 }
 
 #if SIMDJSON_EXCEPTIONS
 
 template<typename T>
-really_inline T& simdjson_result<T>::value() noexcept(false) {
+simdjson_really_inline T& simdjson_result<T>::value() noexcept(false) {
   return internal::simdjson_result_base<T>::value();
 }
 
 template<typename T>
-really_inline T&& simdjson_result<T>::take_value() && noexcept(false) {
+simdjson_really_inline T&& simdjson_result<T>::take_value() && noexcept(false) {
   return std::forward<internal::simdjson_result_base<T>>(*this).take_value();
 }
 
 template<typename T>
-really_inline simdjson_result<T>::operator T&&() && noexcept(false) {
+simdjson_really_inline simdjson_result<T>::operator T&&() && noexcept(false) {
   return std::forward<internal::simdjson_result_base<T>>(*this).take_value();
 }
 
 #endif // SIMDJSON_EXCEPTIONS
 
 template<typename T>
-really_inline simdjson_result<T>::simdjson_result(T &&value, error_code error) noexcept
+simdjson_really_inline simdjson_result<T>::simdjson_result(T &&value, error_code error) noexcept
     : internal::simdjson_result_base<T>(std::forward<T>(value), error) {}
 template<typename T>
-really_inline simdjson_result<T>::simdjson_result(error_code error) noexcept
+simdjson_really_inline simdjson_result<T>::simdjson_result(error_code error) noexcept
     : internal::simdjson_result_base<T>(error) {}
 template<typename T>
-really_inline simdjson_result<T>::simdjson_result(T &&value) noexcept
+simdjson_really_inline simdjson_result<T>::simdjson_result(T &&value) noexcept
     : internal::simdjson_result_base<T>(std::forward<T>(value)) {}
 template<typename T>
-really_inline simdjson_result<T>::simdjson_result() noexcept
+simdjson_really_inline simdjson_result<T>::simdjson_result() noexcept
     : internal::simdjson_result_base<T>() {}
 
 } // namespace simdjson

include/simdjson/error.h

@@ -101,22 +101,22 @@ struct simdjson_result_base : public std::pair<T, error_code> {
   /**
    * Create a new empty result with error = UNINITIALIZED.
    */
-  really_inline simdjson_result_base() noexcept;
+  simdjson_really_inline simdjson_result_base() noexcept;
 
   /**
    * Create a new error result.
    */
-  really_inline simdjson_result_base(error_code error) noexcept;
+  simdjson_really_inline simdjson_result_base(error_code error) noexcept;
 
   /**
    * Create a new successful result.
    */
-  really_inline simdjson_result_base(T &&value) noexcept;
+  simdjson_really_inline simdjson_result_base(T &&value) noexcept;
 
   /**
    * Create a new result with both things (use if you don't want to branch when creating the result).
    */
-  really_inline simdjson_result_base(T &&value, error_code error) noexcept;
+  simdjson_really_inline simdjson_result_base(T &&value, error_code error) noexcept;
 
   /**
    * Move the value and the error to the provided variables.
@@ -124,19 +124,19 @@ struct simdjson_result_base : public std::pair<T, error_code> {
    * @param value The variable to assign the value to. May not be set if there is an error.
    * @param error The variable to assign the error to. Set to SUCCESS if there is no error.
    */
-  really_inline void tie(T &value, error_code &error) && noexcept;
+  simdjson_really_inline void tie(T &value, error_code &error) && noexcept;
 
   /**
    * Move the value to the provided variable.
    *
    * @param value The variable to assign the value to. May not be set if there is an error.
    */
-  really_inline error_code get(T &value) && noexcept;
+  simdjson_really_inline error_code get(T &value) && noexcept;
 
   /**
    * The error.
    */
-  really_inline error_code error() const noexcept;
+  simdjson_really_inline error_code error() const noexcept;
 
 #if SIMDJSON_EXCEPTIONS
 
@@ -145,21 +145,21 @@ struct simdjson_result_base : public std::pair<T, error_code> {
    *
    * @throw simdjson_error if there was an error.
    */
-  really_inline T& value() noexcept(false);
+  simdjson_really_inline T& value() noexcept(false);
 
   /**
    * Take the result value (move it).
    *
    * @throw simdjson_error if there was an error.
    */
-  really_inline T&& take_value() && noexcept(false);
+  simdjson_really_inline T&& take_value() && noexcept(false);
 
   /**
    * Cast to the value (will throw on error).
    *
    * @throw simdjson_error if there was an error.
    */
-  really_inline operator T&&() && noexcept(false);
+  simdjson_really_inline operator T&&() && noexcept(false);
 
 #endif // SIMDJSON_EXCEPTIONS
 }; // struct simdjson_result_base
@@ -176,19 +176,19 @@ struct simdjson_result : public internal::simdjson_result_base<T> {
   /**
    * @private Create a new empty result with error = UNINITIALIZED.
    */
-  really_inline simdjson_result() noexcept;
+  simdjson_really_inline simdjson_result() noexcept;
   /**
    * @private Create a new error result.
    */
-  really_inline simdjson_result(T &&value) noexcept;
+  simdjson_really_inline simdjson_result(T &&value) noexcept;
   /**
    * @private Create a new successful result.
    */
-  really_inline simdjson_result(error_code error_code) noexcept;
+  simdjson_really_inline simdjson_result(error_code error_code) noexcept;
   /**
    * @private Create a new result with both things (use if you don't want to branch when creating the result).
    */
-  really_inline simdjson_result(T &&value, error_code error) noexcept;
+  simdjson_really_inline simdjson_result(T &&value, error_code error) noexcept;
 
   /**
    * Move the value and the error to the provided variables.
@@ -196,19 +196,19 @@ struct simdjson_result : public internal::simdjson_result_base<T> {
    * @param value The variable to assign the value to. May not be set if there is an error.
    * @param error The variable to assign the error to. Set to SUCCESS if there is no error.
    */
-  really_inline void tie(T &value, error_code &error) && noexcept;
+  simdjson_really_inline void tie(T &value, error_code &error) && noexcept;
 
   /**
    * Move the value to the provided variable.
    *
    * @param value The variable to assign the value to. May not be set if there is an error.
    */
-  WARN_UNUSED really_inline error_code get(T &value) && noexcept;
+  SIMDJSON_WARN_UNUSED simdjson_really_inline error_code get(T &value) && noexcept;
 
   /**
    * The error.
    */
-  really_inline error_code error() const noexcept;
+  simdjson_really_inline error_code error() const noexcept;
 
 #if SIMDJSON_EXCEPTIONS
 
@@ -217,21 +217,21 @@ struct simdjson_result : public internal::simdjson_result_base<T> {
    *
    * @throw simdjson_error if there was an error.
    */
-  really_inline T& value() noexcept(false);
+  simdjson_really_inline T& value() noexcept(false);
 
   /**
    * Take the result value (move it).
    *
    * @throw simdjson_error if there was an error.
    */
-  really_inline T&& take_value() && noexcept(false);
+  simdjson_really_inline T&& take_value() && noexcept(false);
 
   /**
    * Cast to the value (will throw on error).
    *
    * @throw simdjson_error if there was an error.
    */
-  really_inline operator T&&() && noexcept(false);
+  simdjson_really_inline operator T&&() && noexcept(false);
 
 #endif // SIMDJSON_EXCEPTIONS
 }; // struct simdjson_result
@@ -245,7 +245,7 @@ using ErrorValues [[deprecated("This is an alias and will be removed, use error_
  * @deprecated Error codes should be stored and returned as `error_code`, use `error_message()` instead.
  */
 [[deprecated("Error codes should be stored and returned as `error_code`, use `error_message()` instead.")]]
-inline const std::string &error_message(int error) noexcept;
+inline const std::string error_message(int error) noexcept;
 
 } // namespace simdjson
 

include/simdjson/implementation.h

@@ -16,7 +16,7 @@ namespace simdjson {
  * @param len the length of the string in bytes.
  * @return true if the string is valid UTF-8.
  */
-WARN_UNUSED bool validate_utf8(const char * buf, size_t len) noexcept;
+SIMDJSON_WARN_UNUSED bool validate_utf8(const char * buf, size_t len) noexcept;
 
 
 /**
@@ -25,7 +25,7 @@ WARN_UNUSED bool validate_utf8(const char * buf, size_t len) noexcept;
  * @param sv the string_view to validate.
  * @return true if the string is valid UTF-8.
  */
-really_inline WARN_UNUSED bool validate_utf8(const std::string_view sv) noexcept {
+simdjson_really_inline SIMDJSON_WARN_UNUSED bool validate_utf8(const std::string_view sv) noexcept {
   return validate_utf8(sv.data(), sv.size());
 }
 
@@ -35,7 +35,7 @@ really_inline WARN_UNUSED bool validate_utf8(const std::string_view sv) noexcept
  * @param p the string to validate.
  * @return true if the string is valid UTF-8.
  */
-really_inline WARN_UNUSED bool validate_utf8(const std::string& s) noexcept {
+simdjson_really_inline SIMDJSON_WARN_UNUSED bool validate_utf8(const std::string& s) noexcept {
   return validate_utf8(s.data(), s.size());
 }
 
@@ -111,7 +111,7 @@ public:
    * @param dst_len the number of bytes written. Output only.
    * @return the error code, or SUCCESS if there was no error.
    */
-  WARN_UNUSED virtual error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept = 0;
+  SIMDJSON_WARN_UNUSED virtual error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept = 0;
   
   
   /**   
@@ -123,11 +123,11 @@ public:
    * @param len the length of the string in bytes.
    * @return true if and only if the string is valid UTF-8.
    */
-  WARN_UNUSED virtual bool validate_utf8(const char *buf, size_t len) const noexcept = 0;
+  SIMDJSON_WARN_UNUSED virtual bool validate_utf8(const char *buf, size_t len) const noexcept = 0;
 
 protected:
   /** @private Construct an implementation with the given name and description. For subclasses. */
-  really_inline implementation(
+  simdjson_really_inline implementation(
     std::string_view name,
     std::string_view description,
     uint32_t required_instruction_sets
@@ -165,7 +165,7 @@ namespace internal {
 class available_implementation_list {
 public:
   /** Get the list of available implementations compiled into simdjson */
-  really_inline available_implementation_list() {}
+  simdjson_really_inline available_implementation_list() {}
   /** Number of implementations */
   size_t size() const noexcept;
   /** STL const begin() iterator */

include/simdjson/internal/dom_parser_implementation.h

@@ -35,7 +35,7 @@ public:
    * @param len The length of the json document.
    * @return The error code, or SUCCESS if there was no error.
    */
-  WARN_UNUSED virtual error_code parse(const uint8_t *buf, size_t len, dom::document &doc) noexcept = 0;
+  SIMDJSON_WARN_UNUSED virtual error_code parse(const uint8_t *buf, size_t len, dom::document &doc) noexcept = 0;
 
   /**
    * @private For internal implementation use
@@ -51,7 +51,7 @@ public:
    * @param streaming Whether this is being called by parser::parse_many.
    * @return The error code, or SUCCESS if there was no error.
    */
-  WARN_UNUSED virtual error_code stage1(const uint8_t *buf, size_t len, bool streaming) noexcept = 0;
+  SIMDJSON_WARN_UNUSED virtual error_code stage1(const uint8_t *buf, size_t len, bool streaming) noexcept = 0;
 
   /**
    * @private For internal implementation use
@@ -65,7 +65,7 @@ public:
    * @param doc The document to output to.
    * @return The error code, or SUCCESS if there was no error.
    */
-  WARN_UNUSED virtual error_code stage2(dom::document &doc) noexcept = 0;
+  SIMDJSON_WARN_UNUSED virtual error_code stage2(dom::document &doc) noexcept = 0;
 
   /**
    * @private For internal implementation use
@@ -78,7 +78,7 @@ public:
    * @param doc The document to output to.
    * @return The error code, SUCCESS if there was no error, or EMPTY if all documents have been parsed.
    */
-  WARN_UNUSED virtual error_code stage2_next(dom::document &doc) noexcept = 0;
+  SIMDJSON_WARN_UNUSED virtual error_code stage2_next(dom::document &doc) noexcept = 0;
 
   /**
    * Change the capacity of this parser.
@@ -119,14 +119,14 @@ public:
    *
    * @return Current capacity, in bytes.
    */
-  really_inline size_t capacity() const noexcept;
+  simdjson_really_inline size_t capacity() const noexcept;
 
   /**
    * The maximum level of nested object and arrays supported by this parser.
    *
    * @return Maximum depth, in bytes.
    */
-  really_inline size_t max_depth() const noexcept;
+  simdjson_really_inline size_t max_depth() const noexcept;
 
   /**
    * Ensure this parser has enough memory to process JSON documents up to `capacity` bytes in length
@@ -136,7 +136,7 @@ public:
    * @param max_depth The new max_depth. Defaults to DEFAULT_MAX_DEPTH.
    * @return The error, if there is one.
    */
-  WARN_UNUSED inline error_code allocate(size_t capacity, size_t max_depth) noexcept;
+  SIMDJSON_WARN_UNUSED inline error_code allocate(size_t capacity, size_t max_depth) noexcept;
 
 protected:
   /**
@@ -154,15 +154,15 @@ protected:
   size_t _max_depth{0};
 }; // class dom_parser_implementation
 
-really_inline size_t dom_parser_implementation::capacity() const noexcept {
+simdjson_really_inline size_t dom_parser_implementation::capacity() const noexcept {
   return _capacity;
 }
 
-really_inline size_t dom_parser_implementation::max_depth() const noexcept {
+simdjson_really_inline size_t dom_parser_implementation::max_depth() const noexcept {
   return _max_depth;
 }
 
-WARN_UNUSED
+SIMDJSON_WARN_UNUSED
 inline error_code dom_parser_implementation::allocate(size_t capacity, size_t max_depth) noexcept {
   if (this->max_depth() != max_depth) {
     error_code err = set_max_depth(max_depth);

include/simdjson/internal/tape_ref-inl.h

@@ -10,34 +10,39 @@ namespace internal {
 //
 // tape_ref inline implementation
 //
-really_inline tape_ref::tape_ref() noexcept : doc{nullptr}, json_index{0} {}
-really_inline tape_ref::tape_ref(const dom::document *_doc, size_t _json_index) noexcept : doc{_doc}, json_index{_json_index} {}
+simdjson_really_inline tape_ref::tape_ref() noexcept : doc{nullptr}, json_index{0} {}
+simdjson_really_inline tape_ref::tape_ref(const dom::document *_doc, size_t _json_index) noexcept : doc{_doc}, json_index{_json_index} {}
+
+
+simdjson_really_inline bool tape_ref::is_document_root() const noexcept {
+  return json_index == 1; // should we ever change the structure of the tape, this should get updated.
+}
 
 // Some value types have a specific on-tape word value. It can be faster
 // to check the type by doing a word-to-word comparison instead of extracting the
 // most significant 8 bits.
 
-really_inline bool tape_ref::is_double() const noexcept {
+simdjson_really_inline bool tape_ref::is_double() const noexcept {
   constexpr uint64_t tape_double = uint64_t(tape_type::DOUBLE)<<56;
   return doc->tape[json_index] == tape_double;
 }
-really_inline bool tape_ref::is_int64() const noexcept {
+simdjson_really_inline bool tape_ref::is_int64() const noexcept {
   constexpr uint64_t tape_int64 = uint64_t(tape_type::INT64)<<56;
   return doc->tape[json_index] == tape_int64;
 }
-really_inline bool tape_ref::is_uint64() const noexcept {
+simdjson_really_inline bool tape_ref::is_uint64() const noexcept {
   constexpr uint64_t tape_uint64 = uint64_t(tape_type::UINT64)<<56;
   return doc->tape[json_index] == tape_uint64;
 }
-really_inline bool tape_ref::is_false() const noexcept {
+simdjson_really_inline bool tape_ref::is_false() const noexcept {
   constexpr uint64_t tape_false = uint64_t(tape_type::FALSE_VALUE)<<56;
   return doc->tape[json_index] == tape_false;
 }
-really_inline bool tape_ref::is_true() const noexcept {
+simdjson_really_inline bool tape_ref::is_true() const noexcept {
   constexpr uint64_t tape_true = uint64_t(tape_type::TRUE_VALUE)<<56;
   return doc->tape[json_index] == tape_true;
 }
-really_inline bool tape_ref::is_null_on_tape() const noexcept {
+simdjson_really_inline bool tape_ref::is_null_on_tape() const noexcept {
   constexpr uint64_t tape_null = uint64_t(tape_type::NULL_VALUE)<<56;
   return doc->tape[json_index] == tape_null;
 }
@@ -55,21 +60,21 @@ inline size_t tape_ref::after_element() const noexcept {
       return json_index + 1;
   }
 }
-really_inline tape_type tape_ref::tape_ref_type() const noexcept {
+simdjson_really_inline tape_type tape_ref::tape_ref_type() const noexcept {
   return static_cast<tape_type>(doc->tape[json_index] >> 56);
 }
-really_inline uint64_t internal::tape_ref::tape_value() const noexcept {
+simdjson_really_inline uint64_t internal::tape_ref::tape_value() const noexcept {
   return doc->tape[json_index] & internal::JSON_VALUE_MASK;
 }
-really_inline uint32_t internal::tape_ref::matching_brace_index() const noexcept {
+simdjson_really_inline uint32_t internal::tape_ref::matching_brace_index() const noexcept {
   return uint32_t(doc->tape[json_index]);
 }
-really_inline uint32_t internal::tape_ref::scope_count() const noexcept {
+simdjson_really_inline uint32_t internal::tape_ref::scope_count() const noexcept {
   return uint32_t((doc->tape[json_index] >> 32) & internal::JSON_COUNT_MASK);
 }
 
 template<typename T>
-really_inline T tape_ref::next_tape_value() const noexcept {
+simdjson_really_inline T tape_ref::next_tape_value() const noexcept {
   static_assert(sizeof(T) == sizeof(uint64_t), "next_tape_value() template parameter must be 64-bit");
   // Though the following is tempting...
   //  return *reinterpret_cast<const T*>(&doc->tape[json_index + 1]);
@@ -80,14 +85,14 @@ really_inline T tape_ref::next_tape_value() const noexcept {
   return x;
 }
 
-really_inline uint32_t internal::tape_ref::get_string_length() const noexcept {
+simdjson_really_inline uint32_t internal::tape_ref::get_string_length() const noexcept {
   size_t string_buf_index = size_t(tape_value());
   uint32_t len;
   memcpy(&len, &doc->string_buf[string_buf_index], sizeof(len));
   return len;
 }
 
-really_inline const char * internal::tape_ref::get_c_str() const noexcept {
+simdjson_really_inline const char * internal::tape_ref::get_c_str() const noexcept {
   size_t string_buf_index = size_t(tape_value());
   return reinterpret_cast<const char *>(&doc->string_buf[string_buf_index + sizeof(uint32_t)]);
 }

include/simdjson/internal/tape_ref.h

@@ -19,24 +19,25 @@ constexpr const uint32_t JSON_COUNT_MASK = 0xFFFFFF;
  */
 class tape_ref {
 public:
-  really_inline tape_ref() noexcept;
-  really_inline tape_ref(const dom::document *doc, size_t json_index) noexcept;
+  simdjson_really_inline tape_ref() noexcept;
+  simdjson_really_inline tape_ref(const dom::document *doc, size_t json_index) noexcept;
   inline size_t after_element() const noexcept;
-  really_inline tape_type tape_ref_type() const noexcept;
-  really_inline uint64_t tape_value() const noexcept;
-  really_inline bool is_double() const noexcept;
-  really_inline bool is_int64() const noexcept;
-  really_inline bool is_uint64() const noexcept;
-  really_inline bool is_false() const noexcept;
-  really_inline bool is_true() const noexcept;
-  really_inline bool is_null_on_tape() const noexcept;// different name to avoid clash with is_null.
-  really_inline uint32_t matching_brace_index() const noexcept;
-  really_inline uint32_t scope_count() const noexcept;
+  simdjson_really_inline tape_type tape_ref_type() const noexcept;
+  simdjson_really_inline uint64_t tape_value() const noexcept;
+  simdjson_really_inline bool is_double() const noexcept;
+  simdjson_really_inline bool is_int64() const noexcept;
+  simdjson_really_inline bool is_uint64() const noexcept;
+  simdjson_really_inline bool is_false() const noexcept;
+  simdjson_really_inline bool is_true() const noexcept;
+  simdjson_really_inline bool is_null_on_tape() const noexcept;// different name to avoid clash with is_null.
+  simdjson_really_inline uint32_t matching_brace_index() const noexcept;
+  simdjson_really_inline uint32_t scope_count() const noexcept;
   template<typename T>
-  really_inline T next_tape_value() const noexcept;
-  really_inline uint32_t get_string_length() const noexcept;
-  really_inline const char * get_c_str() const noexcept;
+  simdjson_really_inline T next_tape_value() const noexcept;
+  simdjson_really_inline uint32_t get_string_length() const noexcept;
+  simdjson_really_inline const char * get_c_str() const noexcept;
   inline std::string_view get_string_view() const noexcept;
+  simdjson_really_inline bool is_document_root() const noexcept;
 
   /** The document this element references. */
   const dom::document *doc;

include/simdjson/minify.h

@@ -24,7 +24,7 @@ namespace simdjson {
  * @param dst_len the number of bytes written. Output only.
  * @return the error code, or SUCCESS if there was no error.
  */
-WARN_UNUSED error_code minify(const char *buf, size_t len, char *dst, size_t &dst_len) noexcept;
+SIMDJSON_WARN_UNUSED error_code minify(const char *buf, size_t len, char *dst, size_t &dst_len) noexcept;
 
 /**
  * Minifies a JSON element or document, printing the smallest possible valid JSON.

include/simdjson/padded_string-inl.h

@@ -12,21 +12,21 @@
 namespace simdjson {
 namespace internal {
 
-// low-level function to allocate memory with padding so we can read past the
-// "length" bytes safely. if you must provide a pointer to some data, create it
-// with this function: length is the max. size in bytes of the string caller is
-// responsible to free the memory (free(...))
+// The allocate_padded_buffer function is a low-level function to allocate memory 
+// with padding so we can read past the "length" bytes safely. It is used by 
+// the padded_string class automatically. It returns nullptr in case
+// of error: the caller should check for a null pointer.
+// The length parameter is the maximum size in bytes of the string. 
+// The caller is responsible to free the memory (e.g., delete[] (...)).
 inline char *allocate_padded_buffer(size_t length) noexcept {
-  // we could do a simple malloc
-  // return (char *) malloc(length + SIMDJSON_PADDING);
-  // However, we might as well align to cache lines...
   size_t totalpaddedlength = length + SIMDJSON_PADDING;
-  char *padded_buffer = aligned_malloc_char(64, totalpaddedlength);
-#ifndef NDEBUG
+  char *padded_buffer = new (std::nothrow) char[totalpaddedlength];
   if (padded_buffer == nullptr) {
     return nullptr;
   }
-#endif // NDEBUG
+  // We write zeroes in the padded region to avoid having uninitized 
+  // garbage. If nothing else, garbage getting read might trigger a 
+  // warning in a memory checking.
   memset(padded_buffer + length, 0, totalpaddedlength - length);
   return padded_buffer;
 } // allocate_padded_buffer()
@@ -69,7 +69,7 @@ inline padded_string::padded_string(padded_string &&o) noexcept
 }
 
 inline padded_string &padded_string::operator=(padded_string &&o) noexcept {
-  aligned_free_char(data_ptr);
+  delete[] data_ptr;
   data_ptr = o.data_ptr;
   viable_size = o.viable_size;
   o.data_ptr = nullptr; // we take ownership
@@ -87,7 +87,7 @@ inline void padded_string::swap(padded_string &o) noexcept {
 }
 
 inline padded_string::~padded_string() noexcept {
-  aligned_free_char(data_ptr);
+  delete[] data_ptr;
 }
 
 inline size_t padded_string::size() const noexcept { return viable_size; }

include/simdjson/padded_string.h

@@ -144,10 +144,12 @@ inline simdjson::padded_string operator "" _padded(const char *str, size_t len)
 namespace simdjson {
 namespace internal {
 
-// low-level function to allocate memory with padding so we can read past the
-// "length" bytes safely. if you must provide a pointer to some data, create it
-// with this function: length is the max. size in bytes of the string caller is
-// responsible to free the memory (free(...))
+// The allocate_padded_buffer function is a low-level function to allocate memory 
+// with padding so we can read past the "length" bytes safely. It is used by 
+// the padded_string class automatically. It returns nullptr in case
+// of error: the caller should check for a null pointer.
+// The length parameter is the maximum size in bytes of the string. 
+// The caller is responsible to free the memory (e.g., delete[] (...)).
 inline char *allocate_padded_buffer(size_t length) noexcept;
 
 } // namespace internal

include/simdjson/portability.h

@@ -5,7 +5,11 @@
 #include <cstdint>
 #include <cstdlib>
 #include <cfloat>
-
+#include <cassert>
+#ifndef _WIN32
+// strcasecmp, strncasecmp 
+#include <strings.h>
+#endif
 
 #ifdef _MSC_VER
 #define SIMDJSON_VISUAL_STUDIO 1
@@ -17,7 +21,6 @@
  * Under clang for Windows, we enable:
  *  * target pragmas so that part and only part of the
  *     code gets compiled for advanced instructions.
- *  * computed gotos.
  *
  */
 #ifdef __clang__
@@ -103,32 +106,27 @@ use a 64-bit target such as x64 or 64-bit ARM.")
 #ifdef __clang__
 // clang does not have GCC push pop
 // warning: clang attribute push can't be used within a namespace in clang up
-// til 8.0 so TARGET_REGION and UNTARGET_REGION must be *outside* of a
+// til 8.0 so SIMDJSON_TARGET_REGION and SIMDJSON_UNTARGET_REGION must be *outside* of a
 // namespace.
-#define TARGET_REGION(T)                                                       \
+#define SIMDJSON_TARGET_REGION(T)                                                       \
   _Pragma(STRINGIFY(                                                           \
       clang attribute push(__attribute__((target(T))), apply_to = function)))
-#define UNTARGET_REGION _Pragma("clang attribute pop")
+#define SIMDJSON_UNTARGET_REGION _Pragma("clang attribute pop")
 #elif defined(__GNUC__)
 // GCC is easier
-#define TARGET_REGION(T)                                                       \
+#define SIMDJSON_TARGET_REGION(T)                                                       \
   _Pragma("GCC push_options") _Pragma(STRINGIFY(GCC target(T)))
-#define UNTARGET_REGION _Pragma("GCC pop_options")
+#define SIMDJSON_UNTARGET_REGION _Pragma("GCC pop_options")
 #endif // clang then gcc
 
 #endif // x86
 
 // Default target region macros don't do anything.
-#ifndef TARGET_REGION
-#define TARGET_REGION(T)
-#define UNTARGET_REGION
+#ifndef SIMDJSON_TARGET_REGION
+#define SIMDJSON_TARGET_REGION(T)
+#define SIMDJSON_UNTARGET_REGION
 #endif
 
-// under GCC and CLANG, we use these two macros
-#define TARGET_HASWELL TARGET_REGION("avx2,bmi,pclmul,lzcnt")
-#define TARGET_WESTMERE TARGET_REGION("sse4.2,pclmul")
-#define TARGET_ARM64
-
 // Is threading enabled?
 #if defined(BOOST_HAS_THREADS) || defined(_REENTRANT) || defined(_MT)
 #ifndef SIMDJSON_THREADS_ENABLED
@@ -150,12 +148,6 @@ use a 64-bit target such as x64 or 64-bit ARM.")
 #endif
 
 
-#if SIMDJSON_DO_NOT_USE_THREADS_NO_MATTER_WHAT
-// No matter what happened, we undefine SIMDJSON_THREADS_ENABLED and so disable threads.
-#undef SIMDJSON_THREADS_ENABLED
-#endif
-
-
 #if defined(__clang__)
 #define NO_SANITIZE_UNDEFINED __attribute__((no_sanitize("undefined")))
 #elif defined(__GNUC__)
@@ -178,48 +170,6 @@ use a 64-bit target such as x64 or 64-bit ARM.")
 #define simdjson_strncasecmp strncasecmp
 #endif
 
-namespace simdjson {
-/** @private portable version of  posix_memalign */
-static inline void *aligned_malloc(size_t alignment, size_t size) {
-  void *p;
-#ifdef SIMDJSON_VISUAL_STUDIO
-  p = _aligned_malloc(size, alignment);
-#elif defined(__MINGW32__) || defined(__MINGW64__)
-  p = __mingw_aligned_malloc(size, alignment);
-#else
-  // somehow, if this is used before including "x86intrin.h", it creates an
-  // implicit defined warning.
-  if (posix_memalign(&p, alignment, size) != 0) {
-    return nullptr;
-  }
-#endif
-  return p;
-}
-
-/** @private */
-static inline char *aligned_malloc_char(size_t alignment, size_t size) {
-  return (char *)aligned_malloc(alignment, size);
-}
-
-/** @private */
-static inline void aligned_free(void *mem_block) {
-  if (mem_block == nullptr) {
-    return;
-  }
-#ifdef SIMDJSON_VISUAL_STUDIO
-  _aligned_free(mem_block);
-#elif defined(__MINGW32__) || defined(__MINGW64__)
-  __mingw_aligned_free(mem_block);
-#else
-  free(mem_block);
-#endif
-}
-
-/** @private */
-static inline void aligned_free_char(char *mem_block) {
-  aligned_free((void *)mem_block);
-}
-
 #ifdef NDEBUG
 
 #ifdef SIMDJSON_VISUAL_STUDIO
@@ -232,11 +182,9 @@ static inline void aligned_free_char(char *mem_block) {
 
 #else // NDEBUG
 
-#include <cassert>
 #define SIMDJSON_UNREACHABLE() assert(0);
 #define SIMDJSON_ASSUME(COND) assert(COND)
 
 #endif
 
-} // namespace simdjson
 #endif // SIMDJSON_PORTABILITY_H

include/simdjson/simdjson_version.h

@@ -4,7 +4,7 @@
 #define SIMDJSON_SIMDJSON_VERSION_H
 
 /** The version of simdjson being used (major.minor.revision) */
-#define SIMDJSON_VERSION 0.4.1
+#define SIMDJSON_VERSION 0.5.0
 
 namespace simdjson {
 enum {
@@ -15,11 +15,11 @@ enum {
   /**
    * The minor version (major.MINOR.revision) of simdjson being used.
    */
-  SIMDJSON_VERSION_MINOR = 4,
+  SIMDJSON_VERSION_MINOR = 5,
   /**
    * The revision (major.minor.REVISION) of simdjson being used.
    */
-  SIMDJSON_VERSION_REVISION = 1
+  SIMDJSON_VERSION_REVISION = 0
 };
 } // namespace simdjson
 

jsonchecker/CMakeLists.txt

@@ -1,3 +1,7 @@
 set(SIMDJSON_TEST_DATA_DIR ${CMAKE_CURRENT_SOURCE_DIR} PARENT_SCOPE)
+set(SIMDJSON_MINEFIELD_TEST_DATA_DIR ${CMAKE_CURRENT_SOURCE_DIR} PARENT_SCOPE)
+
 add_library(jsonchecker-data INTERFACE)
 target_compile_definitions(jsonchecker-data INTERFACE SIMDJSON_TEST_DATA_DIR="${CMAKE_CURRENT_SOURCE_DIR}/")
+
+add_subdirectory(minefield)

jsonchecker/fail80.json

@@ -0,0 +1 @@
+{"":7E-9223372036854775808

jsonchecker/fail81.json

@@ -0,0 +1 @@
+""n

jsonchecker/fail82.json

@@ -0,0 +1 @@
+10.2.2

jsonchecker/minefield/CMakeLists.txt

@@ -0,0 +1,4 @@
+set(SIMDJSON_MINEFIELD_TEST_DATA_DIR ${CMAKE_CURRENT_SOURCE_DIR} PARENT_SCOPE)
+
+add_library(jsonchecker-minefield-data INTERFACE)
+target_compile_definitions(jsonchecker-minefield-data INTERFACE SIMDJSON_MINEFIELD_TEST_DATA_DIR="${CMAKE_CURRENT_SOURCE_DIR}/")

jsonchecker/minefield/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2016 Nicolas Seriot
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

jsonchecker/minefield/i_number_double_huge_neg_exp.json

@@ -0,0 +1 @@
+[123.456e-789]

jsonchecker/minefield/i_number_huge_exp.json

@@ -0,0 +1 @@
+[0.4e00669999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999969999999006]

jsonchecker/minefield/i_number_neg_int_huge_exp.json

@@ -0,0 +1 @@
+[-1e+9999]

jsonchecker/minefield/i_number_pos_double_huge_exp.json

@@ -0,0 +1 @@
+[1.5e+9999]

jsonchecker/minefield/i_number_real_neg_overflow.json

@@ -0,0 +1 @@
+[-123123e100000]

jsonchecker/minefield/i_number_real_pos_overflow.json

@@ -0,0 +1 @@
+[123123e100000]

jsonchecker/minefield/i_number_real_underflow.json

@@ -0,0 +1 @@
+[123e-10000000]

jsonchecker/minefield/i_number_too_big_neg_int.json

@@ -0,0 +1 @@
+[-123123123123123123123123123123]

jsonchecker/minefield/i_number_too_big_pos_int.json

@@ -0,0 +1 @@
+[100000000000000000000]

jsonchecker/minefield/i_number_very_big_negative_int.json

@@ -0,0 +1 @@
+[-237462374673276894279832749832423479823246327846]

jsonchecker/minefield/i_object_key_lone_2nd_surrogate.json

@@ -0,0 +1 @@
+{"\uDFAA":0}

jsonchecker/minefield/i_string_1st_surrogate_but_2nd_missing.json

@@ -0,0 +1 @@
+["\uDADA"]

jsonchecker/minefield/i_string_1st_valid_surrogate_2nd_invalid.json

@@ -0,0 +1 @@
+["\uD888\u1234"]

jsonchecker/minefield/i_string_UTF-8_invalid_sequence.json

@@ -0,0 +1 @@
+["譌･ﾑ淫"]

jsonchecker/minefield/i_string_UTF8_surrogate_U+D800.json

@@ -0,0 +1 @@
+["���"]

jsonchecker/minefield/i_string_incomplete_surrogate_and_escape_valid.json

@@ -0,0 +1 @@
+["\uD800\n"]

jsonchecker/minefield/i_string_incomplete_surrogate_pair.json

@@ -0,0 +1 @@
+["\uDd1ea"]

jsonchecker/minefield/i_string_incomplete_surrogates_escape_valid.json

@@ -0,0 +1 @@
+["\uD800\uD800\n"]

jsonchecker/minefield/i_string_invalid_lonely_surrogate.json

@@ -0,0 +1 @@
+["\ud800"]

jsonchecker/minefield/i_string_invalid_surrogate.json

@@ -0,0 +1 @@
+["\ud800abc"]