Merge branch 'dlemire/nopadding' of github.com:simdjson/simdjson into dlemire/nopadding

a mess whereas a SWAR optimization processing 8 digits would sometimes be called,
despite the fact that it would be unsafe and/or it would not be called, and we would
end up leaving performance on the floor.

.circleci/config.yml

@@ -208,6 +208,14 @@ jobs:
     executor: clang10
     environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON  -DSIMDJSON_NO_FORCE_INLINING=ON -DSIMDJSON_SANITIZE_THREADS=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly }
     steps: [ cmake_test ]
+  nocheckeof-clang10:
+    description: Validate that when __SIMDJSON_CHECK_EOF=0, everything still succeeds
+    environment:
+      CXXFLAGS: -D__SIMDJSON_CHECK_EOF=0
+      CMAKE_BUILD_FLAGS: --target ondemand_tests
+      CTEST_FLAGS: --output-on-failure -R ondemand_
+    executor: clang10
+    steps: [ cmake_test ]
   # dynamic
   dynamic-gcc10:
     description: Build and run tests on GCC 10 and AVX 2 with a cmake dynamic build
@@ -284,7 +292,7 @@ workflows:
       - libcpp-clang10
 
       # full single-implementation tests
-      - sanitize-gcc10
+      # - sanitize-gcc10 # gcc10 + sanitize uses a lot of memory?
       - sanitize-clang10
       - threadsanitize-gcc10
       - threadsanitize-clang10
@@ -303,7 +311,7 @@ workflows:
 
 
       # sanitized single-implementation tests
-      - sanitize-haswell-gcc10
+      # - sanitize-haswell-gcc10 # gcc10 + sanitize uses a lot of memory?
       - sanitize-haswell-clang10
 
       # testing "just the library"

.cirrus.yml

@@ -19,8 +19,8 @@ task:
   build_script:
     - mkdir build
     - cd build
-    - cmake -DSIMDJSON_BASH=OFF  -DSIMDJSON_GIT=OFF ..
-    - make
+    - cmake -DSIMDJSON_BASH=OFF   -DSIMDJSON_DEVELOPER_MODE=ON  -DSIMDJSON_GIT=OFF ..
+    - cmake --build .
   test_script:
     - cd build
     - ctest --output-on-failure -LE explicitonly

.drone.yml

@@ -1,4 +1,46 @@
 kind: pipeline
+name: i386-gcc # we do not support 32-bit systems, but we run tests
+platform: { os: linux, arch: amd64 }
+steps:
+- name: Build and Test
+  image: i386/ubuntu
+  environment:
+    CC: gcc
+    CXX: g++
+    BUILD_FLAGS: -- -j
+    CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF   -DSIMDJSON_DEVELOPER_MODE=ON
+    CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
+  commands:
+    - scripts/addcmakeppa.sh "$(env -i sh -c '. /etc/os-release; echo $VERSION_CODENAME')"
+    - apt-get install -y g++ cmake gcc git
+    - mkdir build
+    - cd build
+    - cmake $CMAKE_FLAGS ..
+    - cmake --build . $BUILD_FLAGS
+    - ctest $CTEST_FLAGS
+---
+kind: pipeline
+name: i386-clang # we do not support 32-bit systems, but we run tests
+platform: { os: linux, arch: amd64 }
+steps:
+- name: Build and Test
+  image: i386/ubuntu
+  environment:
+    CC: clang-6.0
+    CXX: clang++-6.0
+    BUILD_FLAGS: -- -j
+    CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF  -DSIMDJSON_DEVELOPER_MODE=ON
+    CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
+  commands:
+    - scripts/addcmakeppa.sh "$(env -i sh -c '. /etc/os-release; echo $VERSION_CODENAME')"
+    - apt-get install -y clang++-6.0 cmake git
+    - mkdir build
+    - cd build
+    - cmake $CMAKE_FLAGS ..
+    - cmake --build . $BUILD_FLAGS
+    - ctest $CTEST_FLAGS
+---
+kind: pipeline
 name: gcc9
 platform: { os: linux, arch: amd64 }
 steps:
@@ -8,7 +50,7 @@ steps:
     CC: gcc
     CXX: g++
     BUILD_FLAGS: -- -j
-    CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback
+    CMAKE_FLAGS:   -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback
     CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
   commands:
     - echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list
@@ -35,7 +77,7 @@ steps:
     CC: clang-6.0
     CXX: clang++-6.0
     BUILD_FLAGS: -- -j
-    CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback
+    CMAKE_FLAGS:   -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback
     CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
   commands:
     - mkdir build
@@ -58,7 +100,7 @@ steps:
     CC: gcc
     CXX: g++
     BUILD_FLAGS: -- -j
-    CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON
+    CMAKE_FLAGS:   -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=ON
     CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
   commands:
     - echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list
@@ -80,7 +122,7 @@ steps:
   environment:
     CC: clang-9
     CXX: clang++-9
-    CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON
+    CMAKE_FLAGS:   -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=ON
     BUILD_FLAGS: -- -j
     CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
   commands:
@@ -100,7 +142,7 @@ steps:
     CC: gcc
     CXX: g++
     BUILD_FLAGS: -- -j
-    CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback
+    CMAKE_FLAGS:   -DSIMDJSON_DEVELOPER_MODE=ON  -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback
     CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
   commands:
     - echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list
@@ -126,7 +168,7 @@ steps:
   environment:
     CC: clang-9
     CXX: clang++-9
-    CMAKE_FLAGS: -DSIMDJSON_SANITIZE=ON -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback
+    CMAKE_FLAGS:    -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_NO_FORCE_INLINING=ON -DSIMDJSON_SANITIZE=ON -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback
     BUILD_FLAGS: -- -j
     CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
   commands:
@@ -150,7 +192,7 @@ steps:
   environment:
     CC: clang-11
     CXX: clang++-11
-    CMAKE_FLAGS: -GNinja
+    CMAKE_FLAGS:   -DSIMDJSON_DEVELOPER_MODE=ON -GNinja
     BUILD_FLAGS:
     CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
     CXXFLAGS: -std=c++20 -stdlib=libc++
@@ -171,7 +213,7 @@ steps:
     CC: gcc
     CXX: g++
     BUILD_FLAGS: -- -j
-    CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=arm64;fallback
+    CMAKE_FLAGS:   -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=arm64;fallback
     CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
   commands:
     - echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list
@@ -195,7 +237,7 @@ steps:
   environment:
     CC: clang-6.0
     CXX: clang++-6.0
-    CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON
+    CMAKE_FLAGS:   -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=ON
     BUILD_FLAGS: -- -j
     CTEST_FLAGS: -j4 --output-on-failure  -LE explicitonly
   commands:
@@ -218,7 +260,7 @@ steps:
     CC: gcc
     CXX: g++
     BUILD_FLAGS: -- -j
-    CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON
+    CMAKE_FLAGS:   -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=ON
     CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
   commands:
     - echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list
@@ -239,7 +281,7 @@ steps:
   environment:
     CC: clang-6.0
     CXX: clang++-6.0
-    CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON
+    CMAKE_FLAGS:   -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=ON
     BUILD_FLAGS: -- -j
     CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
   commands:
@@ -260,7 +302,7 @@ steps:
   image: gcc:8
   environment:
     BUILD_FLAGS: -- -j
-    CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=arm64;fallback
+    CMAKE_FLAGS:   -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=arm64;fallback
     CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
     CC: gcc
     CXX: g++
@@ -287,7 +329,7 @@ steps:
   environment:
     CC: clang-6.0
     CXX: clang++-6.0
-    CMAKE_FLAGS: -DSIMDJSON_SANITIZE=ON -DSIMDJSON_IMPLEMENTATION=arm64;fallback
+    CMAKE_FLAGS:    -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_NO_FORCE_INLINING=ON -DSIMDJSON_SANITIZE=ON -DSIMDJSON_IMPLEMENTATION=arm64;fallback
     BUILD_FLAGS: -- -j
     CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
   commands:
@@ -314,7 +356,7 @@ steps:
     CC: clang-9
     CXX: clang++-9
     BUILD_FLAGS: -- -j 4
-    CMAKE_FLAGS: -GNinja -DBUILD_SHARED_LIBS=OFF
+    CMAKE_FLAGS:   -DSIMDJSON_DEVELOPER_MODE=ON -GNinja -DBUILD_SHARED_LIBS=OFF
     CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
     CXXFLAGS: -stdlib=libc++
   commands:
@@ -335,7 +377,7 @@ steps:
     CC: clang-9
     CXX: clang++-9
     BUILD_FLAGS: -- -j
-    CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF
+    CMAKE_FLAGS:   -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF
     CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
     CXXFLAGS: -stdlib=libc++
   commands:
@@ -356,7 +398,7 @@ steps:
     CC: clang-7
     CXX: clang++-7
     BUILD_FLAGS: -- -j
-    CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF
+    CMAKE_FLAGS:   -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF
     CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
     CXXFLAGS: -stdlib=libc++
   commands:
@@ -376,7 +418,7 @@ steps:
     CC: gcc
     CXX: g++
     BUILD_FLAGS: -- -j
-    CMAKE_FLAGS: -DSIMDJSON_EXCEPTIONS=OFF
+    CMAKE_FLAGS:   -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_EXCEPTIONS=OFF
     CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
   commands:
     - echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list

.github/workflows/macos-11.yml

@@ -1,34 +0,0 @@
-name: Macos (Xcode 11)
-
-on: [push, pull_request]
-
-jobs:
-  macos-build:
-    if: >-
-      ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
-      ! contains(toJSON(github.event.commits.*.message), '[skip github]')
-    runs-on: macos-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/cache@v2
-        with:
-          path: dependencies/.cache
-          key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
-      - name: Use cmake
-        run: |
-          xcversion select 11.7
-          mkdir builddebug &&
-          cd builddebug &&
-          cmake -DCMAKE_BUILD_TYPE=Debug -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF ..  &&
-          cmake --build .   &&
-          ctest -j --output-on-failure -LE explicitonly   &&
-          cd .. &&
-          mkdir build &&
-          cd build &&
-          cmake  -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination ..  &&
-          cmake --build .   &&
-          ctest -j --output-on-failure -LE explicitonly   &&
-          cmake --install . &&
-          echo -e '#include <simdjson.h>\nint main(int argc,char**argv) {simdjson::dom::parser parser;simdjson::dom::element tweets = parser.load(argv[1]); }' > tmp.cpp && c++ -Idestination/include -Ldestination/lib -std=c++17 -Wl,-rpath,destination/lib -o linkandrun tmp.cpp -lsimdjson && ./linkandrun jsonexamples/twitter.json &&
-          cd ../tests/installation_tests/find &&
-          mkdir build && cd build && cmake -DCMAKE_INSTALL_PREFIX:PATH=../../../build/destination .. &&  cmake --build .

.github/workflows/macos.yml

@@ -1,33 +0,0 @@
-name: Macos
-
-on: [push, pull_request]
-
-jobs:
-  macos-build:
-    if: >-
-      ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
-      ! contains(toJSON(github.event.commits.*.message), '[skip github]')
-    runs-on: macos-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/cache@v2
-        with:
-          path: dependencies/.cache
-          key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
-      - name: Use cmake
-        run: |
-          mkdir builddebug &&
-          cd builddebug &&
-          cmake -DCMAKE_BUILD_TYPE=Debug -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF ..  &&
-          cmake --build .   &&
-          ctest -j --output-on-failure -LE explicitonly   &&
-          cd .. &&
-          mkdir build &&
-          cd build &&
-          cmake  -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination ..  &&
-          cmake --build .   &&
-          ctest -j --output-on-failure -LE explicitonly   &&
-          cmake --install . &&
-          echo -e '#include <simdjson.h>\nint main(int argc,char**argv) {simdjson::dom::parser parser;simdjson::dom::element tweets = parser.load(argv[1]); }' > tmp.cpp && c++ -Idestination/include -Ldestination/lib -std=c++17 -Wl,-rpath,destination/lib -o linkandrun tmp.cpp -lsimdjson && ./linkandrun jsonexamples/twitter.json &&
-          cd ../tests/installation_tests/find &&
-          mkdir build && cd build && cmake -DCMAKE_INSTALL_PREFIX:PATH=../../../build/destination .. &&  cmake --build .

.github/workflows/msys2-clang.yml

@@ -17,9 +17,15 @@ jobs:
           - msystem: "MINGW64"
             install: mingw-w64-x86_64-libxml2 mingw-w64-x86_64-cmake mingw-w64-x86_64-ninja mingw-w64-x86_64-clang
             type: Release
+          - msystem: "MINGW32"
+            install: mingw-w64-i686-libxml2 mingw-w64-i686-cmake mingw-w64-i686-ninja mingw-w64-i686-clang
+            type: Release
           - msystem: "MINGW64"
             install: mingw-w64-x86_64-libxml2 mingw-w64-x86_64-cmake mingw-w64-x86_64-ninja mingw-w64-x86_64-clang
             type: Debug
+          - msystem: "MINGW32"
+            install: mingw-w64-i686-libxml2 mingw-w64-i686-cmake mingw-w64-i686-ninja mingw-w64-i686-clang
+            type: Debug
     env:
       CMAKE_GENERATOR: Ninja
 

.github/workflows/msys2.yml

@@ -19,9 +19,15 @@ jobs:
           - msystem: "MINGW64"
             install: mingw-w64-x86_64-cmake mingw-w64-x86_64-ninja mingw-w64-x86_64-gcc
             type: Release
+          - msystem: "MINGW32"
+            install: mingw-w64-i686-cmake mingw-w64-i686-ninja mingw-w64-i686-gcc
+            type: Release
           - msystem: "MINGW64"
             install: mingw-w64-x86_64-cmake mingw-w64-x86_64-ninja mingw-w64-x86_64-gcc
             type: Debug
+          - msystem: "MINGW32"
+            install: mingw-w64-i686-cmake mingw-w64-i686-ninja mingw-w64-i686-gcc
+            type: Debug
     env:
       CMAKE_GENERATOR: Ninja
 

.github/workflows/ubuntu18-oldclang.yml

@@ -1,31 +0,0 @@
-name: Ubuntu 18.04 CI (LLVM 7)
-
-on: [push, pull_request]
-
-jobs:
-  ubuntu-build:
-    if: >-
-      ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
-      ! contains(toJSON(github.event.commits.*.message), '[skip github]')
-    runs-on: ubuntu-18.04
-
-    env:
-      CC: clang-7
-      CXX: clang++-7
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/cache@v2
-        with:
-          path: dependencies/.cache
-          key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
-      - name: install clang 7
-        run: |
-          sudo apt update
-          sudo apt install clang-7
-      - name: Use cmake
-        run: |
-          mkdir build &&
-          cd build &&
-          cmake  -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination ..  &&
-          cmake --build .   &&
-          ctest -j --output-on-failure -LE explicitonly

.travis.yml

@@ -172,7 +172,7 @@ install:
   - export CMAKE_C_FLAGS="${CMAKE_CXX_FLAGS}"
   - export CMAKE_FLAGS="-DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS} -DSIMDJSON_IMPLEMENTATION=ppc64;fallback";
   - if [[ "${SANITIZE}" == "on" ]]; then
-      export CMAKE_FLAGS="${CMAKE_FLAGS} -DSIMDJSON_SANITIZE=ON";
+      export CMAKE_FLAGS="${CMAKE_FLAGS}  -DSIMDJSON_NO_FORCE_INLINING=ON -DSIMDJSON_SANITIZE=ON";
       export ASAN_OPTIONS="detect_leaks=0";
     fi
   - if [[ "${STATIC}" == "on" ]]; then
@@ -183,7 +183,7 @@ install:
 script:
   - mkdir build
   - cd build
-  - cmake $CMAKE_FLAGS ..
+  - cmake   -DSIMDJSON_DEVELOPER_MODE=ON $CMAKE_FLAGS ..
   - cmake --build . -- -j2
   - SIMDJSON_FORCE_IMPLEMENTATION=ppc64 ctest $CTEST_FLAGS -L per_implementation
   - SIMDJSON_FORCE_IMPLEMENTATION=fallback ctest $CTEST_FLAGS -L per_implementation

.vscode/settings.json

@@ -85,7 +85,6 @@
         "valarray": "cpp",
         "vector": "cpp",
         "*.ipp": "cpp",
-        "__functional_base_03": "cpp",
         "filesystem": "cpp"
     }
 }

CMakeLists.txt

@@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.14)
 project(
     simdjson
     # The version number is modified by tools/release.py
-    VERSION 1.0.2
+    VERSION 0.9.1
     DESCRIPTION "Parsing gigabytes of JSON per second"
     HOMEPAGE_URL "https://simdjson.org/"
     LANGUAGES CXX C
@@ -20,8 +20,8 @@ string(
 # ---- Options, variables ----
 
 # These version numbers are modified by tools/release.py
-set(SIMDJSON_LIB_VERSION "9.0.0" CACHE STRING "simdjson library version")
-set(SIMDJSON_LIB_SOVERSION "9" CACHE STRING "simdjson library soversion")
+set(SIMDJSON_LIB_VERSION "8.0.0" CACHE STRING "simdjson library version")
+set(SIMDJSON_LIB_SOVERSION "8" CACHE STRING "simdjson library soversion")
 
 option(SIMDJSON_ENABLE_THREADS "Link with thread support" ON)
 

Doxyfile

@@ -38,7 +38,7 @@ PROJECT_NAME           = simdjson
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = "1.0.2"
+PROJECT_NUMBER         = "0.9.1"
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a

README.md

@@ -3,7 +3,7 @@
 [![Ubuntu 20.04 CI](https://github.com/simdjson/simdjson/workflows/Ubuntu%2020.04%20CI%20(GCC%209)/badge.svg)](https://simdjson.org/plots.html)
 ![VS16-CI](https://github.com/simdjson/simdjson/workflows/VS16-CI/badge.svg)
 ![MinGW64-CI](https://github.com/simdjson/simdjson/workflows/MinGW64-CI/badge.svg)
-[![][license img]][license]  [![Doxygen Documentation](https://img.shields.io/badge/docs-doxygen-green.svg)](https://simdjson.org/api/1.0.0/index.html)
+[![][license img]][license]  [![Doxygen Documentation](https://img.shields.io/badge/docs-doxygen-green.svg)](https://simdjson.org/api/0.9.0/index.html)
 
 simdjson : Parsing gigabytes of JSON per second
 ===============================================
@@ -53,7 +53,6 @@ The simdjson library is easily consumable with a single .h and .cpp file.
 2. Create `quickstart.cpp`:
 
 ```c++
-#include <iostream>
 #include "simdjson.h"
 using namespace simdjson;
 int main(void) {
@@ -79,7 +78,7 @@ Usage documentation is available:
 * [Performance](doc/performance.md) shows some more advanced scenarios and how to tune for them.
 * [Implementation Selection](doc/implementation-selection.md) describes runtime CPU detection and
   how you can work with it.
-* [API](https://simdjson.org/api/1.0.0/annotated.html) contains the automatically generated API documentation.
+* [API](https://simdjson.org/api/0.9.0/annotated.html) contains the automatically generated API documentation.
 
 Performance results
 -------------------
@@ -159,7 +158,7 @@ We also have an informal [blog post providing some background and context](https
 
 For the video inclined, <br />
 [![simdjson at QCon San Francisco 2019](http://img.youtube.com/vi/wlvKAT7SZIQ/0.jpg)](http://www.youtube.com/watch?v=wlvKAT7SZIQ)<br />
-(It was the best voted talk, we're kinda proud of it.)
+(it was the best voted talk, we're kinda proud of it).
 
 Funding
 -------

benchmark/amazon_cellphones/simdjson_ondemand.h

@@ -18,7 +18,7 @@ struct simdjson_ondemand {
     ondemand::document_stream::iterator i = stream.begin();
     ++i;  // Skip first line
     for (;i != stream.end(); ++i) {
-      auto doc = *i;
+      auto & doc = *i;
       size_t index{0};
       StringType copy;
       double rating;

benchmark/bench_dom_api.cpp

@@ -1,5 +1,4 @@
 #include <benchmark/benchmark.h>
-#include <iostream>
 #include "simdjson.h"
 #include <sstream>
 

benchmark/bench_parse_call.cpp

@@ -1,5 +1,4 @@
 #include <benchmark/benchmark.h>
-#include <iostream>
 #include "simdjson.h"
 using namespace simdjson;
 using namespace benchmark;

benchmark/large_amazon_cellphones/simdjson_ondemand.h

@@ -18,7 +18,7 @@ struct simdjson_ondemand {
     ondemand::document_stream::iterator i = stream.begin();
     ++i;  // Skip first line
     for (;i != stream.end(); ++i) {
-      auto doc = *i;
+      auto & doc = *i;
       size_t index{0};
       StringType copy;
       double rating;

cmake/developer-options.cmake

@@ -3,12 +3,19 @@
 #
 add_library(simdjson-internal-flags INTERFACE)
 
+option(SIMDJSON_NO_FORCE_INLINING "Do not attempt to force function inlining" OFF)
+if(SIMDJSON_NO_FORCE_INLINING)
+  add_compile_definitions(SIMDJSON_NO_FORCE_INLINING=1)
+endif()
 
-option(SIMDJSON_CHECK_EOF "Check for the end of the input buffer. The setting is unnecessary since we require padding of the inputs. You should expect tests to fail with this option turned on." OFF)
+option(SIMDJSON_CHECK_EOF "Check for the end of the input buffer (to avoid buffer overflows). We expect this setting to be ON except for performance testing." ON)
 if(SIMDJSON_CHECK_EOF)
   add_compile_definitions(SIMDJSON_CHECK_EOF=1)
+else()
+  add_compile_definitions(SIMDJSON_CHECK_EOF=0)
 endif()
 
+
 option(SIMDJSON_SANITIZE_UNDEFINED "Sanitize undefined behavior" OFF)
 if(SIMDJSON_SANITIZE_UNDEFINED)
   add_compile_options(-fsanitize=undefined -fno-sanitize-recover=all)

dependencies/CMakeLists.txt

@@ -13,9 +13,8 @@ cmake_dependent_option(SIMDJSON_GOOGLE_BENCHMARKS "compile the Google Benchmark
 if(SIMDJSON_GOOGLE_BENCHMARKS)
   set_off(BENCHMARK_ENABLE_TESTING)
   set_off(BENCHMARK_ENABLE_INSTALL)
-  set_off(BENCHMARK_ENABLE_WERROR)
 
-  import_dependency(google_benchmarks google/benchmark f91b6b4)
+  import_dependency(google_benchmarks google/benchmark 8982e1e)
   add_dependency(google_benchmarks)
 endif()
 

doc/basics_doxygen.md

@@ -1,5 +1,3 @@
-We take our documentation seriously. Please start reading the documentation before you attempt to use simdjson. We hope you will enjoy reading us.
-
 * Basics: https://github.com/simdjson/simdjson/blob/master/doc/basics.md is an overview of how to use simdjson and its APIs.
 * parse_many: https://github.com/simdjson/simdjson/blob/master/doc/parse_many.md describes an interface providing features to work with files or streams containing multiple small JSON documents. As fast and convenient as possible.
 * Performance: https://github.com/simdjson/simdjson/blob/master/doc/performance.md shows some more advanced scenarios and how to tune for them.

doc/dom.md

@@ -15,7 +15,6 @@ An overview of what you need to know to use simdjson, with examples.
 * [Reusing the parser for maximum efficiency](#reusing-the-parser-for-maximum-efficiency)
 * [Server Loops: Long-Running Processes and Memory Capacity](#server-loops-long-running-processes-and-memory-capacity)
 * [Best Use of the DOM API](#best-use-of-the-dom-api)
-* [Padding and Temporary Copies](#padding-and-temporary-copies)
 
 DOM vs On Demand
 ----------------------------------------------
@@ -36,12 +35,11 @@ dom::parser parser;
 dom::element doc = parser.load(filename); // load and parse a file
 ```
 
-Or by creating a padded string (for efficiency reasons, simdjson requires a string with
-SIMDJSON_PADDING bytes at the end) and calling `parse()`:
+Or by creating a string and calling `parse()`:
 
 ```c++
 dom::parser parser;
-dom::element doc = parser.parse("[1,2,3]"_padded); // parse a string, the _padded suffix creates a simdjson::padded_string instance
+dom::element doc = parser.parse(std::string("[1,2,3]")); // parse a string
 ```
 
 The parsed document resulting from the `parser.load` and `parser.parse` calls depends on the `parser` instance. Thus the `parser` instance must remain in scope. Furthermore, you must have at most one parsed document in play per `parser` instance.
@@ -54,8 +52,7 @@ During the`load` or `parse` calls, neither the input file nor the input string a
 For best performance, a `parser` instance should be reused over several files: otherwise you will needlessly reallocate memory, an expensive process. It is also possible to avoid entirely memory allocations during parsing when using simdjson. [See our performance notes for details](performance.md).
 
 If you need a lower-level interface, you may call the function `parser.parse(const char * p, size_t l)` on a pointer `p` while specifying the
-length of your input `l` in bytes. To see how to get the very best performance from a low-level approach, you way want to read our [performance notes](https://github.com/simdjson/simdjson/blob/master/doc/performance.md#padding-and-temporary-copies) on this topic (see the Padding and Temporary Copies section).
-
+length of your input `l` in bytes.
 
 Using the Parsed JSON
 ---------------------
@@ -64,12 +61,12 @@ Once you have an element, you can navigate it with idiomatic C++ iterators, oper
 
 * **Extracting Values (with exceptions):** You can cast a JSON element to a native type: `double(element)` or
   `double x = json_element`. This works for double, uint64_t, int64_t, bool,
-  dom::object and dom::array. An exception (`simdjson::simdjson_error`) is thrown if the cast is not possible.
+  dom::object and dom::array. An exception is thrown if the cast is not possible.
 * **Extracting Values (without exceptions):** You can use a variant usage of `get()` with error codes to avoid exceptions. You first declare the variable of the appropriate type (`double`, `uint64_t`, `int64_t`, `bool`,
   `dom::object` and `dom::array`) and pass it by reference to `get()` which gives you back an error code: e.g.,
   ```c++
   simdjson::error_code error;
-  simdjson::padded_string numberstring = "1.2"_padded; // our JSON input ("1.2")
+  std::string numberstring = "1.2"; // our JSON input ("1.2")
   simdjson::dom::parser parser;
   double value; // variable where we store the value to be parsed
   error = parser.parse(numberstring).get(value);
@@ -96,11 +93,11 @@ Once you have an element, you can navigate it with idiomatic C++ iterators, oper
 The following code illustrates all of the above:
 
 ```c++
-auto cars_json = R"( [
+std::string cars_json = R"( [
   { "make": "Toyota", "model": "Camry",  "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
   { "make": "Kia",    "model": "Soul",   "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
   { "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
-] )"_padded;
+] )";
 dom::parser parser;
 
 // Iterating through an array of objects
@@ -129,10 +126,10 @@ for (dom::object car : parser.parse(cars_json)) {
 Here is a different example illustrating the same ideas:
 
 ```C++
-auto abstract_json = R"( [
+std::string abstract_json = R"( [
     {  "12345" : {"a":12.34, "b":56.78, "c": 9998877}   },
     {  "12545" : {"a":11.44, "b":12.78, "c": 11111111}  }
-  ] )"_padded;
+  ] )";
 dom::parser parser;
 
 // Parse and iterate through an array of objects
@@ -151,8 +148,8 @@ And another one:
 
 
 ```C++
-  auto abstract_json = R"(
-    {  "str" : { "123" : {"abc" : 3.14 } } } )"_padded;
+  std::string abstract_json = R"(
+    {  "str" : { "123" : {"abc" : 3.14 } } } )";
   dom::parser parser;
   double v = parser.parse(abstract_json)["str"]["123"]["abc"];
   cout << "number: " << v << endl;
@@ -165,7 +162,7 @@ C++17 Support
 While the simdjson library can be used in any project using C++ 11 and above, field iteration has special support C++ 17's destructuring syntax. For example:
 
 ```c++
-padded_string json = R"(  { "foo": 1, "bar": 2 }  )"_padded;
+std::string json = R"(  { "foo": 1, "bar": 2 }  )";
 dom::parser parser;
 dom::object object;
 auto error = parser.parse(json).get(object);
@@ -179,7 +176,7 @@ For comparison, here is the C++ 11 version of the same code:
 
 ```c++
 // C++ 11 version for comparison
-padded_string json = R"(  { "foo": 1, "bar": 2 }  )"_padded;
+std::string json = R"(  { "foo": 1, "bar": 2 }  )";
 dom::parser parser;
 dom::object object;
 auto error = parser.parse(json).get(object);
@@ -197,11 +194,11 @@ The simdjson library also supports [JSON pointer](https://tools.ietf.org/html/rf
 `at_pointer()` method, letting you reach further down into the document in a single call:
 
 ```c++
-auto cars_json = R"( [
+std::string cars_json = R"( [
   { "make": "Toyota", "model": "Camry",  "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
   { "make": "Kia",    "model": "Soul",   "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
   { "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
-] )"_padded;
+] )";
 dom::parser parser;
 dom::element cars = parser.parse(cars_json);
 cout << cars.at_pointer("/0/tire_pressure/1") << endl; // Prints 39.9
@@ -218,11 +215,11 @@ You can apply a JSON path to any node and the path gets interpreted relatively,
 Consider the following example:
 
 ```c++
-auto cars_json = R"( [
+std::string cars_json = R"( [
   { "make": "Toyota", "model": "Camry",  "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
   { "make": "Kia",    "model": "Soul",   "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
   { "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
-] )"_padded;
+] )";
 dom::parser parser;
 dom::element cars = parser.parse(cars_json);
 cout << cars.at_pointer("/0/tire_pressure/1") << endl; // Prints 39.9
@@ -274,7 +271,6 @@ Our program loads the file, selects value corresponding to key "search_metadata"
 it selects the key "count" within that object.
 
 ```C++
-#include <iostream>
 #include "simdjson.h"
 
 int main(void) {
@@ -302,7 +298,6 @@ triggering exceptions. To do this, we use `["statuses"].at(0)["id"]`. We break t
 Observe how we use the `at` method when querying an index into an array, and not the bracket operator.
 
 ```C++
-#include <iostream>
 #include "simdjson.h"
 
 int main(void) {
@@ -323,11 +318,11 @@ int main(void) {
 This is how the example in "Using the Parsed JSON" could be written using only error code checking:
 
 ```c++
-auto cars_json = R"( [
+std::string cars_json = R"( [
   { "make": "Toyota", "model": "Camry",  "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
   { "make": "Kia",    "model": "Soul",   "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
   { "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
-] )"_padded;
+] )";
 dom::parser parser;
 dom::array cars;
 auto error = parser.parse(cars_json).get(cars);
@@ -370,10 +365,10 @@ for (dom::element car_element : cars) {
 Here is another example:
 
 ```C++
-auto abstract_json = R"( [
+std::string abstract_json = R"( [
     {  "12345" : {"a":12.34, "b":56.78, "c": 9998877}   },
     {  "12545" : {"a":11.44, "b":12.78, "c": 11111111}  }
-  ] )"_padded;
+  ] )";
 dom::parser parser;
 dom::array array;
 auto error = parser.parse(abstract_json).get(array);
@@ -403,8 +398,8 @@ for (dom::element elem : array) {
 And another one:
 
 ```C++
-  auto abstract_json = R"(
-    {  "str" : { "123" : {"abc" : 3.14 } } } )"_padded;
+  std::string abstract_json = R"(
+    {  "str" : { "123" : {"abc" : 3.14 } } } )";
   dom::parser parser;
   double v;
   auto error = parser.parse(abstract_json)["str"]["123"]["abc"].get(v);
@@ -459,7 +454,6 @@ program from continuing if there was an error.
 If one is willing to trigger exceptions, it is possible to write simpler code:
 
 ```C++
-#include <iostream>
 #include "simdjson.h"
 
 int main(void) {
@@ -539,15 +533,16 @@ you can parse terabytes of JSON data without doing any new allocation.
 dom::parser parser;
 
 // This initializes buffers and a document big enough to handle this JSON.
-dom::element doc = parser.parse("[ true, false ]"_padded);
+dom::element doc = parser.parse(std::string("[ true, false ]"));
+// std::string("[ true, false ]") may be replaced by "[ true, false ]"s
 cout << doc << endl;
 
 // This reuses the existing buffers, and reuses and *overwrites* the old document
-doc = parser.parse("[1, 2, 3]"_padded);
+doc = parser.parse(std::string("[1, 2, 3]"));
 cout << doc << endl;
 
 // This also reuses the existing buffers, and reuses and *overwrites* the old document
-dom::element doc2 = parser.parse("true"_padded);
+dom::element doc2 = parser.parse(std::string("true"));
 // Even if you keep the old reference around, doc and doc2 refer to the same document.
 cout << doc << endl;
 cout << doc2 << endl;
@@ -613,26 +608,3 @@ Best Use of the DOM API
 -------------------------
 
 The simdjson API provides access to the JSON DOM (document-object-model) content as a tree of `dom::element` instances, each representing an object, an array or an atomic type (null, true, false, number). These `dom::element` instances are lightweight objects (e.g., spanning 16 bytes) and it might be advantageous to pass them by value, as opposed to passing them by reference or by pointer.
-
-Padding and Temporary Copies
---------------
-
-The simdjson function `parser.parse` reads data from a padded  buffer, containing SIMDJSON_PADDING extra bytes added at the end.
-If you are passing a `padded_string` to `parser.parse` or loading the JSON directly from
-disk (`parser.load`), padding is automatically  handled.
-When calling `parser.parse` on a pointer (e.g., `parser.parse(my_char_pointer, my_length_in_bytes)`) a temporary copy  is made by default with adequate padding and you, again, do not need to be concerned with padding.
-
-Some users may not be able use our `padded_string` class or to load the data directly from disk (`parser.load`). They may need to pass data pointers to the library.  If these users wish to avoid temporary copies and corresponding temporary memory allocations, they may want to call `parser.parse` with the `realloc_if_needed` parameter set to false (e.g., `parser.parse(my_char_pointer, my_length_in_bytes, false)`). In such cases, they need to ensure that there are at least SIMDJSON_PADDING extra bytes at the end that can be safely accessed and read. They do not need to initialize the padded bytes to any value in particular. The following example is safe:
-
-
-```C++
-const char *json      = R"({"key":"value"})";
-const size_t json_len = std::strlen(json);
-std::unique_ptr<char[]> padded_json_copy{new char[json_len + SIMDJSON_PADDING]};
-memcpy(padded_json_copy.get(), json, json_len);
-memset(padded_json_copy.get() + json_len, 0, SIMDJSON_PADDING);
-simdjson::dom::parser parser;
-simdjson::dom::element element = parser.parse(padded_json_copy.get(), json_len, false);
-````
-
-Setting the `realloc_if_needed` parameter `false` in this manner may lead to better performance since copies are avoided, but it requires that the user takes more responsibilities: the simdjson library cannot verify that the input buffer was padded with SIMDJSON_PADDING extra bytes.

doc/iterate_many.md

@@ -1,16 +1,8 @@
 iterate_many
 ==========
 
-When serializing large databases, it is often better to write out many independent JSON
-documents, instead of one large monolithic document containing many records. The simdjson
-library provides high-speed access to files or streams containing multiple small JSON documents separated by ASCII white-space characters. Given an input such as
-```JSON
-{"text":"a"}
-{"text":"b"}
-{"text":"c"}
-...
-```
-... you want to read the entries (individual JSON documents) as quickly and as conveniently as possible. Importantly, the input might span several gigabytes, but you want to use a small (fixed) amount of memory. Ideally, you'd also like the parallelize the processing (using more than one core) to speed up the process.
+An interface providing features to work with files or streams containing multiple small JSON documents.
+As fast and convenient as possible.
 
 Contents
 --------
@@ -116,9 +108,7 @@ Whitespace Characters:
 - **Linefeed**
 - **Carriage return**
 - **Horizontal tab**
-
-If your documents are all objects or arrays, then you may even have nothing between them.
-E.g., `[1,2]{"32":1}` is recognized as two documents.
+- **Nothing**
 
 Some official formats **(non-exhaustive list)**:
 - [Newline-Delimited JSON (NDJSON)](http://ndjson.org/)
@@ -177,7 +167,7 @@ Let us illustrate the idea with code:
 
 
 ```C++
-    auto json = R"([1,2,3]  {"1":1,"2":3,"4":4} [1,2,3]  )"_padded;
+    std::string json = R"([1,2,3]  {"1":1,"2":3,"4":4} [1,2,3]  )";
     simdjson::ondemand::parser parser;
     simdjson::ondemand::document_stream stream;
     auto error = parser.iterate_many(json).get(stream);
@@ -185,7 +175,7 @@ Let us illustrate the idea with code:
     auto i = stream.begin();
 	 size_t count{0};
     for(; i != stream.end(); ++i) {
-        auto doc = *i;
+        auto & doc = *i;
         if(!i.error()) {
           std::cout << "got full document at " << i.current_index() << std::endl;
           std::cout << i.source() << std::endl;
@@ -218,7 +208,7 @@ Some users may need to work with truncated streams. The simdjson may truncate do
 Consider the following example where a truncated document (`{"key":"intentionally unclosed string  `) containing 39 bytes has been left within the stream. In such cases, the first two whole documents are parsed and returned, and the `truncated_bytes()` method returns 39.
 
 ```C++
-    auto json = R"([1,2,3]  {"1":1,"2":3,"4":4} {"key":"intentionally unclosed string  )"_padded;
+    std::string json = R"([1,2,3]  {"1":1,"2":3,"4":4} {"key":"intentionally unclosed string  )";
     simdjson::ondemand::parser parser;
     simdjson::ondemand::document_stream stream;
     auto error = parser.iterate_many(json,json.size()).get(stream);
@@ -236,4 +226,4 @@ This will print:
 39 bytes
 ```
 
-Importantly, you should only call `truncated_bytes()` after iterating through all of the documents since the stream cannot tell whether there are truncated documents at the very end when it may not have accessed that part of the data yet.
+Importantly, you should only call `truncated_bytes()` after iterating through all of the documents since the stream cannot tell whether there are truncated documents at the very end when it may not have accessed that part of the data yet.

doc/ondemand_design.md

@@ -709,6 +709,42 @@ in production systems:
     }
 ```
 
+
+### Long-Running Processes and Memory Capacity
+
+The On Demand approach also automatically expands its memory capacity when larger documents are parsed. However, for longer processes where very large files are processed (such as server loops), this capacity is not resized down. Similarly to the DOM-based approach (see [here](https://github.com/simdjson/simdjson/blob/master/doc/dom.md#server-loops-long-running-processes-and-memory-capacity)]), On Demand also lets you adjust the maximal capacity that the parser can process:
+
+* You can set an upper bound (*max_capacity*) when construction the parser:
+```C++
+    ondemand::parser parser(1000*1000);  // Never grows past documents > 1 MB
+    auto doc = parser.iterate(json);
+    for (web_request request : listen()) {
+      padded_string json;
+      padded_string json = padded_string::load(request.body);
+      auto error = parser.iterate(json);
+      // If the document was above our limit, emit 413 = payload too large
+      if (error == CAPACITY) { request.respond(413); continue; }
+      // ...
+    }
+```
+
+The capacity will grow as the parser encounters larger documents up to 1 MB.
+
+* You can also allocate a *fixed capacity* that will never grow:
+```C++
+    ondemand::parser parser(1000*1000);
+    parser.allocate(1000*1000)  // Fix the capacity to 1 MB
+    auto doc = parser.iterate(json);
+    for (web_request request : listen()) {
+      padded_string json;
+      padded_string json = padded_string::load(request.body);
+      auto error = parser.iterate(json);
+      // If the document was above our limit, emit 413 = payload too large
+      if (error == CAPACITY) { request.respond(413); continue; }
+      // ...
+    }
+```
+You can also manually set the maximal capacity using the method `set_max_capacity()`.
 ### Benefits of the On Demand Approach
 
 We expect that the On Demand approach has many of the performance benefits of the schema-based approach, while providing a flexibility that is similar to that of the DOM-based approach.
@@ -728,6 +764,7 @@ There are currently additional technical limitations which we expect to resolve
 
 * The simdjson library offers runtime dispatching which allows you to compile one binary and have it run at full speed on different processors, taking advantage of the specific features of the processor. The On Demand API has limited runtime dispatch support. Under x64 systems, to fully benefit from the On Demand API, we recommend that you compile your code for a specific processor. E.g., if your processor supports AVX2 instructions, you should compile your binary executable with AVX2 instruction support (by using your compiler's commands). If you are sufficiently technically proficient, you can implement runtime dispatching within your application, by compiling your On Demand code for different processors.
 * There is an initial phase which scans the entire document quickly, irrespective of the size of the document. We plan to break this phase into distinct steps for large files in a future release as we have done with other components of our API (e.g., `parse_many`).
+* The On Demand API does not support JSON Pointer. This capability is currently limited to our core API.
 
 ### Applicability of the On Demand Approach
 

doc/parse_many.md

@@ -1,14 +1,8 @@
 parse_many
 ==========
 
-An interface providing features to work with files or streams containing multiple small JSON documents. Given an input such as
-```JSON
-{"text":"a"}
-{"text":"b"}
-{"text":"c"}
-...
-```
-... you want to read the entries (individual JSON documents) as quickly and as conveniently as possible. Importantly, the input might span several gigabytes, but you want to use a small (fixed) amount of memory. Ideally, you'd also like the parallelize the processing (using more than one core) to speed up the process.
+An interface providing features to work with files or streams containing multiple small JSON documents.
+As fast and convenient as possible.
 
 Contents
 --------
@@ -180,7 +174,7 @@ Let us illustrate the idea with code:
 
 
 ```C++
-    auto json = R"([1,2,3]  {"1":1,"2":3,"4":4} [1,2,3]  )"_padded;
+    std::string json = R"([1,2,3]  {"1":1,"2":3,"4":4} [1,2,3]  )";
     simdjson::dom::parser parser;
     simdjson::dom::document_stream stream;
     auto error = parser.parse_many(json).get(stream);
@@ -221,7 +215,7 @@ Some users may need to work with truncated streams. The simdjson may truncate do
 Consider the following example where a truncated document (`{"key":"intentionally unclosed string  `) containing 39 bytes has been left within the stream. In such cases, the first two whole documents are parsed and returned, and the `truncated_bytes()` method returns 39.
 
 ```C++
-    auto json = R"([1,2,3]  {"1":1,"2":3,"4":4} {"key":"intentionally unclosed string  )"_padded;
+    std::string json = R"([1,2,3]  {"1":1,"2":3,"4":4} {"key":"intentionally unclosed string  )";
     simdjson::dom::parser parser;
     simdjson::dom::document_stream stream;
     auto error = parser.parse_many(json,json.size()).get(stream);
@@ -233,4 +227,4 @@ Consider the following example where a truncated document (`{"key":"intentionall
 ```
 
 
-Importantly, you should only call `truncated_bytes()` after iterating through all of the documents since the stream cannot tell whether there are truncated documents at the very end when it may not have accessed that part of the data yet.
+Importantly, you should only call `truncated_bytes()` after iterating through all of the documents since the stream cannot tell whether there are truncated documents at the very end when it may not have accessed that part of the data yet.

doc/performance.md

@@ -10,7 +10,7 @@ are still some scenarios where tuning can enhance performance.
 * [Large files and huge page support](#large-files-and-huge-page-support)
 * [Number parsing](#number-parsing)
 * [Visual Studio](#visual-studio)
-* [Power Usage and Downclocking](#power-usage-and-downclocking)
+* [Downclocking](#downclocking)
 
 Reusing the parser for maximum efficiency
 -----------------------------------------
@@ -24,14 +24,14 @@ you can parse terabytes of JSON data without doing any new allocation.
 ondemand::parser parser;
 
 // This initializes buffers  big enough to handle this JSON.
-auto json = "[ true, false ]"_padded;
+std::string json = "[ true, false ]";
 auto doc = parser.iterate(json);
 for(bool i : doc.get_array()) {
   cout << i << endl;
 }
 
 // This reuses the existing buffers
-auto number_json = "[1, 2, 3]"_padded;
+std::string number_json = "[1, 2, 3]";
 doc = parser.iterate(number_json);
 for(int64_t i : doc.get_array()) {
   cout << i << endl;
@@ -45,53 +45,10 @@ Reusing string buffers
 We recommend against creating many `std::string` or `simdjson::padded_string` instances to store the JSON content in your application. [Creating many non-trivial objects is convenient but often surprisingly slow](https://lemire.me/blog/2020/08/08/performance-tip-constructing-many-non-trivial-objects-is-slow/). Instead, as much as possible, you should allocate (once or a few times) reusable memory buffers where you write your JSON content. If you have a buffer `json_str` (of type `char*`) allocated for  `capacity` bytes and you store a JSON document spanning `length` bytes, you can pass it to simdjson as follows:
 
 ```c++
- auto doc = parser.iterate(padded_string_view(json_str, length, capacity));
-```
-
-or simply
-
-
-```c++
- auto doc = parser.iterate(json_str, length, capacity);
+ auto doc = parser.iterate(json_str, length));
 ```
 
 
-Server Loops: Long-Running Processes and Memory Capacity
----------------------------------
-
-The On Demand approach also automatically expands its memory capacity when larger documents are parsed. However, for longer processes where very large files are processed (such as server loops), this capacity is not resized down. On Demand also lets you adjust the maximal capacity that the parser can process:
-
-* You can set an upper bound (*max_capacity*) when construction the parser:
-```C++
-    ondemand::parser parser(1000*1000);  // Never grows past documents > 1 MB
-    auto doc = parser.iterate(json);
-    for (web_request request : listen()) {
-      padded_string json;
-      padded_string json = padded_string::load(request.body);
-      auto error = parser.iterate(json);
-      // If the document was above our limit, emit 413 = payload too large
-      if (error == CAPACITY) { request.respond(413); continue; }
-      // ...
-    }
-```
-
-The capacity will grow as the parser encounters larger documents up to 1 MB.
-
-* You can also allocate a *fixed capacity* that will never grow:
-```C++
-    ondemand::parser parser(1000*1000);
-    parser.allocate(1000*1000)  // Fix the capacity to 1 MB
-    auto doc = parser.iterate(json);
-    for (web_request request : listen()) {
-      padded_string json;
-      padded_string json = padded_string::load(request.body);
-      auto error = parser.iterate(json);
-      // If the document was above our limit, emit 413 = payload too large
-      if (error == CAPACITY) { request.respond(413); continue; }
-      // ...
-    }
-```
-You can also manually set the maximal capacity using the method `set_max_capacity()`.
 
 Large files and huge page support
 ---------------------------------
@@ -147,15 +104,19 @@ Recent versions of Microsoft Visual Studio on Windows provides support for the L
 Under Windows, we also support the GNU GCC compiler via MSYS2. The performance of 64-bit MSYS2 under Windows excellent (on par with Linux).
 
 
-Power Usage and Downclocking
+Downclocking
 --------------
 
-The simdjson library relies on SIMD instructions. SIMD instructions are the public transportation of computing. Instead of using 4 distinct instructions to add numbers, you can replace them with a single instruction that does the same work. Though the one instruction is slightly more expensive, the energy used per unit of work is much less with SIMD. If you can increase your speed using SIMD instructions (NEON, SSE, AVX), you should expect to reduce your power usage.
+
+
+SIMD instructions are the public transportation of computing. Instead of using 4 distinct instructions to add numbers, you can replace them with a single instruction that does the same work. Though the one instruction is slightly more expensive, the energy used per unit of work is much less with SIMD. If you can increase your speed using SIMD instructions (NEON, SSE, AVX), you should expect to reduce your power usage.
 
 The SIMD instructions that simdjson relies upon (SSE and AVX under x64, NEON under ARM, ALTIVEC under PPC) are routinely part of runtime libraries (e.g., [Go](https://golang.org/src/runtime/memmove_amd64.s), [Glibc](https://github.com/ihtsae/glibc/commit/5f3d0b78e011d2a72f9e88b0e9ef5bc081d18f97), [LLVM](https://github.com/llvm/llvm-project/blob/96f3ea0d21b48ca088355db10d4d1a2e9bc9f884/lldb/tools/debugserver/source/MacOSX/i386/DNBArchImplI386.cpp), [Rust](https://github.com/rust-lang/rust/commit/070fad1701fb36b112853b0a6a9787a7bb7ff34c), [Java](http://hg.openjdk.java.net/jdk8u/jdk8u/hotspot/file/c1374141598c/src/cpu/x86/vm/stubGenerator_x86_64.cpp#l1297), [PHP](https://github.com/php/php-src/blob/e5cb53ec68603d4dbdd780fd3ecfca943b4fd383/ext/standard/string.c)). What distinguishes the simdjson library is that it is built from the ground up to benefit from these instructions.
 
 
-You should not expect the simdjson library to cause *downclocking* of your recent Intel CPU cores. On some Intel processors, using SIMD instructions in a sustained manner on the same CPU core may result in a phenomenon called downclocking whereas the processor initially runs these instructions at a slow speed before reducing the frequency of the core for a short time (milliseconds). Intel refers to these states as licenses. On some current Intel processors, it occurs under two scenarios:
+You should not expect the simdjson library to cause *downclocking* of your recent Intel CPU cores.
+
+On some Intel processors, using SIMD instructions in a sustained manner on the same CPU core may result in a phenomenon called downclocking whereas the processor initially runs these instructions at a slow speed before reducing the frequency of the core for a short time (milliseconds). Intel refers to these states as licenses. On some current Intel processors, it occurs under two scenarios:
 
 - [Whenever 512-bit AVX-512 instructions are used](https://lemire.me/blog/2018/09/07/avx-512-when-and-how-to-use-these-new-instructions/).
 - Whenever heavy 256-bit or wider instructions are used. Heavy instructions are those involving floating point operations or integer multiplications (since these execute on the floating point unit).

examples/quickstart/quickstart.cpp

@@ -1,8 +1,7 @@
-#include <iostream>
 #include "simdjson.h"
 
 int main(void) {
   simdjson::dom::parser parser;
   simdjson::dom::element tweets = parser.load("twitter.json");
   std::cout << tweets["search_metadata"]["count"] << " results." << std::endl;
-}
+}

examples/quickstart/quickstart2.cpp

@@ -1,4 +1,3 @@
-#include <iostream>
 #include "simdjson.h"
 
 int main(void) {

examples/quickstart/quickstart2_noexceptions.cpp

@@ -1,4 +1,3 @@
-#include <iostream>
 #include "simdjson.h"
 
 int main(void) {

examples/quickstart/quickstart_noexceptions.cpp

@@ -1,4 +1,3 @@
-#include <iostream>
 #include "simdjson.h"
 
 int main(void) {

examples/quickstart/quickstart_ondemand.cpp

@@ -1,4 +1,3 @@
-#include <iostream>
 #include "simdjson.h"
 using namespace simdjson;
 int main(void) {

examples/quickstart/quickstart_ondemand_noexceptions.cpp

@@ -1,4 +1,3 @@
-#include <iostream>
 #include "simdjson.h"
 using namespace simdjson;
 int main(void) {

fuzz/Fuzzing.md

@@ -18,7 +18,7 @@ The fuzzers are used in several ways.
 * oss-fuzz - heavy duty 24/7 fuzzing provided by the google driven oss-fuzz project
 
 ## Local fuzzing
-Just invoke fuzz/quick_check.sh, it will download the latest corpus (kept up to date by the CI fuzzers) and run the fuzzers for a short time. In case you want to run the fuzzers for longer, modify the timeout value in the script or invoke the fuzzer directly.
+Just invoke fuzz/quick_check.sh, it will download the latest corpus from bintray (kept up to date by the CI fuzzers) and run the fuzzers for a short time. In case you want to run the fuzzers for longer, modify the timeout value in the script or invoke the fuzzer directly.
 
 This requires linux with clang and cmake installed (recent Debian and Ubuntu are known to work fine).
 
@@ -69,7 +69,7 @@ As little code as possible is kept at oss-fuzz since it is inconvenient to chang
 
 The simdjson library does not benefit from a corpus as much as other projects, because the library is very fast and explores the input space very well. With that said, it is still beneficial to have one. The CI job stores the corpus on a remote server between runs, and is available at [www.pauldreik.se](https://readonly:readonly@www.pauldreik.se/fuzzdata/index.php?project=simdjson).
 
-One can also grab the corpus as an artifact from the github actions job if you are logged in at github. Pick a run, then go to artifacts and download.
+One can also grab the corpus as an artifact from the github actions job. Pick a run, then go to artifacts and download.
 
 ## Fuzzing coverage
 

fuzz/fuzz_implementations.cpp

@@ -11,7 +11,6 @@
 #include <cstddef>
 #include <cstdint>
 #include <cstdlib>
-#include <iostream>
 #include <string>
 #include <array>
 #include "supported_implementations.h"

fuzz/fuzz_minifyimpl.cpp

@@ -12,7 +12,6 @@
 #include "simdjson.h"
 #include <cstddef>
 #include <cstdlib>
-#include <iostream>
 #include <vector>
 #include "supported_implementations.h"
 

fuzz/fuzz_ndjson.cpp

@@ -8,26 +8,24 @@
 
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
   FuzzData fd(Data, Size);
-  const auto batch_size = static_cast<size_t>(fd.getInt<0,1000>());
-  const auto json = simdjson::padded_string{fd.remainder_as_stringview()};
+  const auto batch_size=static_cast<size_t>(fd.getInt<0,1000>());
+  const auto json=simdjson::padded_string{fd.remainder_as_stringview()};
   simdjson::dom::parser parser;
-  simdjson::dom::document_stream docs;
-  if(parser.parse_many(json,batch_size).get(docs)) { return 0; }
-  size_t bool_count1 = 0;
-  size_t total_count1 = 0;
+#if SIMDJSON_EXCEPTIONS
+  try {
+#endif
+    simdjson::dom::document_stream docs;
+    if(parser.parse_many(json,batch_size).get(docs)) {
+      return 0;
+    }
+
+  size_t bool_count=0;
   for (auto doc : docs) {
-    total_count1++;
-    bool_count1 += doc.is_bool();
+    bool_count+=doc.is_bool();
   }
-  // Restart, if we made it this far, the document *must* be accessible.
-  if(parser.parse_many(json,batch_size).get(docs)) { return EXIT_FAILURE; }
-  size_t bool_count2 = 0;
-  size_t total_count2 = 0;
-  for (auto doc : docs) {
-    total_count2++;
-    bool_count2 += doc.is_bool();
+#if SIMDJSON_EXCEPTIONS
+  } catch(...) {
   }
-  // They should agree!!!
-  if((total_count2 != total_count1) || (bool_count2 != bool_count1)) { return EXIT_FAILURE; }
+#endif
   return 0;
 }

fuzz/fuzz_utf8.cpp

@@ -10,7 +10,6 @@
 #include "simdjson.h"
 #include <cstddef>
 #include <cstdlib>
-#include <iostream>
 #include "supported_implementations.h"
 
 extern "C" int VerboseTestOneInput(const uint8_t *Data, size_t Size) {

fuzz/quick_check.sh

@@ -4,7 +4,7 @@
 # good when working locally developing the fuzzers or making
 # sure code changes still pass the fuzzers.
 #
-# It will download the corpus from external store (kept up to date
+# It will download the corpus from bintray (kept up to date
 # by the crontab github actions) unless a local out/ directory
 # already exists.
 #
@@ -23,9 +23,10 @@ done
 
 #download the corpus if it does not already exist
 if [ ! -d out ] ; then
-  # the corpus is also available for download from the artifacts page on https://github.com/simdjson/simdjson/actions/workflows/fuzzers.yml
-  # but that requires being logged in so can not be easily done from this script.
-  wget -O - https://readonly:readonly@www.pauldreik.se/fuzzdata/index.php?project=simdjson |tar xzf -
+  # ideally, we would download the github artifact but that requires being logged in which can not
+  # easily be fixed from this shell script.
+  echo "NOTE! please go to the artifacts page on https://github.com/simdjson/simdjson/actions/workflows/fuzzers.yml and download the latest corpus.tar.zip artifact manually to speed up fuzzing"
+  sleep 5s
 fi
 
 # By default, use the debug friendly variant since this script is intended

include/simdjson/arm64/numberparsing.h

@@ -5,7 +5,6 @@ namespace simdjson {
 namespace SIMDJSON_IMPLEMENTATION {
 namespace {
 
-// we don't have SSE, so let us use a scalar function
 // credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/
 static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) {
   uint64_t val;

include/simdjson/arm64/simd.h

@@ -57,19 +57,6 @@ simdjson_really_inline uint8x16_t make_uint8x16_t(uint8_t x1,  uint8_t x2,  uint
   return x;
 }
 
-simdjson_really_inline uint8x8_t make_uint8x8_t(uint8_t x1,  uint8_t x2,  uint8_t x3,  uint8_t x4,
-                                         uint8_t x5,  uint8_t x6,  uint8_t x7,  uint8_t x8) {
-  uint8x8_t x{};
-  x = vset_lane_u8(x1, x, 0);
-  x = vset_lane_u8(x2, x, 1);
-  x = vset_lane_u8(x3, x, 2);
-  x = vset_lane_u8(x4, x, 3);
-  x = vset_lane_u8(x5, x, 4);
-  x = vset_lane_u8(x6, x, 5);
-  x = vset_lane_u8(x7, x, 6);
-  x = vset_lane_u8(x8, x, 7);
-  return x;
-}
 
 // We have to do the same work for make_int8x16_t
 simdjson_really_inline int8x16_t make_int8x16_t(int8_t x1,  int8_t x2,  int8_t x3,  int8_t x4,
@@ -302,27 +289,6 @@ simdjson_really_inline int8x16_t make_int8x16_t(int8_t x1,  int8_t x2,  int8_t x
       vst1q_u8(reinterpret_cast<uint8_t*>(output), answer);
     }
 
-    // Copies all bytes corresponding to a 0 in the low half of the mask (interpreted as a
-    // bitset) to output1, then those corresponding to a 0 in the high half to output2.
-    template<typename L>
-    simdjson_really_inline void compress_halves(uint16_t mask, L *output1, L *output2) const {
-      using internal::thintable_epi8;
-      uint8_t mask1 = uint8_t(mask); // least significant 8 bits
-      uint8_t mask2 = uint8_t(mask >> 8); // most significant 8 bits
-      uint8x8_t compactmask1 = vcreate_u8(thintable_epi8[mask1]);
-      uint8x8_t compactmask2 = vcreate_u8(thintable_epi8[mask2]);
-      // we increment by 0x08 the second half of the mask
-#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
-      uint8x8_t inc = make_uint8x8_t(0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08);
-#else
-      uint8x8_t inc = {0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08};
-#endif
-      compactmask2 = vadd_u8(compactmask2, inc);
-      // store each result (with the second store possibly overlapping the first)
-      vst1_u8((uint8_t*)output1, vqtbl1_u8(*this, compactmask1));
-      vst1_u8((uint8_t*)output2, vqtbl1_u8(*this, compactmask2));
-    }
-
     template<typename L>
     simdjson_really_inline simd8<L> lookup_16(
         L replace0,  L replace1,  L replace2,  L replace3,
@@ -473,15 +439,11 @@ simdjson_really_inline int8x16_t make_int8x16_t(int8_t x1,  int8_t x2,  int8_t x
     }
 
 
-    simdjson_really_inline uint64_t compress(uint64_t mask, T * output) const {
-      uint64_t popcounts = vget_lane_u64(vreinterpret_u64_u8(vcnt_u8(vcreate_u8(~mask))), 0);
-      // compute the prefix sum of the popcounts of each byte
-      uint64_t offsets = popcounts * 0x0101010101010101;
-      this->chunks[0].compress_halves(uint16_t(mask), output, &output[popcounts & 0xFF]);
-      this->chunks[1].compress_halves(uint16_t(mask >> 16), &output[(offsets >> 8) & 0xFF], &output[(offsets >> 16) & 0xFF]);
-      this->chunks[2].compress_halves(uint16_t(mask >> 32), &output[(offsets >> 24) & 0xFF], &output[(offsets >> 32) & 0xFF]);
-      this->chunks[3].compress_halves(uint16_t(mask >> 48), &output[(offsets >> 40) & 0xFF], &output[(offsets >> 48) & 0xFF]);
-      return offsets >> 56;
+    simdjson_really_inline void compress(uint64_t mask, T * output) const {
+      this->chunks[0].compress(uint16_t(mask), output);
+      this->chunks[1].compress(uint16_t(mask >> 16), output + 16 - count_ones(mask & 0xFFFF));
+      this->chunks[2].compress(uint16_t(mask >> 32), output + 32 - count_ones(mask & 0xFFFFFFFF));
+      this->chunks[3].compress(uint16_t(mask >> 48), output + 48 - count_ones(mask & 0xFFFFFFFFFFFF));
     }
 
     simdjson_really_inline uint64_t to_bitmask() const {

include/simdjson/arm64/stringparsing.h

@@ -15,7 +15,10 @@ using namespace simd;
 struct backslash_and_quote {
 public:
   static constexpr uint32_t BYTES_PROCESSED = 32;
-  simdjson_really_inline static backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst);
+  simdjson_really_inline static backslash_and_quote
+  copy_and_find(const uint8_t *src, uint8_t *dst, const uint8_t *last_full_buf);
+  simdjson_really_inline static backslash_and_quote
+  copy_and_find(const uint8_t *src, uint8_t *dst);
 
   simdjson_really_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; }
   simdjson_really_inline bool has_backslash() { return bs_bits != 0; }
@@ -26,10 +29,21 @@ public:
   uint32_t quote_bits;
 }; // struct backslash_and_quote
 
-simdjson_really_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) {
-  // this can read up to 31 bytes beyond the buffer size, but we require
-  // SIMDJSON_PADDING of padding
-  static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes");
+simdjson_really_inline backslash_and_quote
+backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst, const uint8_t *last_full_buf) {
+  // If we don't have enough memory left to load a whole simd register, copy it first.
+  uint8_t tmpbuf[BYTES_PROCESSED];
+  if (simdjson_unlikely(src > last_full_buf)) {
+    SIMDJSON_ASSUME(last_full_buf+BYTES_PROCESSED > src);
+    std::memset(tmpbuf, 0, BYTES_PROCESSED);
+    std::memcpy(tmpbuf, src, last_full_buf+BYTES_PROCESSED-src);
+    src = tmpbuf;
+  }
+  return copy_and_find(src, dst);
+}
+
+simdjson_really_inline backslash_and_quote
+backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) {
   simd8<uint8_t> v0(src);
   simd8<uint8_t> v1(src + sizeof(v0));
   v0.store(dst);

include/simdjson/common_defs.h

@@ -35,12 +35,7 @@ double from_chars(const char *first, const char* end) noexcept;
 constexpr size_t SIMDJSON_MAXSIZE_BYTES = 0xFFFFFFFF;
 
 /**
- * The amount of padding needed in a buffer to parse JSON.
- *
- * the input buf should be readable up to buf + SIMDJSON_PADDING
- * this is a stopgap; there should be a better description of the
- * main loop and its behavior that abstracts over this
- * See https://github.com/simdjson/simdjson/issues/174
+ * Padding requirement.
  */
 constexpr size_t SIMDJSON_PADDING = 32;
 
@@ -71,8 +66,12 @@ constexpr size_t DEFAULT_MAX_DEPTH = 1024;
 #define SIMDJSON_ISALIGNED_N(ptr, n) (((uintptr_t)(ptr) & ((n)-1)) == 0)
 
 #if defined(SIMDJSON_REGULAR_VISUAL_STUDIO)
-
+  #if SIMDJSON_NO_FORCE_INLINING
+  // forcing inlining can increase stack usage.
+  #define simdjson_really_inline inline
+  #else
   #define simdjson_really_inline __forceinline
+  #endif
   #define simdjson_never_inline __declspec(noinline)
 
   #define simdjson_unused
@@ -106,8 +105,12 @@ constexpr size_t DEFAULT_MAX_DEPTH = 1024;
   #define SIMDJSON_POP_DISABLE_WARNINGS __pragma(warning( pop ))
 
 #else // SIMDJSON_REGULAR_VISUAL_STUDIO
-
+  #if SIMDJSON_NO_FORCE_INLINING
+  // forcing inlining can increase stack usage.
+  #define simdjson_really_inline inline
+  #else
   #define simdjson_really_inline inline __attribute__((always_inline))
+  #endif
   #define simdjson_never_inline inline __attribute__((noinline))
 
   #define simdjson_unused __attribute__((unused))
@@ -254,7 +257,15 @@ namespace std {
 #endif
 
 // The SIMDJSON_CHECK_EOF macro is a feature flag for the "don't require padding"
-// feature.
+// feature in the On Demand API.
+// When we have padding, we do not need to check for the end of the input buffer.
+// However, without padding, it is unsafe not to have end-of-buffer checks.
+// Thus this SIMDJSON_CHECK_EOF should be set to true (1) for safety as it activates
+// several safety checks. We still allow expert users to disable it.
+// Note that this only affects the On Demand API.
+#ifndef SIMDJSON_CHECK_EOF
+# define SIMDJSON_CHECK_EOF 1
+#endif
 
 #if SIMDJSON_CPLUSPLUS17
 // if we have C++, then fallthrough is a default attribute
@@ -267,6 +278,7 @@ namespace std {
 # define simdjson_fallthrough                    __attribute__((__fallthrough__))
 #endif // __has_attribute(__fallthrough__)
 #endif // SIMDJSON_CPLUSPLUS17
+
 // on some systems, we simply do not have support for fallthrough, so use a default:
 #ifndef simdjson_fallthrough
 # define simdjson_fallthrough do {} while (0)  /* fallthrough */

include/simdjson/dom/document_stream-inl.h

@@ -256,7 +256,6 @@ inline size_t document_stream::size_in_bytes() const noexcept {
 }
 
 inline size_t document_stream::truncated_bytes() const noexcept {
-  if(error == CAPACITY) { return len - batch_start; }
   return parser->implementation->structural_indexes[parser->implementation->n_structural_indexes] - parser->implementation->structural_indexes[parser->implementation->n_structural_indexes + 1];
 }
 

include/simdjson/dom/parsedjson_iterator.h

@@ -5,7 +5,7 @@
 
 #include <cstring>
 #include <string>
-#include <ostream>
+#include <iostream>
 #include <iterator>
 #include <limits>
 #include <stdexcept>

include/simdjson/dom/parser-inl.h

@@ -100,7 +100,8 @@ inline simdjson_result<element> parser::parse_into_document(document& provided_d
   // Important: It is possible that provided_doc is actually the internal 'doc' within the parser!!!
   error_code _error = ensure_capacity(provided_doc, len);
   if (_error) { return _error; }
-  if (realloc_if_needed) {
+  (void)realloc_if_needed;
+  /*if (realloc_if_needed) {
     // Make sure we have enough capacity to copy len bytes
     if (!loaded_bytes || _loaded_bytes_capacity < len) {
       loaded_bytes.reset( internal::allocate_padded_buffer(len) );
@@ -112,6 +113,8 @@ inline simdjson_result<element> parser::parse_into_document(document& provided_d
     std::memcpy(static_cast<void *>(loaded_bytes.get()), buf, len);
   }
   _error = implementation->parse(realloc_if_needed ? reinterpret_cast<const uint8_t*>(loaded_bytes.get()): buf, len, provided_doc);
+  */
+ _error = implementation->parse(buf, len, provided_doc);
 
   if (_error) { return _error; }
 
@@ -122,8 +125,16 @@ simdjson_really_inline simdjson_result<element> parser::parse_into_document(docu
   return parse_into_document(provided_doc, reinterpret_cast<const uint8_t *>(buf), len, realloc_if_needed);
 }
 simdjson_really_inline simdjson_result<element> parser::parse_into_document(document& provided_doc, const std::string &s) & noexcept {
-  return parse_into_document(provided_doc, s.data(), s.length(), s.capacity() - s.length() < SIMDJSON_PADDING);
+  return parse_into_document(provided_doc, s.data(), s.length());
 }
+simdjson_really_inline simdjson_result<element> parser::parse_into_document(document& provided_doc, std::string&& s) & noexcept {
+  std::string local_string(s);
+  return parse_into_document(provided_doc, local_string.data(), local_string.length());
+}
+simdjson_really_inline simdjson_result<element> parser::parse_into_document(document& provided_doc, std::string_view s) & noexcept {
+  return parse_into_document(provided_doc, s.data(), s.length());
+}
+
 simdjson_really_inline simdjson_result<element> parser::parse_into_document(document& provided_doc, const padded_string &s) & noexcept {
   return parse_into_document(provided_doc, s.data(), s.length(), false);
 }
@@ -137,7 +148,14 @@ simdjson_really_inline simdjson_result<element> parser::parse(const char *buf, s
   return parse(reinterpret_cast<const uint8_t *>(buf), len, realloc_if_needed);
 }
 simdjson_really_inline simdjson_result<element> parser::parse(const std::string &s) & noexcept {
-  return parse(s.data(), s.length(), s.capacity() - s.length() < SIMDJSON_PADDING);
+  return parse(s.data(), s.length());
+}
+simdjson_really_inline simdjson_result<element> parser::parse(std::string &&s) & noexcept {
+  std::string local_string(s);
+  return parse(local_string.data(), local_string.length());
+}
+simdjson_really_inline simdjson_result<element> parser::parse(std::string_view s) & noexcept {
+  return parse(s.data(), s.length());
 }
 simdjson_really_inline simdjson_result<element> parser::parse(const padded_string &s) & noexcept {
   return parse(s.data(), s.length(), false);
@@ -153,6 +171,9 @@ inline simdjson_result<document_stream> parser::parse_many(const char *buf, size
 inline simdjson_result<document_stream> parser::parse_many(const std::string &s, size_t batch_size) noexcept {
   return parse_many(s.data(), s.length(), batch_size);
 }
+inline simdjson_result<document_stream> parser::parse_many(const std::string_view s, size_t batch_size) noexcept {
+  return parse_many(s.data(), s.length(), batch_size);
+}
 inline simdjson_result<document_stream> parser::parse_many(const padded_string &s, size_t batch_size) noexcept {
   return parse_many(s.data(), s.length(), batch_size);
 }

include/simdjson/dom/parser.h

@@ -142,63 +142,43 @@ public:
    * the parser instance without moving it by wrapping it inside an `unique_ptr` instance like
    * so: `std::unique_ptr<dom::parser> parser(new dom::parser{});`.
    *
-   * ### REQUIRED: Buffer Padding
+   * ### Buffer Padding
    *
-   * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what
-   * those bytes are initialized to, as long as they are allocated.
-   *
-   * If realloc_if_needed is true (the default), it is assumed that the buffer does *not* have enough padding,
-   * and it is copied into an enlarged temporary buffer before parsing. Thus the following is safe:
-   *
-   *   const char *json      = R"({"key":"value"})";
-   *   const size_t json_len = std::strlen(json);
-   *   simdjson::dom::parser parser;
-   *   simdjson::dom::element element = parser.parse(json, json_len);
-   *
-   * If you set realloc_if_needed to false (e.g., parser.parse(json, json_len, false)),
-   * you must provide a buffer with at least SIMDJSON_PADDING extra bytes at the end.
-   * The benefit of setting realloc_if_needed to false is that you avoid a temporary
-   * memory allocation and a copy.
-   *
-   * The padded bytes may be read. It is not important how you initialize
-   * these bytes though we recommend a sensible default like null character values or spaces.
-   * For example, the following low-level code is safe:
-   *
-   *   const char *json      = R"({"key":"value"})";
-   *   const size_t json_len = std::strlen(json);
-   *   std::unique_ptr<char[]> padded_json_copy{new char[json_len + SIMDJSON_PADDING]};
-   *   std::memcpy(padded_json_copy.get(), json, json_len);
-   *   std::memset(padded_json_copy.get() + json_len, '\0', SIMDJSON_PADDING);
-   *   simdjson::dom::parser parser;
-   *   simdjson::dom::element element = parser.parse(padded_json_copy.get(), json_len, false);
+   * We do not require that the input buffer be padded, but for backward compatibility, we have
+   * a realloc_if_needed parameter with a default value. It can be ignored.
    *
    * ### Parser Capacity
    *
    * If the parser's current capacity is less than len, it will allocate enough capacity
    * to handle it (up to max_capacity).
    *
-   * @param buf The JSON to parse. Must have at least len + SIMDJSON_PADDING allocated bytes, unless
-   *            realloc_if_needed is true.
+   * @param buf The JSON to parse.
    * @param len The length of the JSON.
-   * @param realloc_if_needed Whether to reallocate and enlarge the JSON buffer to add padding.
+   * @param realloc_if_needed this parameter is left in place for backward compatibility, it can be ignored (defaults to false)
    * @return An element pointing at the root of the document, or an error:
-   *         - MEMALLOC if realloc_if_needed is true or the parser does not have enough capacity,
+   *         - MEMALLOC if the parser does not have enough capacity,
    *           and memory allocation fails.
    *         - CAPACITY if the parser does not have enough capacity and len > max_capacity.
    *         - other json errors if parsing fails. You should not rely on these errors to always the same for the
    *           same document: they may vary under runtime dispatch (so they may vary depending on your system and hardware).
    */
-  inline simdjson_result<element> parse(const uint8_t *buf, size_t len, bool realloc_if_needed = true) & noexcept;
-  inline simdjson_result<element> parse(const uint8_t *buf, size_t len, bool realloc_if_needed = true) && =delete;
+  inline simdjson_result<element> parse(const uint8_t *buf, size_t len, bool realloc_if_needed = false) & noexcept;
+  inline simdjson_result<element> parse(const uint8_t *buf, size_t len, bool realloc_if_needed = false) && =delete;
   /** @overload parse(const uint8_t *buf, size_t len, bool realloc_if_needed) */
-  simdjson_really_inline simdjson_result<element> parse(const char *buf, size_t len, bool realloc_if_needed = true) & noexcept;
-  simdjson_really_inline simdjson_result<element> parse(const char *buf, size_t len, bool realloc_if_needed = true) && =delete;
+  simdjson_really_inline simdjson_result<element> parse(const char *buf, size_t len, bool realloc_if_needed = false) & noexcept;
+  simdjson_really_inline simdjson_result<element> parse(const char *buf, size_t len, bool realloc_if_needed = false) && =delete;
   /** @overload parse(const uint8_t *buf, size_t len, bool realloc_if_needed) */
+  simdjson_really_inline simdjson_result<element> parse(std::string_view s) & noexcept;
+  simdjson_really_inline simdjson_result<element> parse(std::string_view s) && = delete;
   simdjson_really_inline simdjson_result<element> parse(const std::string &s) & noexcept;
-  simdjson_really_inline simdjson_result<element> parse(const std::string &s) && =delete;
+  simdjson_really_inline simdjson_result<element> parse(const std::string &s) && = delete;
+  simdjson_really_inline simdjson_result<element> parse(std::string &&s) & noexcept;
+  simdjson_really_inline simdjson_result<element> parse(std::string &&s) && = delete;
+
   /** @overload parse(const uint8_t *buf, size_t len, bool realloc_if_needed) */
   simdjson_really_inline simdjson_result<element> parse(const padded_string &s) & noexcept;
   simdjson_really_inline simdjson_result<element> parse(const padded_string &s) && =delete;
+  simdjson_really_inline simdjson_result<element> parse(const padded_string &&s) = delete;
 
   /** @private We do not want to allow implicit conversion from C string to std::string. */
   simdjson_really_inline simdjson_result<element> parse(const char *buf) noexcept = delete;
@@ -236,25 +216,28 @@ public:
    * moving a document, you can recover safe access to the document root with its `root()` method.
    *
    * @param doc The document instance where the parsed data will be stored (on success).
-   * @param buf The JSON to parse. Must have at least len + SIMDJSON_PADDING allocated bytes, unless
-   *            realloc_if_needed is true.
+   * @param buf The JSON to parse.
    * @param len The length of the JSON.
-   * @param realloc_if_needed Whether to reallocate and enlarge the JSON buffer to add padding.
+   * @param realloc_if_needed this parameter is left in place for backward compatibility, it can be ignored (defaults to false)
    * @return An element pointing at the root of document, or an error:
-   *         - MEMALLOC if realloc_if_needed is true or the parser does not have enough capacity,
+   *         - MEMALLOC if the parser does not have enough capacity,
    *           and memory allocation fails.
    *         - CAPACITY if the parser does not have enough capacity and len > max_capacity.
    *         - other json errors if parsing fails. You should not rely on these errors to always the same for the
    *           same document: they may vary under runtime dispatch (so they may vary depending on your system and hardware).
    */
-  inline simdjson_result<element> parse_into_document(document& doc, const uint8_t *buf, size_t len, bool realloc_if_needed = true) & noexcept;
-  inline simdjson_result<element> parse_into_document(document& doc, const uint8_t *buf, size_t len, bool realloc_if_needed = true) && =delete;
+  inline simdjson_result<element> parse_into_document(document& doc, const uint8_t *buf, size_t len, bool realloc_if_needed = false) & noexcept;
+  inline simdjson_result<element> parse_into_document(document& doc, const uint8_t *buf, size_t len, bool realloc_if_needed = false) && =delete;
   /** @overload parse_into_document(const uint8_t *buf, size_t len, bool realloc_if_needed) */
-  simdjson_really_inline simdjson_result<element> parse_into_document(document& doc, const char *buf, size_t len, bool realloc_if_needed = true) & noexcept;
-  simdjson_really_inline simdjson_result<element> parse_into_document(document& doc, const char *buf, size_t len, bool realloc_if_needed = true) && =delete;
+  simdjson_really_inline simdjson_result<element> parse_into_document(document& doc, const char *buf, size_t len, bool realloc_if_needed = false) & noexcept;
+  simdjson_really_inline simdjson_result<element> parse_into_document(document& doc, const char *buf, size_t len, bool realloc_if_needed = false) && =delete;
   /** @overload parse_into_document(const uint8_t *buf, size_t len, bool realloc_if_needed) */
   simdjson_really_inline simdjson_result<element> parse_into_document(document& doc, const std::string &s) & noexcept;
   simdjson_really_inline simdjson_result<element> parse_into_document(document& doc, const std::string &s) && =delete;
+  simdjson_really_inline simdjson_result<element> parse_into_document(document& doc, std::string &&s) & noexcept;
+  simdjson_really_inline simdjson_result<element> parse_into_document(document& doc, std::string &&s) && =delete;
+  simdjson_really_inline simdjson_result<element> parse_into_document(document& doc, std::string_view s) & noexcept;
+  simdjson_really_inline simdjson_result<element> parse_into_document(document& doc, std::string_view s) && =delete;
   /** @overload parse_into_document(const uint8_t *buf, size_t len, bool realloc_if_needed) */
   simdjson_really_inline simdjson_result<element> parse_into_document(document& doc, const padded_string &s) & noexcept;
   simdjson_really_inline simdjson_result<element> parse_into_document(document& doc, const padded_string &s) && =delete;
@@ -402,10 +385,6 @@ public:
    *     cout << title << endl;
    *   }
    *
-   * ### REQUIRED: Buffer Padding
-   *
-   * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what
-   * those bytes are initialized to, as long as they are allocated.
    *
    * ### Threads
    *
@@ -417,7 +396,7 @@ public:
    * If the parser's current capacity is less than batch_size, it will allocate enough capacity
    * to handle it (up to max_capacity).
    *
-   * @param buf The concatenated JSON to parse. Must have at least len + SIMDJSON_PADDING allocated bytes.
+   * @param buf The concatenated JSON to parse.
    * @param len The length of the concatenated JSON.
    * @param batch_size The batch size to use. MUST be larger than the largest document. The sweet
    *                   spot is cache-related: small enough to fit in cache, yet big enough to
@@ -434,10 +413,11 @@ public:
   inline simdjson_result<document_stream> parse_many(const char *buf, size_t len, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept;
   /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */
   inline simdjson_result<document_stream> parse_many(const std::string &s, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept;
-  inline simdjson_result<document_stream> parse_many(const std::string &&s, size_t batch_size) = delete;// unsafe
+  inline simdjson_result<document_stream> parse_many(const std::string_view s, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept;
+  inline simdjson_result<document_stream> parse_many(std::string &&s, size_t batch_size) = delete;// unsafe
   /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */
   inline simdjson_result<document_stream> parse_many(const padded_string &s, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept;
-  inline simdjson_result<document_stream> parse_many(const padded_string &&s, size_t batch_size) = delete;// unsafe
+  inline simdjson_result<document_stream> parse_many(padded_string &&s, size_t batch_size) = delete;// unsafe
 
   /** @private We do not want to allow implicit conversion from C string to std::string. */
   simdjson_result<document_stream> parse_many(const char *buf, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept = delete;

include/simdjson/dom/serialization-inl.h

@@ -157,10 +157,7 @@ simdjson_really_inline void mini_formatter::string(std::string_view unescaped) {
   size_t i = 0;
   // Fast path for the case where we have no control character, no ", and no backslash.
   // This should include most keys.
-  //
-  // We would like to use 'bool' but some compilers take offense to bitwise operation
-  // with bool types.
-  constexpr static char needs_escaping[] = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  constexpr static bool needs_escaping[] = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0,
     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0,
@@ -172,8 +169,6 @@ simdjson_really_inline void mini_formatter::string(std::string_view unescaped) {
     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
   for(;i + 8 <= unescaped.length(); i += 8) {
     // Poor's man vectorization. This could get much faster if we used SIMD.
-    //
-    // It is not the case that replacing '|' with '||' would be neutral performance-wise.
     if(needs_escaping[uint8_t(unescaped[i])] | needs_escaping[uint8_t(unescaped[i+1])]
       | needs_escaping[uint8_t(unescaped[i+2])] | needs_escaping[uint8_t(unescaped[i+3])]
       | needs_escaping[uint8_t(unescaped[i+4])] | needs_escaping[uint8_t(unescaped[i+5])]

include/simdjson/error.h

@@ -38,8 +38,6 @@ enum error_code {
   OUT_OF_ORDER_ITERATION,     ///< tried to iterate an array or object out of order
   INSUFFICIENT_PADDING,       ///< The JSON doesn't have enough padding for simdjson to safely parse it.
   INCOMPLETE_ARRAY_OR_OBJECT, ///< The document ends early.
-  SCALAR_DOCUMENT_AS_VALUE,   ///< A scalar document is treated as a value.
-  OUT_OF_BOUNDS,              ///< Attempted to access location outside of document.
   NUM_ERROR_CODES
 };
 
@@ -289,7 +287,7 @@ struct simdjson_result : public internal::simdjson_result_base<T> {
 #if SIMDJSON_EXCEPTIONS
 
 template<typename T>
-inline std::ostream& operator<<(std::ostream& out, simdjson_result<T> value) { return out << value.value(); }
+inline std::ostream& operator<<(std::ostream& out, simdjson_result<T> value) noexcept { return out << value.value(); }
 #endif // SIMDJSON_EXCEPTIONS
 
 #ifndef SIMDJSON_DISABLE_DEPRECATED_API

include/simdjson/fallback/numberparsing.h

@@ -28,7 +28,6 @@ static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t
 } // namespace simdjson
 
 #define SIMDJSON_SWAR_NUMBER_PARSING 1
-
 #include "simdjson/generic/numberparsing.h"
 
 #endif // SIMDJSON_FALLBACK_NUMBERPARSING_H

include/simdjson/fallback/stringparsing.h

@@ -11,7 +11,10 @@ namespace {
 struct backslash_and_quote {
 public:
   static constexpr uint32_t BYTES_PROCESSED = 1;
-  simdjson_really_inline static backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst);
+  simdjson_really_inline static backslash_and_quote
+  copy_and_find(const uint8_t *src, uint8_t *dst, const uint8_t *last_full_buf);
+  simdjson_really_inline static backslash_and_quote
+  copy_and_find(const uint8_t *src, uint8_t *dst);
 
   simdjson_really_inline bool has_quote_first() { return c == '"'; }
   simdjson_really_inline bool has_backslash() { return c == '\\'; }
@@ -21,6 +24,10 @@ public:
   uint8_t c;
 }; // struct backslash_and_quote
 
+simdjson_really_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst, const uint8_t *) {
+  return copy_and_find(src, dst);
+}
+
 simdjson_really_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) {
   // store to dest unconditionally - we can overwrite the bits we don't like later
   dst[0] = src[0];

include/simdjson/generic/atomparsing.h

@@ -15,47 +15,50 @@ simdjson_really_inline uint32_t string_to_uint32(const char* str) { uint32_t val
 
 // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive.
 // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about.
+// It is unsafe because you must ensure that you have at least 4 characters before calling this function!
 simdjson_warn_unused
-simdjson_really_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) {
+simdjson_really_inline uint32_t str4ncmp_unsafe(const uint8_t *src, const char* atom) {
   uint32_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++)
-  static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 4 bytes");
   std::memcpy(&srcval, src, sizeof(uint32_t));
   return srcval ^ string_to_uint32(atom);
 }
 
+// It is unsafe because you must ensure that you have at least 5 characters before calling this function!
 simdjson_warn_unused
-simdjson_really_inline bool is_valid_true_atom(const uint8_t *src) {
-  return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0;
+simdjson_really_inline bool is_valid_true_atom_unsafe(const uint8_t *src) {
+  return (str4ncmp_unsafe(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0;
 }
 
 simdjson_warn_unused
 simdjson_really_inline bool is_valid_true_atom(const uint8_t *src, size_t len) {
-  if (len > 4) { return is_valid_true_atom(src); }
-  else if (len == 4) { return !str4ncmp(src, "true"); }
+  if (len > 4) { return is_valid_true_atom_unsafe(src); }
+  else if (len == 4) { return !str4ncmp_unsafe(src, "true"); }
   else { return false; }
 }
 
+// It is unsafe because you must ensure that you have at least 5 characters before calling this function!
 simdjson_warn_unused
-simdjson_really_inline bool is_valid_false_atom(const uint8_t *src) {
-  return (str4ncmp(src+1, "alse") | jsoncharutils::is_not_structural_or_whitespace(src[5])) == 0;
+simdjson_really_inline bool is_valid_false_atom_unsafe(const uint8_t *src) {
+  return (str4ncmp_unsafe(src+1, "alse") | jsoncharutils::is_not_structural_or_whitespace(src[5])) == 0;
 }
 
 simdjson_warn_unused
 simdjson_really_inline bool is_valid_false_atom(const uint8_t *src, size_t len) {
-  if (len > 5) { return is_valid_false_atom(src); }
-  else if (len == 5) { return !str4ncmp(src+1, "alse"); }
+  if (len > 5) { return is_valid_false_atom_unsafe(src); }
+  else if (len == 5) { return !str4ncmp_unsafe(src+1, "alse"); }
   else { return false; }
 }
 
+// It is unsafe because you must ensure that you have at least 5 characters before calling this function!
 simdjson_warn_unused
-simdjson_really_inline bool is_valid_null_atom(const uint8_t *src) {
-  return (str4ncmp(src, "null") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0;
+simdjson_really_inline bool is_valid_null_atom_unsafe(const uint8_t *src) {
+  return (str4ncmp_unsafe(src, "null") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0;
 }
 
 simdjson_warn_unused
 simdjson_really_inline bool is_valid_null_atom(const uint8_t *src, size_t len) {
-  if (len > 4) { return is_valid_null_atom(src); }
-  else if (len == 4) { return !str4ncmp(src, "null"); }
+  if (len > 4) { return is_valid_null_atom_unsafe(src); }
+  else if (len == 4) { return !str4ncmp_unsafe(src, "null"); }
   else { return false; }
 }
 

include/simdjson/generic/dom_parser_implementation.h

@@ -4,27 +4,8 @@
 namespace simdjson {
 namespace SIMDJSON_IMPLEMENTATION {
 
-// expectation: sizeof(open_container) = 64/8.
-struct open_container {
-  uint32_t tape_index; // where, on the tape, does the scope ([,{) begins
-  uint32_t count; // how many elements in the scope
-}; // struct open_container
-
-static_assert(sizeof(open_container) == 64/8, "Open container must be 64 bits");
-
 class dom_parser_implementation final : public internal::dom_parser_implementation {
 public:
-  /** Tape location of each open { or [ */
-  std::unique_ptr<open_container[]> open_containers{};
-  /** Whether each open container is a [ or { */
-  std::unique_ptr<bool[]> is_array{};
-  /** Buffer passed to stage 1 */
-  const uint8_t *buf{};
-  /** Length passed to stage 1 */
-  size_t len{0};
-  /** Document passed to stage 2 */
-  dom::document *doc{};
-
   inline dom_parser_implementation() noexcept;
   inline dom_parser_implementation(dom_parser_implementation &&other) noexcept;
   inline dom_parser_implementation &operator=(dom_parser_implementation &&other) noexcept;
@@ -68,7 +49,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s
 
 inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept {
   // Stage 2 stacks
-  open_containers.reset(new (std::nothrow) open_container[max_depth]);
+  open_containers.reset(new (std::nothrow) internal::open_container[max_depth]);
   is_array.reset(new (std::nothrow) bool[max_depth]);
   if (!is_array || !open_containers) { _max_depth = 0; return MEMALLOC; }
 

include/simdjson/generic/implementation_simdjson_result_base-inl.h

@@ -49,23 +49,18 @@ simdjson_really_inline implementation_simdjson_result_base<T>::operator T&&() &&
   return std::forward<implementation_simdjson_result_base<T>>(*this).take_value();
 }
 
-#endif // SIMDJSON_EXCEPTIONS
-
 template<typename T>
 simdjson_really_inline const T& implementation_simdjson_result_base<T>::value_unsafe() const& noexcept {
   return this->first;
 }
 
-template<typename T>
-simdjson_really_inline T& implementation_simdjson_result_base<T>::value_unsafe() & noexcept {
-  return this->first;
-}
-
 template<typename T>
 simdjson_really_inline T&& implementation_simdjson_result_base<T>::value_unsafe() && noexcept {
   return std::forward<T>(this->first);
 }
 
+#endif // SIMDJSON_EXCEPTIONS
+
 template<typename T>
 simdjson_really_inline implementation_simdjson_result_base<T>::implementation_simdjson_result_base(T &&value, error_code error) noexcept
     : first{std::forward<T>(value)}, second{error} {}

include/simdjson/generic/implementation_simdjson_result_base.h

@@ -97,28 +97,22 @@ struct implementation_simdjson_result_base {
    */
   simdjson_really_inline operator T&&() && noexcept(false);
 
-
-#endif // SIMDJSON_EXCEPTIONS
-
   /**
    * Get the result value. This function is safe if and only
    * the error() method returns a value that evaluates to false.
    */
   simdjson_really_inline const T& value_unsafe() const& noexcept;
-  /**
-   * Get the result value. This function is safe if and only
-   * the error() method returns a value that evaluates to false.
-   */
-  simdjson_really_inline T& value_unsafe() & noexcept;
+
   /**
    * Take the result value (move it). This function is safe if and only
    * the error() method returns a value that evaluates to false.
    */
   simdjson_really_inline T&& value_unsafe() && noexcept;
-protected:
-  /** users should never directly access first and second. **/
-  T first{}; /** Users should never directly access 'first'. **/
-  error_code second{UNINITIALIZED}; /** Users should never directly access 'second'. **/
+
+#endif // SIMDJSON_EXCEPTIONS
+
+  T first{};
+  error_code second{UNINITIALIZED};
 }; // struct implementation_simdjson_result_base
 
 } // namespace SIMDJSON_IMPLEMENTATION

include/simdjson/generic/numberparsing.h

@@ -3,18 +3,6 @@
 
 namespace simdjson {
 namespace SIMDJSON_IMPLEMENTATION {
-
-namespace ondemand {
-/**
- * The type of a JSON number
- */
-enum class number_type {
-    floating_point_number=1, /// a binary64 number
-    signed_integer,          /// a signed integer that fits in a 64-bit word using two's complement
-    unsigned_integer         /// a positive integer larger or equal to 1<<63
-};
-}
-
 namespace {
 /// @private
 namespace numberparsing {
@@ -303,8 +291,8 @@ simdjson_really_inline bool compute_float_64(int64_t power, uint64_t i, bool neg
 // It will return an error (false) if the parsed number is infinite.
 // The string parsing itself always succeeds. We know that there is at least
 // one digit.
-static bool parse_float_fallback(const uint8_t *ptr, double *outDouble) {
-  *outDouble = simdjson::internal::from_chars(reinterpret_cast<const char *>(ptr));
+bool parse_float_fallback(const uint8_t * const ptr, const uint8_t * const end, double *outDouble) {
+  *outDouble = simdjson::internal::from_chars(reinterpret_cast<const char *>(ptr), reinterpret_cast<const char *>(end));
   // We do not accept infinite values.
 
   // Detecting finite values in a portable manner is ridiculously hard, ideally
@@ -317,29 +305,17 @@ static bool parse_float_fallback(const uint8_t *ptr, double *outDouble) {
   // to handle that max may be a macro on windows).
   return !(*outDouble > (std::numeric_limits<double>::max)() || *outDouble < std::numeric_limits<double>::lowest());
 }
-static bool parse_float_fallback(const uint8_t *ptr, const uint8_t *end_ptr, double *outDouble) {
-  *outDouble = simdjson::internal::from_chars(reinterpret_cast<const char *>(ptr), reinterpret_cast<const char *>(end_ptr));
-  // We do not accept infinite values.
-
-  // Detecting finite values in a portable manner is ridiculously hard, ideally
-  // we would want to do:
-  // return !std::isfinite(*outDouble);
-  // but that mysteriously fails under legacy/old libc++ libraries, see
-  // https://github.com/simdjson/simdjson/issues/1286
-  //
-  // Therefore, fall back to this solution (the extra parens are there
-  // to handle that max may be a macro on windows).
-  return !(*outDouble > (std::numeric_limits<double>::max)() || *outDouble < std::numeric_limits<double>::lowest());
-}
-
 // check quickly whether the next 8 chars are made of digits
 // at a glance, it looks better than Mula's
 // http://0x80.pl/articles/swar-digits-validate.html
-simdjson_really_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) {
+simdjson_really_inline bool is_made_of_eight_digits_fast(const uint8_t * const chars, const uint8_t * const end) {
   uint64_t val;
-  // this can read up to 7 bytes beyond the buffer size, but we require
-  // SIMDJSON_PADDING of padding
-  static_assert(7 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 7");
+  // end == nullptr is forbidden here since we have SIMDJSON_SWAR_NUMBER_PARSING
+#ifndef SIMDJSON_SWAR_NUMBER_PARSING
+#warning "You should never call is_made_of_eight_digits_fast given that SIMDJSON_SWAR_NUMBER_PARSING is undefined."
+#endif
+  SIMDJSON_ASSUME(end != nullptr);
+  if ((end-chars) < 8) { return false; }
   std::memcpy(&val, chars, 8);
   // a branchy method might be faster:
   // return (( val & 0xF0F0F0F0F0F0F0F0 ) == 0x3030303030303030)
@@ -351,19 +327,34 @@ simdjson_really_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) {
 }
 
 template<typename W>
-error_code slow_float_parsing(simdjson_unused const uint8_t * src, W writer) {
+error_code slow_float_parsing(const uint8_t * src, W writer) {
   double d;
-  if (parse_float_fallback(src, &d)) {
+  if (parse_float_fallback(src, nullptr, &d)) {
     writer.append_double(d);
     return SUCCESS;
   }
   return INVALID_NUMBER(src);
 }
 
+template<typename W>
+error_code slow_float_parsing(const uint8_t * src, const uint8_t * const end, W writer) {
+  double d;
+  if (parse_float_fallback(src, end, &d)) {
+    writer.append_double(d);
+    return SUCCESS;
+  }
+  return INVALID_NUMBER(src);
+}
+
+simdjson_really_inline bool at_end(const uint8_t * const p, const uint8_t * const end) {
+  return end != nullptr && p >= end;
+}
+
 template<typename I>
 NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later
-simdjson_really_inline bool parse_digit(const uint8_t c, I &i) {
-  const uint8_t digit = static_cast<uint8_t>(c - '0');
+simdjson_really_inline bool parse_digit(const uint8_t * const p, const uint8_t * const end, I &i) {
+  if (at_end(p, end)) { return false; }
+  const uint8_t digit = static_cast<uint8_t>(*p - '0');
   if (digit > 9) {
     return false;
   }
@@ -372,26 +363,24 @@ simdjson_really_inline bool parse_digit(const uint8_t c, I &i) {
   return true;
 }
 
-simdjson_really_inline error_code parse_decimal(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) {
+simdjson_really_inline error_code parse_decimal(simdjson_unused const uint8_t * const src, const uint8_t * const end, const uint8_t *&p, uint64_t &i, int64_t &exponent) {
   // we continue with the fiction that we have an integer. If the
   // floating point number is representable as x * 10^z for some integer
   // z that fits in 53 bits, then we will be able to convert back the
   // the integer into a float in a lossless manner.
-  const uint8_t *const first_after_period = p;
+  const uint8_t * const first_after_period = p;
 
 #ifdef SIMDJSON_SWAR_NUMBER_PARSING
-#if SIMDJSON_SWAR_NUMBER_PARSING
   // this helps if we have lots of decimals!
   // this turns out to be frequent enough.
-  if (is_made_of_eight_digits_fast(p)) {
+  if (is_made_of_eight_digits_fast(p, end)) {
     i = i * 100000000 + parse_eight_digits_unrolled(p);
     p += 8;
   }
-#endif // SIMDJSON_SWAR_NUMBER_PARSING
-#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING
+#endif
   // Unrolling the first digit makes a small difference on some implementations (e.g. westmere)
-  if (parse_digit(*p, i)) { ++p; }
-  while (parse_digit(*p, i)) { p++; }
+  if (parse_digit(p, end, i)) { ++p; }
+  while (parse_digit(p, end, i)) { ++p; }
   exponent = first_after_period - p;
   // Decimal without digits (123.) is illegal
   if (exponent == 0) {
@@ -400,15 +389,15 @@ simdjson_really_inline error_code parse_decimal(simdjson_unused const uint8_t *c
   return SUCCESS;
 }
 
-simdjson_really_inline error_code parse_exponent(simdjson_unused const uint8_t *const src, const uint8_t *&p, int64_t &exponent) {
+simdjson_really_inline error_code parse_exponent(simdjson_unused const uint8_t * const src, const uint8_t * const end, const uint8_t *&p, int64_t &exponent) {
   // Exp Sign: -123.456e[-]78
-  bool neg_exp = ('-' == *p);
-  if (neg_exp || '+' == *p) { p++; } // Skip + as well
+  bool neg_exp = (!at_end(p, end) && '-' == *p);
+  if (neg_exp || (!at_end(p, end) && '+' == *p)) { ++p; } // Skip + as well
 
   // Exponent: -123.456e-[78]
   auto start_exp = p;
   int64_t exp_number = 0;
-  while (parse_digit(*p, exp_number)) { ++p; }
+  while (parse_digit(p, end, exp_number)) { ++p; }
   // It is possible for parse_digit to overflow.
   // In particular, it could overflow to INT64_MIN, and we cannot do - INT64_MIN.
   // Thus we *must* check for possible overflow before we negate exp_number.
@@ -430,7 +419,7 @@ simdjson_really_inline error_code parse_exponent(simdjson_unused const uint8_t *
   // something!!!!
   if (simdjson_unlikely(p > start_exp+18)) {
     // Skip leading zeroes: 1e000000000000000000001 is technically valid and doesn't overflow
-    while (*start_exp == '0') { start_exp++; }
+    while (*start_exp == '0') { ++start_exp; }
     // 19 digits could overflow int64_t and is kind of absurd anyway. We don't
     // support exponents smaller than -999,999,999,999,999,999 and bigger
     // than 999,999,999,999,999,999.
@@ -455,13 +444,15 @@ simdjson_really_inline size_t significant_digits(const uint8_t * start_digits, s
   // It is possible that the integer had an overflow.
   // We have to handle the case where we have 0.0000somenumber.
   const uint8_t *start = start_digits;
-  while ((*start == '0') || (*start == '.')) { ++start; }
+  while ((*start == '0') || (*start == '.')) {
+    ++start;
+  }
   // we over-decrement by one when there is a '.'
   return digit_count - size_t(start - start_digits);
 }
 
 template<typename W>
-simdjson_really_inline error_code write_float(const uint8_t *const src, bool negative, uint64_t i, const uint8_t * start_digits, size_t digit_count, int64_t exponent, W &writer) {
+simdjson_really_inline error_code write_float(const uint8_t * const src, const uint8_t * const end, bool negative, uint64_t i, const uint8_t * start_digits, size_t digit_count, int64_t exponent, W &writer) {
   // If we frequently had to deal with long strings of digits,
   // we could extend our code by using a 128-bit integer instead
   // of a 64-bit integer. However, this is uncommon in practice.
@@ -482,7 +473,7 @@ simdjson_really_inline error_code write_float(const uint8_t *const src, bool neg
     // it, it would force it to be stored in memory, preventing the compiler from picking it apart
     // and putting into registers. i.e. if we pass it as reference, it gets slow.
     // This is what forces the skip_double, as well.
-    error_code error = slow_float_parsing(src, writer);
+    error_code error = slow_float_parsing(src, end, writer);
     writer.skip_double();
     return error;
   }
@@ -507,7 +498,7 @@ simdjson_really_inline error_code write_float(const uint8_t *const src, bool neg
   double d;
   if (!compute_float_64(exponent, i, negative, d)) {
     // we are almost never going to get here.
-    if (!parse_float_fallback(src, &d)) { return INVALID_NUMBER(src); }
+    if (!parse_float_fallback(src, end, &d)) { return INVALID_NUMBER(src); }
   }
   WRITE_DOUBLE(d, src, writer);
   return SUCCESS;
@@ -517,20 +508,15 @@ simdjson_really_inline error_code write_float(const uint8_t *const src, bool neg
 #ifdef SIMDJSON_SKIPNUMBERPARSING
 
 template<typename W>
-simdjson_really_inline error_code parse_number(const uint8_t *const, W &writer) {
+simdjson_really_inline error_code parse_number(const uint8_t * const, W &writer, const uint8_t * const) {
   writer.append_s64(0);        // always write zero
   return SUCCESS;              // always succeeds
 }
 
-simdjson_unused simdjson_really_inline simdjson_result<uint64_t> parse_unsigned(const uint8_t * const src) noexcept { return 0; }
-simdjson_unused simdjson_really_inline simdjson_result<int64_t> parse_integer(const uint8_t * const src) noexcept { return 0; }
-simdjson_unused simdjson_really_inline simdjson_result<double> parse_double(const uint8_t * const src) noexcept { return 0; }
-simdjson_unused simdjson_really_inline simdjson_result<uint64_t> parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; }
-simdjson_unused simdjson_really_inline simdjson_result<int64_t> parse_integer_in_string(const uint8_t * const src) noexcept { return 0; }
-simdjson_unused simdjson_really_inline simdjson_result<double> parse_double_in_string(const uint8_t * const src) noexcept { return 0; }
-simdjson_unused simdjson_really_inline bool is_negative(const uint8_t * src) noexcept  { return false; }
-simdjson_unused simdjson_really_inline simdjson_result<bool> is_integer(const uint8_t * src) noexcept  { return false; }
-simdjson_unused simdjson_really_inline simdjson_result<ondemand::number_type> get_number_type(const uint8_t * src) noexcept { return ondemand::number_type::signed_integer; }
+simdjson_really_inline simdjson_result<uint64_t> parse_unsigned(const uint8_t * const src, const uint8_t * const end = nullptr) noexcept { return 0; }
+simdjson_really_inline simdjson_result<int64_t> parse_integer(const uint8_t * const src, const uint8_t * const end = nullptr) noexcept { return 0; }
+simdjson_really_inline simdjson_result<double> parse_double(const uint8_t * const src, const uint8_t * const end = nullptr) noexcept { return 0; }
+
 #else
 
 // parse the number at src
@@ -543,8 +529,8 @@ simdjson_unused simdjson_really_inline simdjson_result<ondemand::number_type> ge
 //
 // Our objective is accurate parsing (ULP of 0) at high speed.
 template<typename W>
-simdjson_really_inline error_code parse_number(const uint8_t *const src, W &writer) {
-
+simdjson_really_inline error_code parse_number(const uint8_t * const src, W &writer, const uint8_t * const end) {
+  SIMDJSON_ASSUME(end != nullptr);
   //
   // Check for minus sign
   //
@@ -555,9 +541,9 @@ simdjson_really_inline error_code parse_number(const uint8_t *const src, W &writ
   // Parse the integer part.
   //
   // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare
-  const uint8_t *const start_digits = p;
+  const uint8_t * const start_digits = p;
   uint64_t i = 0;
-  while (parse_digit(*p, i)) { p++; }
+  while (parse_digit(p, end, i)) { ++p; }
 
   // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error.
   // Optimization note: size_t is expected to be unsigned.
@@ -569,20 +555,20 @@ simdjson_really_inline error_code parse_number(const uint8_t *const src, W &writ
   //
   int64_t exponent = 0;
   bool is_float = false;
-  if ('.' == *p) {
+  if (!at_end(p, end) && '.' == *p) {
     is_float = true;
     ++p;
-    SIMDJSON_TRY( parse_decimal(src, p, i, exponent) );
+    SIMDJSON_TRY( parse_decimal(src, end, p, i, exponent) );
     digit_count = int(p - start_digits); // used later to guard against overflows
   }
-  if (('e' == *p) || ('E' == *p)) {
+  if (!at_end(p, end) && ('e' == *p || 'E' == *p)) {
     is_float = true;
     ++p;
-    SIMDJSON_TRY( parse_exponent(src, p, exponent) );
+    SIMDJSON_TRY( parse_exponent(src, end, p, exponent) );
   }
   if (is_float) {
-    const bool dirty_end = jsoncharutils::is_not_structural_or_whitespace(*p);
-    SIMDJSON_TRY( write_float(src, negative, i, start_digits, digit_count, exponent, writer) );
+    const bool dirty_end = !at_end(p, end) && jsoncharutils::is_not_structural_or_whitespace(*p);
+    SIMDJSON_TRY( write_float(src, end, negative, i, start_digits, digit_count, exponent, writer) );
     if (dirty_end) { return INVALID_NUMBER(src); }
     return SUCCESS;
   }
@@ -597,7 +583,7 @@ simdjson_really_inline error_code parse_number(const uint8_t *const src, W &writ
       // Anything negative above INT64_MAX+1 is invalid
       if (i > uint64_t(INT64_MAX)+1) { return INVALID_NUMBER(src);  }
       WRITE_INTEGER(~i+1, src, writer);
-      if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); }
+      if (!at_end(p, end) && jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); }
       return SUCCESS;
     // Positive overflow check:
     // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the
@@ -609,7 +595,7 @@ simdjson_really_inline error_code parse_number(const uint8_t *const src, W &writ
     // - That is smaller than the smallest possible 20-digit number the user could write:
     //   10,000,000,000,000,000,000.
     // - Therefore, if the number is positive and lower than that, it's overflow.
-    // - The value we are looking at is less than or equal to INT64_MAX.
+    // - The value we are looking at is less than or equal to 9,223,372,036,854,775,808 (INT64_MAX).
     //
     }  else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); }
   }
@@ -620,7 +606,7 @@ simdjson_really_inline error_code parse_number(const uint8_t *const src, W &writ
   } else {
     WRITE_INTEGER(negative ? (~i+1) : i, src, writer);
   }
-  if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); }
+  if (!at_end(p, end) && jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); }
   return SUCCESS;
 }
 
@@ -693,15 +679,15 @@ const uint8_t integer_string_finisher[256] = {
     NUMBER_ERROR};
 
 // Parse any number from 0 to 18,446,744,073,709,551,615
-simdjson_unused simdjson_really_inline simdjson_result<uint64_t> parse_unsigned(const uint8_t * const src) noexcept {
+simdjson_really_inline simdjson_result<uint64_t> parse_unsigned(const uint8_t * const src, const uint8_t * const end = nullptr) noexcept {
   const uint8_t *p = src;
   //
   // Parse the integer part.
   //
   // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare
-  const uint8_t *const start_digits = p;
+  const uint8_t * const start_digits = p;
   uint64_t i = 0;
-  while (parse_digit(*p, i)) { p++; }
+  while (parse_digit(p, end, i)) { ++p; }
 
   // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error.
   // Optimization note: size_t is expected to be unsigned.
@@ -719,7 +705,7 @@ simdjson_unused simdjson_really_inline simdjson_result<uint64_t> parse_unsigned(
   //  return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR;
   // }
   // as a single table lookup:
-  if (integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); }
+  if (!at_end(p, end) && integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); }
 
   if (digit_count == 20) {
     // Positive overflow check:
@@ -732,7 +718,7 @@ simdjson_unused simdjson_really_inline simdjson_result<uint64_t> parse_unsigned(
     // - That is smaller than the smallest possible 20-digit number the user could write:
     //   10,000,000,000,000,000,000.
     // - Therefore, if the number is positive and lower than that, it's overflow.
-    // - The value we are looking at is less than or equal to INT64_MAX.
+    // - The value we are looking at is less than or equal to 9,223,372,036,854,775,808 (INT64_MAX).
     //
     if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; }
   }
@@ -741,120 +727,21 @@ simdjson_unused simdjson_really_inline simdjson_result<uint64_t> parse_unsigned(
 }
 
 
-// Parse any number from 0 to 18,446,744,073,709,551,615
-// Never read at src_end or beyond
-simdjson_unused simdjson_really_inline simdjson_result<uint64_t> parse_unsigned(const uint8_t * const src, const uint8_t * const src_end) noexcept {
-  const uint8_t *p = src;
-  //
-  // Parse the integer part.
-  //
-  // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare
-  const uint8_t *const start_digits = p;
-  uint64_t i = 0;
-  while ((p != src_end) && parse_digit(*p, i)) { p++; }
-
-  // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error.
-  // Optimization note: size_t is expected to be unsigned.
-  size_t digit_count = size_t(p - start_digits);
-  // The longest positive 64-bit number is 20 digits.
-  // We do it this way so we don't trigger this branch unless we must.
-  // Optimization note: the compiler can probably merge
-  // ((digit_count == 0) || (digit_count > 20))
-  // into a single  branch since digit_count is unsigned.
-  if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; }
-  // Here digit_count > 0.
-  if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; }
-  // We can do the following...
-  // if (!jsoncharutils::is_structural_or_whitespace(*p)) {
-  //  return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR;
-  // }
-  // as a single table lookup:
-  if ((p != src_end) && integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); }
-
-  if (digit_count == 20) {
-    // Positive overflow check:
-    // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the
-    //   biggest uint64_t.
-    // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX.
-    //   If we got here, it's a 20 digit number starting with the digit "1".
-    // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller
-    //   than 1,553,255,926,290,448,384.
-    // - That is smaller than the smallest possible 20-digit number the user could write:
-    //   10,000,000,000,000,000,000.
-    // - Therefore, if the number is positive and lower than that, it's overflow.
-    // - The value we are looking at is less than or equal to INT64_MAX.
-    //
-    if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; }
-  }
-
-  return i;
-}
-
-// Parse any number from 0 to 18,446,744,073,709,551,615
-simdjson_unused simdjson_really_inline simdjson_result<uint64_t> parse_unsigned_in_string(const uint8_t * const src) noexcept {
-  const uint8_t *p = src + 1;
-  //
-  // Parse the integer part.
-  //
-  // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare
-  const uint8_t *const start_digits = p;
-  uint64_t i = 0;
-  while (parse_digit(*p, i)) { p++; }
-
-  // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error.
-  // Optimization note: size_t is expected to be unsigned.
-  size_t digit_count = size_t(p - start_digits);
-  // The longest positive 64-bit number is 20 digits.
-  // We do it this way so we don't trigger this branch unless we must.
-  // Optimization note: the compiler can probably merge
-  // ((digit_count == 0) || (digit_count > 20))
-  // into a single  branch since digit_count is unsigned.
-  if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; }
-  // Here digit_count > 0.
-  if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; }
-  // We can do the following...
-  // if (!jsoncharutils::is_structural_or_whitespace(*p)) {
-  //  return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR;
-  // }
-  // as a single table lookup:
-  if (*p != '"') { return NUMBER_ERROR; }
-
-  if (digit_count == 20) {
-    // Positive overflow check:
-    // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the
-    //   biggest uint64_t.
-    // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX.
-    //   If we got here, it's a 20 digit number starting with the digit "1".
-    // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller
-    //   than 1,553,255,926,290,448,384.
-    // - That is smaller than the smallest possible 20-digit number the user could write:
-    //   10,000,000,000,000,000,000.
-    // - Therefore, if the number is positive and lower than that, it's overflow.
-    // - The value we are looking at is less than or equal to INT64_MAX.
-    //
-    // Note: we use src[1] and not src[0] because src[0] is the quote character in this
-    // instance.
-    if (src[1] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; }
-  }
-
-  return i;
-}
-
 // Parse any number from  -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807
-simdjson_unused simdjson_really_inline simdjson_result<int64_t> parse_integer(const uint8_t *src) noexcept {
+simdjson_really_inline simdjson_result<int64_t> parse_integer(const uint8_t * src, const uint8_t * const end = nullptr) noexcept {
   //
   // Check for minus sign
   //
-  bool negative = (*src == '-');
+  bool negative = (!at_end(src, end) && *src == '-');
   const uint8_t *p = src + negative;
 
   //
   // Parse the integer part.
   //
   // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare
-  const uint8_t *const start_digits = p;
+  const uint8_t * const start_digits = p;
   uint64_t i = 0;
-  while (parse_digit(*p, i)) { p++; }
+  while (parse_digit(p, end, i)) { ++p; }
 
   // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error.
   // Optimization note: size_t is expected to be unsigned.
@@ -874,7 +761,7 @@ simdjson_unused simdjson_really_inline simdjson_result<int64_t> parse_integer(co
   //  return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR;
   // }
   // as a single table lookup:
-  if(integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); }
+  if (!at_end(p, end) && integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); }
   // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX.
   // Performance note: This check is only needed when digit_count == longest_digit_count but it is
   // so cheap that we might as well always make it.
@@ -882,97 +769,11 @@ simdjson_unused simdjson_really_inline simdjson_result<int64_t> parse_integer(co
   return negative ? (~i+1) : i;
 }
 
-// Parse any number from  -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807
-// Never read at src_end or beyond
-simdjson_unused simdjson_really_inline simdjson_result<int64_t> parse_integer(const uint8_t * const src, const uint8_t * const src_end) noexcept {
+simdjson_really_inline simdjson_result<double> parse_double(const uint8_t * src, const uint8_t * const end = nullptr) noexcept {
   //
   // Check for minus sign
   //
-  if(src == src_end) { return NUMBER_ERROR; }
-  bool negative = (*src == '-');
-  const uint8_t *p = src + negative;
-
-  //
-  // Parse the integer part.
-  //
-  // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare
-  const uint8_t *const start_digits = p;
-  uint64_t i = 0;
-  while ((p != src_end) && parse_digit(*p, i)) { p++; }
-
-  // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error.
-  // Optimization note: size_t is expected to be unsigned.
-  size_t digit_count = size_t(p - start_digits);
-  // We go from
-  // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807
-  // so we can never represent numbers that have more than 19 digits.
-  size_t longest_digit_count = 19;
-  // Optimization note: the compiler can probably merge
-  // ((digit_count == 0) || (digit_count > longest_digit_count))
-  // into a single  branch since digit_count is unsigned.
-  if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; }
-  // Here digit_count > 0.
-  if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; }
-  // We can do the following...
-  // if (!jsoncharutils::is_structural_or_whitespace(*p)) {
-  //  return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR;
-  // }
-  // as a single table lookup:
-  if((p != src_end) && integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); }
-  // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX.
-  // Performance note: This check is only needed when digit_count == longest_digit_count but it is
-  // so cheap that we might as well always make it.
-  if(i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; }
-  return negative ? (~i+1) : i;
-}
-
-// Parse any number from  -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807
-simdjson_unused simdjson_really_inline simdjson_result<int64_t> parse_integer_in_string(const uint8_t *src) noexcept {
-  //
-  // Check for minus sign
-  //
-  bool negative = (*(src + 1) == '-');
-  const uint8_t *p = src + negative + 1;
-
-  //
-  // Parse the integer part.
-  //
-  // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare
-  const uint8_t *const start_digits = p;
-  uint64_t i = 0;
-  while (parse_digit(*p, i)) { p++; }
-
-  // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error.
-  // Optimization note: size_t is expected to be unsigned.
-  size_t digit_count = size_t(p - start_digits);
-  // We go from
-  // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807
-  // so we can never represent numbers that have more than 19 digits.
-  size_t longest_digit_count = 19;
-  // Optimization note: the compiler can probably merge
-  // ((digit_count == 0) || (digit_count > longest_digit_count))
-  // into a single  branch since digit_count is unsigned.
-  if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; }
-  // Here digit_count > 0.
-  if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; }
-  // We can do the following...
-  // if (!jsoncharutils::is_structural_or_whitespace(*p)) {
-  //  return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR;
-  // }
-  // as a single table lookup:
-  if(*p != '"') { return NUMBER_ERROR; }
-  // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX.
-  // Performance note: This check is only needed when digit_count == longest_digit_count but it is
-  // so cheap that we might as well always make it.
-  if(i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; }
-  return negative ? (~i+1) : i;
-}
-
-simdjson_unused simdjson_really_inline simdjson_result<double> parse_double(const uint8_t * src) noexcept {
-  //
-  // Check for minus sign
-  //
-  bool negative = (*src == '-');
+  bool negative = (!at_end(src, end) && *src == '-');
   src += negative;
 
   //
@@ -980,9 +781,9 @@ simdjson_unused simdjson_really_inline simdjson_result<double> parse_double(cons
   //
   uint64_t i = 0;
   const uint8_t *p = src;
-  p += parse_digit(*p, i);
+  p += parse_digit(p, end, i);
   bool leading_zero = (i == 0);
-  while (parse_digit(*p, i)) { p++; }
+  while (parse_digit(p, end, i)) { ++p; }
   // no integer digits, or 0123 (zero must be solo)
   if ( p == src ) { return INCORRECT_TYPE; }
   if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; }
@@ -990,14 +791,20 @@ simdjson_unused simdjson_really_inline simdjson_result<double> parse_double(cons
   //
   // Parse the decimal part.
   //
+  // performance todo: we could use SWAR here to quickly processing
+  // digits in blocs of, say, eight digits. Doing so requires that
+  // we always have access to some estimation of the end of the buffer.
+  // Currently, we only have that for root numbers in the On Demand API,
+  // so it is not terribly useful for performance purposes.
+  //
   int64_t exponent = 0;
   bool overflow;
-  if (simdjson_likely(*p == '.')) {
-    p++;
+  if (simdjson_likely(!at_end(p, end) && '.' == *p)) {
+    ++p;
     const uint8_t *start_decimal_digits = p;
-    if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits
-    p++;
-    while (parse_digit(*p, i)) { p++; }
+    if (!parse_digit(p, end, i)) { return NUMBER_ERROR; } // no decimal digits
+    ++p;
+    while (parse_digit(p, end, i)) { ++p; }
     exponent = -(p - start_decimal_digits);
 
     // Overflow check. More than 19 digits (minus the decimal) may be overflow.
@@ -1005,7 +812,7 @@ simdjson_unused simdjson_really_inline simdjson_result<double> parse_double(cons
     if (simdjson_unlikely(overflow && leading_zero)) {
       // Skip leading 0.00000 and see if it still overflows
       const uint8_t *start_digits = src + 2;
-      while (*start_digits == '0') { start_digits++; }
+      while (!at_end(start_digits, end) && *start_digits == '0') { ++start_digits; }
       overflow = start_digits-src > 19;
     }
   } else {
@@ -1015,21 +822,21 @@ simdjson_unused simdjson_really_inline simdjson_result<double> parse_double(cons
   //
   // Parse the exponent
   //
-  if (*p == 'e' || *p == 'E') {
-    p++;
-    bool exp_neg = *p == '-';
-    p += exp_neg || *p == '+';
+  if (!at_end(p, end) && ('e' == *p || 'E' == *p)) {
+    ++p;
+    bool exp_neg = (!at_end(p, end) && '-' == *p);
+    p += exp_neg || (!at_end(p, end) && '+' == *p);
 
     uint64_t exp = 0;
     const uint8_t *start_exp_digits = p;
-    while (parse_digit(*p, exp)) { p++; }
+    while (parse_digit(p, end, exp)) { ++p; }
     // no exp digits, or 20+ exp digits
     if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; }
 
     exponent += exp_neg ? 0-exp : exp;
   }
 
-  if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; }
+  if (!at_end(p, end) && jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; }
 
   overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power;
 
@@ -1040,204 +847,12 @@ simdjson_unused simdjson_really_inline simdjson_result<double> parse_double(cons
   if (simdjson_likely(!overflow)) {
     if (compute_float_64(exponent, i, negative, d)) { return d; }
   }
-  if (!parse_float_fallback(src-negative, &d)) {
+  if (!parse_float_fallback(src-negative, end, &d)) {
     return NUMBER_ERROR;
   }
   return d;
 }
 
-simdjson_unused simdjson_really_inline bool is_negative(const uint8_t * src) noexcept {
-  return (*src == '-');
-}
-
-simdjson_unused simdjson_really_inline simdjson_result<bool> is_integer(const uint8_t * src) noexcept {
-  bool negative = (*src == '-');
-  src += negative;
-  const uint8_t *p = src;
-  while(static_cast<uint8_t>(*p - '0') <= 9) { p++; }
-  if ( p == src ) { return NUMBER_ERROR; }
-  if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; }
-  return false;
-}
-
-simdjson_unused simdjson_really_inline simdjson_result<ondemand::number_type> get_number_type(const uint8_t * src) noexcept {
-  bool negative = (*src == '-');
-  src += negative;
-  const uint8_t *p = src;
-  while(static_cast<uint8_t>(*p - '0') <= 9) { p++; }
-  if ( p == src ) { return NUMBER_ERROR; }
-  if (jsoncharutils::is_structural_or_whitespace(*p)) {
-    int digit_count = int(p - src);
-    if(digit_count >= 19) {
-      const uint8_t * smaller_big_integer = reinterpret_cast<const uint8_t *>("9223372036854775808");
-      if((digit_count >= 20) || (memcmp(src, smaller_big_integer, 19) >= 0)) {
-        return ondemand::number_type::unsigned_integer;
-      }
-    }
-    return ondemand::number_type::signed_integer;
-  }
-  return ondemand::number_type::floating_point_number;
-}
-
-// Never read at src_end or beyond
-simdjson_unused simdjson_really_inline simdjson_result<double> parse_double(const uint8_t * src, const uint8_t * const src_end) noexcept {
-  if(src == src_end) { return NUMBER_ERROR; }
-  //
-  // Check for minus sign
-  //
-  bool negative = (*src == '-');
-  src += negative;
-
-  //
-  // Parse the integer part.
-  //
-  uint64_t i = 0;
-  const uint8_t *p = src;
-  if(p == src_end) { return NUMBER_ERROR; }
-  p += parse_digit(*p, i);
-  bool leading_zero = (i == 0);
-  while ((p != src_end) && parse_digit(*p, i)) { p++; }
-  // no integer digits, or 0123 (zero must be solo)
-  if ( p == src ) { return INCORRECT_TYPE; }
-  if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; }
-
-  //
-  // Parse the decimal part.
-  //
-  int64_t exponent = 0;
-  bool overflow;
-  if (simdjson_likely((p != src_end) && (*p == '.'))) {
-    p++;
-    const uint8_t *start_decimal_digits = p;
-    if ((p == src_end) || !parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits
-    p++;
-    while ((p != src_end) && parse_digit(*p, i)) { p++; }
-    exponent = -(p - start_decimal_digits);
-
-    // Overflow check. More than 19 digits (minus the decimal) may be overflow.
-    overflow = p-src-1 > 19;
-    if (simdjson_unlikely(overflow && leading_zero)) {
-      // Skip leading 0.00000 and see if it still overflows
-      const uint8_t *start_digits = src + 2;
-      while (*start_digits == '0') { start_digits++; }
-      overflow = start_digits-src > 19;
-    }
-  } else {
-    overflow = p-src > 19;
-  }
-
-  //
-  // Parse the exponent
-  //
-  if ((p != src_end) && (*p == 'e' || *p == 'E')) {
-    p++;
-    if(p == src_end) { return NUMBER_ERROR; }
-    bool exp_neg = *p == '-';
-    p += exp_neg || *p == '+';
-
-    uint64_t exp = 0;
-    const uint8_t *start_exp_digits = p;
-    while ((p != src_end) && parse_digit(*p, exp)) { p++; }
-    // no exp digits, or 20+ exp digits
-    if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; }
-
-    exponent += exp_neg ? 0-exp : exp;
-  }
-
-  if ((p != src_end) && jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; }
-
-  overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power;
-
-  //
-  // Assemble (or slow-parse) the float
-  //
-  double d;
-  if (simdjson_likely(!overflow)) {
-    if (compute_float_64(exponent, i, negative, d)) { return d; }
-  }
-  if (!parse_float_fallback(src-negative, src_end, &d)) {
-    return NUMBER_ERROR;
-  }
-  return d;
-}
-
-simdjson_unused simdjson_really_inline simdjson_result<double> parse_double_in_string(const uint8_t * src) noexcept {
-  //
-  // Check for minus sign
-  //
-  bool negative = (*(src + 1) == '-');
-  src += negative + 1;
-
-  //
-  // Parse the integer part.
-  //
-  uint64_t i = 0;
-  const uint8_t *p = src;
-  p += parse_digit(*p, i);
-  bool leading_zero = (i == 0);
-  while (parse_digit(*p, i)) { p++; }
-  // no integer digits, or 0123 (zero must be solo)
-  if ( p == src ) { return INCORRECT_TYPE; }
-  if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; }
-
-  //
-  // Parse the decimal part.
-  //
-  int64_t exponent = 0;
-  bool overflow;
-  if (simdjson_likely(*p == '.')) {
-    p++;
-    const uint8_t *start_decimal_digits = p;
-    if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits
-    p++;
-    while (parse_digit(*p, i)) { p++; }
-    exponent = -(p - start_decimal_digits);
-
-    // Overflow check. More than 19 digits (minus the decimal) may be overflow.
-    overflow = p-src-1 > 19;
-    if (simdjson_unlikely(overflow && leading_zero)) {
-      // Skip leading 0.00000 and see if it still overflows
-      const uint8_t *start_digits = src + 2;
-      while (*start_digits == '0') { start_digits++; }
-      overflow = start_digits-src > 19;
-    }
-  } else {
-    overflow = p-src > 19;
-  }
-
-  //
-  // Parse the exponent
-  //
-  if (*p == 'e' || *p == 'E') {
-    p++;
-    bool exp_neg = *p == '-';
-    p += exp_neg || *p == '+';
-
-    uint64_t exp = 0;
-    const uint8_t *start_exp_digits = p;
-    while (parse_digit(*p, exp)) { p++; }
-    // no exp digits, or 20+ exp digits
-    if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; }
-
-    exponent += exp_neg ? 0-exp : exp;
-  }
-
-  if (*p != '"') { return NUMBER_ERROR; }
-
-  overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power;
-
-  //
-  // Assemble (or slow-parse) the float
-  //
-  double d;
-  if (simdjson_likely(!overflow)) {
-    if (compute_float_64(exponent, i, negative, d)) { return d; }
-  }
-  if (!parse_float_fallback(src-negative, &d)) {
-    return NUMBER_ERROR;
-  }
-  return d;
-}
 } //namespace {}
 #endif // SIMDJSON_SKIPNUMBERPARSING
 

include/simdjson/generic/ondemand/array-inl.h

@@ -89,7 +89,6 @@ simdjson_really_inline simdjson_result<std::string_view> array::raw_json() noexc
   return std::string_view(reinterpret_cast<const char*>(starting_point), size_t(final_point - starting_point));
 }
 
-SIMDJSON_DISABLE_STRICT_OVERFLOW_WARNING
 simdjson_really_inline simdjson_result<size_t> array::count_elements() & noexcept {
   size_t count{0};
   // Important: we do not consume any of the values.
@@ -102,17 +101,6 @@ simdjson_really_inline simdjson_result<size_t> array::count_elements() & noexcep
   return count;
 }
 
-simdjson_really_inline simdjson_result<bool> array::is_empty() & noexcept {
-  bool is_not_empty;
-  auto error = iter.reset_array().get(is_not_empty);
-  if(error) { return error; }
-  return !is_not_empty;
-}
-
-inline simdjson_result<bool> array::reset() & noexcept {
-  return iter.reset_array();
-}
-
 inline simdjson_result<value> array::at_pointer(std::string_view json_pointer) noexcept {
   if (json_pointer[0] != '/') { return INVALID_JSON_POINTER; }
   json_pointer = json_pointer.substr(1);
@@ -191,10 +179,6 @@ simdjson_really_inline  simdjson_result<size_t> simdjson_result<SIMDJSON_IMPLEME
   if (error()) { return error(); }
   return first.count_elements();
 }
-simdjson_really_inline  simdjson_result<bool> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array>::is_empty() & noexcept {
-  if (error()) { return error(); }
-  return first.is_empty();
-}
 simdjson_really_inline  simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array>::at(size_t index) noexcept {
   if (error()) { return error(); }
   return first.at(index);

include/simdjson/generic/ondemand/array.h

@@ -41,32 +41,9 @@ public:
    * beginning as if it had never been accessed. If the JSON is malformed (e.g.,
    * there is a missing comma), then an error is returned and it is no longer
    * safe to continue.
-   *
-   * To check that an array is empty, it is more performant to use
-   * the is_empty() method.
    */
   simdjson_really_inline simdjson_result<size_t> count_elements() & noexcept;
-  /**
-   * This method scans the beginning of the array and checks whether the
-   * array is empty.
-   * The runtime complexity is constant time. After
-   * calling this function, if successful, the array is 'rewinded' at its
-   * beginning as if it had never been accessed. If the JSON is malformed (e.g.,
-   * there is a missing comma), then an error is returned and it is no longer
-   * safe to continue.
-   */
-  simdjson_really_inline simdjson_result<bool> is_empty() & noexcept;
-  /**
-   * Reset the iterator so that we are pointing back at the
-   * beginning of the array. You should still consume values only once even if you
-   * can iterate through the array more than once. If you unescape a string
-   * within the array more than once, you have unsafe code. Note that rewinding
-   * an array means that you may need to reparse it anew: it is not a free
-   * operation.
-   *
-   * @returns true if the array contains some elements (not empty)
-   */
-  inline simdjson_result<bool> reset() & noexcept;
+
   /**
    * Get the value associated with the given JSON pointer.  We use the RFC 6901
    * https://tools.ietf.org/html/rfc6901 standard, interpreting the current node
@@ -102,14 +79,6 @@ public:
    */
   simdjson_really_inline simdjson_result<std::string_view> raw_json() noexcept;
 
-  /**
-   * Get the value at the given index. This function has linear-time complexity.
-   * This function should only be called once as the array iterator is not reset between each call.
-   *
-   * @return The value at the given index, or:
-   *         - INDEX_OUT_OF_BOUNDS if the array index is larger than an array length
-   */
-  simdjson_really_inline simdjson_result<value> at(size_t index) noexcept;
 protected:
   /**
    * Go to the end of the array, no matter where you are right now.
@@ -152,6 +121,15 @@ protected:
    */
   simdjson_really_inline array(const value_iterator &iter) noexcept;
 
+  /**
+   * Get the value at the given index. This function has linear-time complexity.
+   * This function should only be called once as the array iterator is not reset between each call.
+   *
+   * @return The value at the given index, or:
+   *         - INDEX_OUT_OF_BOUNDS if the array index is larger than an array length
+   */
+  simdjson_really_inline simdjson_result<value> at(size_t index) noexcept;
+
   /**
    * Iterator marking current position.
    *
@@ -181,9 +159,7 @@ public:
 
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> begin() noexcept;
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> end() noexcept;
-  inline simdjson_result<size_t> count_elements() & noexcept;
-  inline simdjson_result<bool> is_empty() & noexcept;
-  inline simdjson_result<bool> reset() & noexcept;
+  simdjson_really_inline simdjson_result<size_t> count_elements() & noexcept;
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> at(size_t index) noexcept;
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> at_pointer(std::string_view json_pointer) noexcept;
 };

include/simdjson/generic/ondemand/document-inl.h

@@ -20,13 +20,6 @@ inline std::string document::to_debug_string() noexcept {
   return iter.to_string();
 }
 
-inline simdjson_result<const char *> document::current_location() noexcept {
-  return iter.current_location();
-}
-
-inline bool document::is_alive() noexcept {
-  return iter.is_alive();
-}
 simdjson_really_inline value_iterator document::resume_value_iterator() noexcept {
   return value_iterator(&iter, 1, iter.root_position());
 }
@@ -40,19 +33,24 @@ simdjson_really_inline simdjson_result<object> document::start_or_resume_object(
     return object::resume(resume_value_iterator());
   }
 }
-simdjson_really_inline simdjson_result<value> document::get_value() noexcept {
+simdjson_really_inline simdjson_result<value> document::get_value_unsafe() noexcept {
   // Make sure we start any arrays or objects before returning, so that start_root_<object/array>()
   // gets called.
-  iter.assert_at_document_depth();
   switch (*iter.peek()) {
-    case '[':
-    case '{':
-      return value(get_root_value_iterator());
+    case '[': {
+      array result;
+      SIMDJSON_TRY( get_array().get(result) );
+      return value(result.iter);
+    }
+    case '{': {
+      object result;
+      SIMDJSON_TRY( get_object().get(result) );
+      return value(result.iter);
+    }
     default:
-      // Unfortunately, scalar documents are a special case in simdjson and they cannot
-      // be safely converted to value instances.
-      return SCALAR_DOCUMENT_AS_VALUE;
-      // return value(get_root_value_iterator());
+      // TODO it is still wrong to convert this to a value! get_root_bool / etc. will not be
+      // called if you do this.
+      return value(get_root_value_iterator());
   }
 }
 simdjson_really_inline simdjson_result<array> document::get_array() & noexcept {
@@ -66,21 +64,12 @@ simdjson_really_inline simdjson_result<object> document::get_object() & noexcept
 simdjson_really_inline simdjson_result<uint64_t> document::get_uint64() noexcept {
   return get_root_value_iterator().get_root_uint64();
 }
-simdjson_really_inline simdjson_result<uint64_t> document::get_uint64_in_string() noexcept {
-  return get_root_value_iterator().get_root_uint64_in_string();
-}
 simdjson_really_inline simdjson_result<int64_t> document::get_int64() noexcept {
   return get_root_value_iterator().get_root_int64();
 }
-simdjson_really_inline simdjson_result<int64_t> document::get_int64_in_string() noexcept {
-  return get_root_value_iterator().get_root_int64_in_string();
-}
 simdjson_really_inline simdjson_result<double> document::get_double() noexcept {
   return get_root_value_iterator().get_root_double();
 }
-simdjson_really_inline simdjson_result<double> document::get_double_in_string() noexcept {
-  return get_root_value_iterator().get_root_double_in_string();
-}
 simdjson_really_inline simdjson_result<std::string_view> document::get_string() noexcept {
   return get_root_value_iterator().get_root_string();
 }
@@ -102,7 +91,6 @@ template<> simdjson_really_inline simdjson_result<double> document::get() & noex
 template<> simdjson_really_inline simdjson_result<uint64_t> document::get() & noexcept { return get_uint64(); }
 template<> simdjson_really_inline simdjson_result<int64_t> document::get() & noexcept { return get_int64(); }
 template<> simdjson_really_inline simdjson_result<bool> document::get() & noexcept { return get_bool(); }
-template<> simdjson_really_inline simdjson_result<value> document::get() & noexcept { return get_value(); }
 
 template<> simdjson_really_inline simdjson_result<raw_json_string> document::get() && noexcept { return get_raw_json_string(); }
 template<> simdjson_really_inline simdjson_result<std::string_view> document::get() && noexcept { return get_string(); }
@@ -110,7 +98,6 @@ template<> simdjson_really_inline simdjson_result<double> document::get() && noe
 template<> simdjson_really_inline simdjson_result<uint64_t> document::get() && noexcept { return std::forward<document>(*this).get_uint64(); }
 template<> simdjson_really_inline simdjson_result<int64_t> document::get() && noexcept { return std::forward<document>(*this).get_int64(); }
 template<> simdjson_really_inline simdjson_result<bool> document::get() && noexcept { return std::forward<document>(*this).get_bool(); }
-template<> simdjson_really_inline simdjson_result<value> document::get() && noexcept { return get_value(); }
 
 template<typename T> simdjson_really_inline error_code document::get(T &out) & noexcept {
   return get<T>().get(out);
@@ -128,33 +115,14 @@ simdjson_really_inline document::operator double() noexcept(false) { return get_
 simdjson_really_inline document::operator std::string_view() noexcept(false) { return get_string(); }
 simdjson_really_inline document::operator raw_json_string() noexcept(false) { return get_raw_json_string(); }
 simdjson_really_inline document::operator bool() noexcept(false) { return get_bool(); }
-simdjson_really_inline document::operator value() noexcept(false) { return get_value(); }
-
 #endif
 simdjson_really_inline simdjson_result<size_t> document::count_elements() & noexcept {
   auto a = get_array();
   simdjson_result<size_t> answer = a.count_elements();
   /* If there was an array, we are now left pointing at its first element. */
-  if(answer.error() == SUCCESS) {
-    iter._depth = 1 ; /* undoing the increment so we go back at the doc depth.*/
-    iter.assert_at_document_depth();
-  }
+  if(answer.error() == SUCCESS) { iter._depth -= 1 ; /* undoing the increment so we go back at the doc depth.*/ }
   return answer;
 }
-simdjson_really_inline simdjson_result<size_t> document::count_fields() & noexcept {
-  auto a = get_object();
-  simdjson_result<size_t> answer = a.count_fields();
-  /* If there was an array, we are now left pointing at its first element. */
-  if(answer.error() == SUCCESS) {
-    iter._depth = 1 ; /* undoing the increment so we go back at the doc depth.*/
-    iter.assert_at_document_depth();
-  }
-  return answer;
-}
-simdjson_really_inline simdjson_result<value> document::at(size_t index) & noexcept {
-  auto a = get_array();
-  return a.at(index);
-}
 simdjson_really_inline simdjson_result<array_iterator> document::begin() & noexcept {
   return get_array().begin();
 }
@@ -203,30 +171,6 @@ simdjson_really_inline simdjson_result<json_type> document::type() noexcept {
   return get_root_value_iterator().type();
 }
 
-simdjson_really_inline simdjson_result<bool> document::is_scalar() noexcept {
-  json_type this_type;
-  auto error = type().get(this_type);
-  if(error) { return error; }
-  return ! ((this_type == json_type::array) || (this_type == json_type::object));
-}
-
-simdjson_really_inline bool document::is_negative() noexcept {
-  return get_root_value_iterator().is_root_negative();
-}
-
-simdjson_really_inline simdjson_result<bool> document::is_integer() noexcept {
-  return get_root_value_iterator().is_root_integer();
-}
-
-simdjson_really_inline simdjson_result<number_type> document::get_number_type() noexcept {
-  return get_root_value_iterator().get_root_number_type();
-}
-
-simdjson_really_inline simdjson_result<number> document::get_number() noexcept {
-  return get_root_value_iterator().get_root_number();
-}
-
-
 simdjson_really_inline simdjson_result<std::string_view> document::raw_json_token() noexcept {
   auto _iter = get_root_value_iterator();
   return std::string_view(reinterpret_cast<const char*>(_iter.peek_start()), _iter.peek_start_length());
@@ -235,7 +179,7 @@ simdjson_really_inline simdjson_result<std::string_view> document::raw_json_toke
 simdjson_really_inline simdjson_result<value> document::at_pointer(std::string_view json_pointer) noexcept {
   rewind(); // Rewind the document each time at_pointer is called
   if (json_pointer.empty()) {
-    return this->get_value();
+    return this->get_value_unsafe();
   }
   json_type t;
   SIMDJSON_TRY(type().get(t));
@@ -276,14 +220,6 @@ simdjson_really_inline simdjson_result<size_t> simdjson_result<SIMDJSON_IMPLEMEN
   if (error()) { return error(); }
   return first.count_elements();
 }
-simdjson_really_inline simdjson_result<size_t> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::count_fields() & noexcept {
-  if (error()) { return error(); }
-  return first.count_fields();
-}
-simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::at(size_t index) & noexcept {
-  if (error()) { return error(); }
-  return first.at(index);
-}
 simdjson_really_inline error_code simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::rewind() noexcept {
   if (error()) { return error(); }
   first.rewind();
@@ -352,10 +288,6 @@ simdjson_really_inline simdjson_result<bool> simdjson_result<SIMDJSON_IMPLEMENTA
   if (error()) { return error(); }
   return first.get_bool();
 }
-simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::get_value() noexcept {
-  if (error()) { return error(); }
-  return first.get_value();
-}
 simdjson_really_inline bool simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::is_null() noexcept {
   if (error()) { return error(); }
   return first.is_null();
@@ -399,33 +331,6 @@ simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::json_t
   return first.type();
 }
 
-simdjson_really_inline simdjson_result<bool> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::is_scalar() noexcept {
-  if (error()) { return error(); }
-  return first.is_scalar();
-}
-
-
-simdjson_really_inline bool simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::is_negative() noexcept {
-  if (error()) { return error(); }
-  return first.is_negative();
-}
-
-simdjson_really_inline simdjson_result<bool> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::is_integer() noexcept {
-  if (error()) { return error(); }
-  return first.is_integer();
-}
-
-simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number_type> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::get_number_type() noexcept {
-  if (error()) { return error(); }
-  return first.get_number_type();
-}
-
-simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::get_number() noexcept {
-  if (error()) { return error(); }
-  return first.get_number();
-}
-
-
 #if SIMDJSON_EXCEPTIONS
 simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::operator SIMDJSON_IMPLEMENTATION::ondemand::array() & noexcept(false) {
   if (error()) { throw simdjson_error(error()); }
@@ -459,18 +364,8 @@ simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::docume
   if (error()) { throw simdjson_error(error()); }
   return first;
 }
-simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::operator SIMDJSON_IMPLEMENTATION::ondemand::value() noexcept(false) {
-  if (error()) { throw simdjson_error(error()); }
-  return first;
-}
 #endif
 
-
-simdjson_really_inline simdjson_result<const char *> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::current_location() noexcept {
-  if (error()) { return error(); }
-  return first.current_location();
-}
-
 simdjson_really_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::raw_json_token() noexcept {
   if (error()) { return error(); }
   return first.raw_json_token();
@@ -481,238 +376,4 @@ simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>
   return first.at_pointer(json_pointer);
 }
 
-
-} // namespace simdjson
-
-
-namespace simdjson {
-namespace SIMDJSON_IMPLEMENTATION {
-namespace ondemand {
-
-simdjson_really_inline document_reference::document_reference() noexcept : doc{nullptr} {}
-simdjson_really_inline document_reference::document_reference(document &d) noexcept : doc(&d) {}
-simdjson_really_inline void document_reference::rewind() noexcept { doc->rewind(); }
-simdjson_really_inline simdjson_result<array> document_reference::get_array() & noexcept { return doc->get_array(); }
-simdjson_really_inline simdjson_result<object> document_reference::get_object() & noexcept { return doc->get_object(); }
-simdjson_really_inline simdjson_result<uint64_t> document_reference::get_uint64() noexcept { return doc->get_uint64(); }
-simdjson_really_inline simdjson_result<int64_t> document_reference::get_int64() noexcept { return doc->get_int64(); }
-simdjson_really_inline simdjson_result<double> document_reference::get_double() noexcept { return doc->get_double(); }
-simdjson_really_inline simdjson_result<std::string_view> document_reference::get_string() noexcept { return doc->get_string(); }
-simdjson_really_inline simdjson_result<raw_json_string> document_reference::get_raw_json_string() noexcept { return doc->get_raw_json_string(); }
-simdjson_really_inline simdjson_result<bool> document_reference::get_bool() noexcept { return doc->get_bool(); }
-simdjson_really_inline simdjson_result<value> document_reference::get_value() noexcept { return doc->get_value(); }
-simdjson_really_inline bool document_reference::is_null() noexcept { return doc->is_null(); }
-
-#if SIMDJSON_EXCEPTIONS
-simdjson_really_inline document_reference::operator array() & noexcept(false) { return array(*doc); }
-simdjson_really_inline document_reference::operator object() & noexcept(false) { return object(*doc); }
-simdjson_really_inline document_reference::operator uint64_t() noexcept(false) { return uint64_t(*doc); }
-simdjson_really_inline document_reference::operator int64_t() noexcept(false) { return int64_t(*doc); }
-simdjson_really_inline document_reference::operator double() noexcept(false) { return double(*doc); }
-simdjson_really_inline document_reference::operator std::string_view() noexcept(false) { return std::string_view(*doc); }
-simdjson_really_inline document_reference::operator raw_json_string() noexcept(false) { return raw_json_string(*doc); }
-simdjson_really_inline document_reference::operator bool() noexcept(false) { return bool(*doc); }
-simdjson_really_inline document_reference::operator value() noexcept(false) { return value(*doc); }
-#endif
-simdjson_really_inline simdjson_result<size_t> document_reference::count_elements() & noexcept { return doc->count_elements(); }
-simdjson_really_inline simdjson_result<size_t> document_reference::count_fields() & noexcept { return doc->count_fields(); }
-simdjson_really_inline simdjson_result<value> document_reference::at(size_t index) & noexcept { return doc->at(index); }
-simdjson_really_inline simdjson_result<array_iterator> document_reference::begin() & noexcept { return doc->begin(); }
-simdjson_really_inline simdjson_result<array_iterator> document_reference::end() & noexcept { return doc->end(); }
-simdjson_really_inline simdjson_result<value> document_reference::find_field(std::string_view key) & noexcept { return doc->find_field(key); }
-simdjson_really_inline simdjson_result<value> document_reference::find_field(const char *key) & noexcept { return doc->find_field(key); }
-simdjson_really_inline simdjson_result<value> document_reference::operator[](std::string_view key) & noexcept { return (*doc)[key]; }
-simdjson_really_inline simdjson_result<value> document_reference::operator[](const char *key) & noexcept { return (*doc)[key]; }
-simdjson_really_inline simdjson_result<value> document_reference::find_field_unordered(std::string_view key) & noexcept { return doc->find_field_unordered(key); }
-simdjson_really_inline simdjson_result<value> document_reference::find_field_unordered(const char *key) & noexcept { return doc->find_field_unordered(key); }
-simdjson_really_inline simdjson_result<json_type> document_reference::type() noexcept { return doc->type(); }
-simdjson_really_inline simdjson_result<bool> document_reference::is_scalar() noexcept { return doc->is_scalar(); }
-simdjson_really_inline simdjson_result<const char *> document_reference::current_location() noexcept { return doc->current_location(); }
-simdjson_really_inline bool document_reference::is_negative() noexcept { return doc->is_negative(); }
-simdjson_really_inline simdjson_result<bool> document_reference::is_integer() noexcept { return doc->is_integer(); }
-simdjson_really_inline simdjson_result<number_type> document_reference::get_number_type() noexcept { return doc->get_number_type(); }
-simdjson_really_inline simdjson_result<number> document_reference::get_number() noexcept { return doc->get_number(); }
-simdjson_really_inline simdjson_result<std::string_view> document_reference::raw_json_token() noexcept { return doc->raw_json_token(); }
-simdjson_really_inline simdjson_result<value> document_reference::at_pointer(std::string_view json_pointer) noexcept { return doc->at_pointer(json_pointer); }
-simdjson_really_inline simdjson_result<std::string_view> document_reference::raw_json() noexcept { return doc->raw_json();}
-simdjson_really_inline document_reference::operator document&() const noexcept { return *doc; }
-
-} // namespace ondemand
-} // namespace SIMDJSON_IMPLEMENTATION
-} // namespace simdjson
-
-
-
-namespace simdjson {
-simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::simdjson_result(SIMDJSON_IMPLEMENTATION::ondemand::document_reference value, error_code error)
-  noexcept : implementation_simdjson_result_base<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>(std::forward<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>(value), error) {}
-
-
-simdjson_really_inline simdjson_result<size_t> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::count_elements() & noexcept {
-  if (error()) { return error(); }
-  return first.count_elements();
-}
-simdjson_really_inline simdjson_result<size_t> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::count_fields() & noexcept {
-  if (error()) { return error(); }
-  return first.count_fields();
-}
-simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::at(size_t index) & noexcept {
-  if (error()) { return error(); }
-  return first.at(index);
-}
-simdjson_really_inline error_code simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::rewind() noexcept {
-  if (error()) { return error(); }
-  first.rewind();
-  return SUCCESS;
-}
-simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::begin() & noexcept {
-  if (error()) { return error(); }
-  return first.begin();
-}
-simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::end() & noexcept {
-  return {};
-}
-simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::find_field_unordered(std::string_view key) & noexcept {
-  if (error()) { return error(); }
-  return first.find_field_unordered(key);
-}
-simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::find_field_unordered(const char *key) & noexcept {
-  if (error()) { return error(); }
-  return first.find_field_unordered(key);
-}
-simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::operator[](std::string_view key) & noexcept {
-  if (error()) { return error(); }
-  return first[key];
-}
-simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::operator[](const char *key) & noexcept {
-  if (error()) { return error(); }
-  return first[key];
-}
-simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::find_field(std::string_view key) & noexcept {
-  if (error()) { return error(); }
-  return first.find_field(key);
-}
-simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::find_field(const char *key) & noexcept {
-  if (error()) { return error(); }
-  return first.find_field(key);
-}
-simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::get_array() & noexcept {
-  if (error()) { return error(); }
-  return first.get_array();
-}
-simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::object> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::get_object() & noexcept {
-  if (error()) { return error(); }
-  return first.get_object();
-}
-simdjson_really_inline simdjson_result<uint64_t> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::get_uint64() noexcept {
-  if (error()) { return error(); }
-  return first.get_uint64();
-}
-simdjson_really_inline simdjson_result<int64_t> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::get_int64() noexcept {
-  if (error()) { return error(); }
-  return first.get_int64();
-}
-simdjson_really_inline simdjson_result<double> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::get_double() noexcept {
-  if (error()) { return error(); }
-  return first.get_double();
-}
-simdjson_really_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::get_string() noexcept {
-  if (error()) { return error(); }
-  return first.get_string();
-}
-simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::raw_json_string> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::get_raw_json_string() noexcept {
-  if (error()) { return error(); }
-  return first.get_raw_json_string();
-}
-simdjson_really_inline simdjson_result<bool> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::get_bool() noexcept {
-  if (error()) { return error(); }
-  return first.get_bool();
-}
-simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::get_value() noexcept {
-  if (error()) { return error(); }
-  return first.get_value();
-}
-simdjson_really_inline bool simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::is_null() noexcept {
-  if (error()) { return error(); }
-  return first.is_null();
-}
-simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::json_type> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::type() noexcept {
-  if (error()) { return error(); }
-  return first.type();
-}
-simdjson_really_inline simdjson_result<bool> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::is_scalar() noexcept {
-  if (error()) { return error(); }
-  return first.is_scalar();
-}
-simdjson_really_inline bool simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::is_negative() noexcept {
-  if (error()) { return error(); }
-  return first.is_negative();
-}
-simdjson_really_inline simdjson_result<bool> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::is_integer() noexcept {
-  if (error()) { return error(); }
-  return first.is_integer();
-}
-simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number_type> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::get_number_type() noexcept {
-  if (error()) { return error(); }
-  return first.get_number_type();
-}
-simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::get_number() noexcept {
-  if (error()) { return error(); }
-  return first.get_number();
-}
-#if SIMDJSON_EXCEPTIONS
-simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::operator SIMDJSON_IMPLEMENTATION::ondemand::array() & noexcept(false) {
-  if (error()) { throw simdjson_error(error()); }
-  return first;
-}
-simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::operator SIMDJSON_IMPLEMENTATION::ondemand::object() & noexcept(false) {
-  if (error()) { throw simdjson_error(error()); }
-  return first;
-}
-simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::operator uint64_t() noexcept(false) {
-  if (error()) { throw simdjson_error(error()); }
-  return first;
-}
-simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::operator int64_t() noexcept(false) {
-  if (error()) { throw simdjson_error(error()); }
-  return first;
-}
-simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::operator double() noexcept(false) {
-  if (error()) { throw simdjson_error(error()); }
-  return first;
-}
-simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::operator std::string_view() noexcept(false) {
-  if (error()) { throw simdjson_error(error()); }
-  return first;
-}
-simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::operator SIMDJSON_IMPLEMENTATION::ondemand::raw_json_string() noexcept(false) {
-  if (error()) { throw simdjson_error(error()); }
-  return first;
-}
-simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::operator bool() noexcept(false) {
-  if (error()) { throw simdjson_error(error()); }
-  return first;
-}
-simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::operator SIMDJSON_IMPLEMENTATION::ondemand::value() noexcept(false) {
-  if (error()) { throw simdjson_error(error()); }
-  return first;
-}
-#endif
-
-simdjson_really_inline simdjson_result<const char *> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::current_location() noexcept {
-  if (error()) { return error(); }
-  return first.current_location();
-}
-
-simdjson_really_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::raw_json_token() noexcept {
-  if (error()) { return error(); }
-  return first.raw_json_token();
-}
-
-simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::at_pointer(std::string_view json_pointer) noexcept {
-  if (error()) { return error(); }
-  return first.at_pointer(json_pointer);
-}
-
-
 } // namespace simdjson

include/simdjson/generic/ondemand/document.h

@@ -13,7 +13,7 @@ class array_iterator;
 class document_stream;
 
 /**
- * A JSON document. It holds a json_iterator instance.
+ * A JSON document iteration.
  *
  * Used by tokens to get text, and string buffer location.
  *
@@ -27,7 +27,7 @@ public:
    * Exists so you can declare a variable and later assign to it before use.
    */
   simdjson_really_inline document() noexcept = default;
-  simdjson_really_inline document(const document &other) noexcept = delete; // pass your documents by reference, not by copy
+  simdjson_really_inline document(const document &other) noexcept = delete;
   simdjson_really_inline document(document &&other) noexcept = default;
   simdjson_really_inline document &operator=(const document &other) noexcept = delete;
   simdjson_really_inline document &operator=(document &&other) noexcept = default;
@@ -53,13 +53,6 @@ public:
    * @returns INCORRECT_TYPE If the JSON value is not a 64-bit unsigned integer.
    */
   simdjson_really_inline simdjson_result<uint64_t> get_uint64() noexcept;
-  /**
-   * Cast this JSON value (inside string) to an unsigned integer.
-   *
-   * @returns A signed 64-bit integer.
-   * @returns INCORRECT_TYPE If the JSON value is not a 64-bit unsigned integer.
-   */
-  simdjson_really_inline simdjson_result<uint64_t> get_uint64_in_string() noexcept;
   /**
    * Cast this JSON value to a signed integer.
    *
@@ -67,13 +60,6 @@ public:
    * @returns INCORRECT_TYPE If the JSON value is not a 64-bit integer.
    */
   simdjson_really_inline simdjson_result<int64_t> get_int64() noexcept;
-  /**
-   * Cast this JSON value (inside string) to a signed integer.
-   *
-   * @returns A signed 64-bit integer.
-   * @returns INCORRECT_TYPE If the JSON value is not a 64-bit integer.
-   */
-  simdjson_really_inline simdjson_result<int64_t> get_int64_in_string() noexcept;
   /**
    * Cast this JSON value to a double.
    *
@@ -81,14 +67,6 @@ public:
    * @returns INCORRECT_TYPE If the JSON value is not a valid floating-point number.
    */
   simdjson_really_inline simdjson_result<double> get_double() noexcept;
-
-  /**
-   * Cast this JSON value (inside string) to a double.
-   *
-   * @returns A double.
-   * @returns INCORRECT_TYPE If the JSON value is not a valid floating-point number.
-   */
-  simdjson_really_inline simdjson_result<double> get_double_in_string() noexcept;
   /**
    * Cast this JSON value to a string.
    *
@@ -115,14 +93,6 @@ public:
    * @returns INCORRECT_TYPE if the JSON value is not true or false.
    */
   simdjson_really_inline simdjson_result<bool> get_bool() noexcept;
-  /**
-   * Cast this JSON value to a value when the document is an object or an array.
-   *
-   * @returns A value if a JSON array or object cannot be found.
-   * @returns SCALAR_DOCUMENT_AS_VALUE error is the document is a scalar (see is_scalar() function).
-   */
-  simdjson_really_inline simdjson_result<value> get_value() noexcept;
-
   /**
    * Checks if this JSON value is null.
    *
@@ -156,9 +126,7 @@ public:
   /**
    * Get this value as the given type.
    *
-   * Supported types: object, array, raw_json_string, string_view, uint64_t, int64_t, double, bool, value
-   *
-   * Be mindful that the document instance must remain in scope while you are accessing object, array and value instances.
+   * Supported types: object, array, raw_json_string, string_view, uint64_t, int64_t, double, bool
    *
    * @param out This is set to a value of the given type, parsed from the JSON. If there is an error, this may not be initialized.
    * @returns INCORRECT_TYPE If the JSON value is not an object.
@@ -230,13 +198,6 @@ public:
    * @exception simdjson_error(INCORRECT_TYPE) if the JSON value is not true or false.
    */
   simdjson_really_inline operator bool() noexcept(false);
-  /**
-   * Cast this JSON value to a value.
-   *
-   * @returns A value value.
-   * @exception if a JSON value cannot be found
-   */
-  simdjson_really_inline operator value() noexcept(false);
 #endif
   /**
    * This method scans the array and counts the number of elements.
@@ -250,29 +211,6 @@ public:
    * safe to continue.
    */
   simdjson_really_inline simdjson_result<size_t> count_elements() & noexcept;
-   /**
-   * This method scans the object and counts the number of key-value pairs.
-   * The count_fields method should always be called before you have begun
-   * iterating through the object: it is expected that you are pointing at
-   * the beginning of the object.
-   * The runtime complexity is linear in the size of the object. After
-   * calling this function, if successful, the object is 'rewinded' at its
-   * beginning as if it had never been accessed. If the JSON is malformed (e.g.,
-   * there is a missing comma), then an error is returned and it is no longer
-   * safe to continue.
-   *
-   * To check that an object is empty, it is more performant to use
-   * the is_empty() method.
-   */
-  simdjson_really_inline simdjson_result<size_t> count_fields() & noexcept;
-  /**
-   * Get the value at the given index in the array. This function has linear-time complexity.
-   * This function should only be called once as the array iterator is not reset between each call.
-   *
-   * @return The value at the given index, or:
-   *         - INDEX_OUT_OF_BOUNDS if the array index is larger than an array length
-   */
-  simdjson_really_inline simdjson_result<value> at(size_t index) & noexcept;
   /**
    * Begin array iteration.
    *
@@ -303,14 +241,6 @@ public:
    * **Raw Keys:** The lookup will be done against the *raw* key, and will not unescape keys.
    * e.g. `object["a"]` will match `{ "a": 1 }`, but will *not* match `{ "\u0061": 1 }`.
    *
-   *
-   * You must consume the fields on an object one at a time. A request for a new key
-   * invalidates previous field values: it makes them unsafe. E.g., the array
-   * given by content["bids"].get_array() should not be accessed after you have called
-   * content["asks"].get_array(). You can detect such mistakes by first compiling and running
-   * the code in Debug mode (or with the macro `SIMDJSON_DEVELOPMENT_CHECKS` set to 1): an
-   * OUT_OF_ORDER_ITERATION error is generated.
-   *
    * @param key The key to look up.
    * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object.
    */
@@ -334,13 +264,6 @@ public:
    * Use find_field() if you are sure fields will be in order (or are willing to treat it as if the
    * field wasn't there when they aren't).
    *
-   * You must consume the fields on an object one at a time. A request for a new key
-   * invalidates previous field values: it makes them unsafe. E.g., the array
-   * given by content["bids"].get_array() should not be accessed after you have called
-   * content["asks"].get_array(). You can detect such mistakes by first compiling and running
-   * the code in Debug mode (or with the macro `SIMDJSON_DEVELOPMENT_CHECKS` set to 1): an
-   * OUT_OF_ORDER_ITERATION error is generated.
-   *
    * @param key The key to look up.
    * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object.
    */
@@ -363,77 +286,6 @@ public:
    */
   simdjson_really_inline simdjson_result<json_type> type() noexcept;
 
-  /**
-   * Checks whether the document is a scalar (string, number, null, Boolean).
-   * Returns false when there it is an array or object.
-   *
-   * @returns true if the type is string, number, null, Boolean
-   * @error TAPE_ERROR when the JSON value is a bad token like "}" "," or "alse".
-   */
-  simdjson_really_inline simdjson_result<bool> is_scalar() noexcept;
-
-  /**
-   * Checks whether the document is a negative number.
-   *
-   * @returns true if the number if negative.
-   */
-  simdjson_really_inline bool is_negative() noexcept;
-  /**
-   * Checks whether the document is an integer number. Note that
-   * this requires to partially parse the number string. If
-   * the value is determined to be an integer, it may still
-   * not parse properly as an integer in subsequent steps
-   * (e.g., it might overflow).
-   *
-   * @returns true if the number if negative.
-   */
-  simdjson_really_inline simdjson_result<bool> is_integer() noexcept;
-  /**
-   * Determine the number type (integer or floating-point number).
-   *
-   * get_number_type() is number_type::unsigned_integer if we have
-   * an integer greater or equal to 9223372036854775808
-   * get_number_type() is number_type::signed_integer if we have an
-   * integer that is less than 9223372036854775808
-   * Otherwise, get_number_type() has value number_type::floating_point_number
-   *
-   * This function req
-   * uires processing the number string, but it is expected
-   * to be faster than get_number().get_number_type() because it is does not
-   * parse the number value.
-   *
-   * @returns the type of the number
-   */
-  simdjson_really_inline simdjson_result<number_type> get_number_type() noexcept;
-
-  /**
-   * Attempt to parse an ondemand::number. An ondemand::number may
-   * contain an integer value or a floating-point value, the simdjson
-   * library will autodetect the type. Thus it is a dynamically typed
-   * number. Before accessing the value, you must determine the detected
-   * type.
-   *
-   * number.get_number_type() is number_type::signed_integer if we have
-   * an integer in [-9223372036854775808,9223372036854775808)
-   * You can recover the value by calling number.get_int64() and you
-   * have that number.is_int64() is true.
-   *
-   * number.get_number_type() is number_type::unsigned_integer if we have
-   * an integer in [9223372036854775808,18446744073709551616)
-   * You can recover the value by calling number.get_uint64() and you
-   * have that number.is_uint64() is true.
-   *
-   * Otherwise, number.get_number_type() has value number_type::floating_point_number
-   * and we have a binary64 number.
-   * You can recover the value by calling number.get_double() and you
-   * have that number.is_double() is true.
-   *
-   * You must check the type before accessing the value: it is an error
-   * to call "get_int64()" when number.get_number_type() is not
-   * number_type::signed_integer and when number.is_int64() is false.
-   */
-  simdjson_warn_unused simdjson_really_inline simdjson_result<number> get_number() noexcept;
-
   /**
    * Get the raw JSON for this token.
    *
@@ -468,16 +320,6 @@ public:
    * Returns debugging information.
    */
   inline std::string to_debug_string() noexcept;
-  /**
-   * Some unrecoverable error conditions may render the document instance unusable.
-   * The is_alive() method returns true when the document is still suitable.
-   */
-  inline bool is_alive() noexcept;
-
-  /**
-   * Returns the current location in the document if in bounds.
-   */
-  inline simdjson_result<const char *> current_location() noexcept;
 
   /**
    * Get the value associated with the given JSON pointer.  We use the RFC 6901
@@ -508,7 +350,6 @@ public:
    *         - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length
    *         - INCORRECT_TYPE if a non-integer is used to access an array
    *         - INVALID_JSON_POINTER if the JSON pointer is invalid and cannot be parsed
-   *         - SCALAR_DOCUMENT_AS_VALUE if the json_pointer is empty and the document is not a scalar (see is_scalar() function).
    */
   simdjson_really_inline simdjson_result<value> at_pointer(std::string_view json_pointer) noexcept;
   /**
@@ -528,6 +369,7 @@ protected:
 
   simdjson_really_inline value_iterator resume_value_iterator() noexcept;
   simdjson_really_inline value_iterator get_root_value_iterator() noexcept;
+  simdjson_really_inline simdjson_result<value> get_value_unsafe() noexcept;
   simdjson_really_inline simdjson_result<object> start_or_resume_object() noexcept;
   static simdjson_really_inline document start(ondemand::json_iterator &&iter) noexcept;
 
@@ -547,67 +389,6 @@ protected:
   friend class document_stream;
 };
 
-
-/**
- * A document_reference is a thin wrapper around a document reference instance.
- */
-class document_reference {
-public:
-  simdjson_really_inline document_reference() noexcept;
-  simdjson_really_inline document_reference(document &d) noexcept;
-  simdjson_really_inline document_reference(const document_reference &other) noexcept = default;
-  simdjson_really_inline document_reference& operator=(const document_reference &other) noexcept = default;
-  simdjson_really_inline void rewind() noexcept;
-  simdjson_really_inline simdjson_result<array> get_array() & noexcept;
-  simdjson_really_inline simdjson_result<object> get_object() & noexcept;
-  simdjson_really_inline simdjson_result<uint64_t> get_uint64() noexcept;
-  simdjson_really_inline simdjson_result<int64_t> get_int64() noexcept;
-  simdjson_really_inline simdjson_result<double> get_double() noexcept;
-  simdjson_really_inline simdjson_result<std::string_view> get_string() noexcept;
-  simdjson_really_inline simdjson_result<raw_json_string> get_raw_json_string() noexcept;
-  simdjson_really_inline simdjson_result<bool> get_bool() noexcept;
-  simdjson_really_inline simdjson_result<value> get_value() noexcept;
-
-  simdjson_really_inline bool is_null() noexcept;
-  simdjson_really_inline simdjson_result<std::string_view> raw_json() noexcept;
-  simdjson_really_inline operator document&() const noexcept;
-
-#if SIMDJSON_EXCEPTIONS
-  simdjson_really_inline operator array() & noexcept(false);
-  simdjson_really_inline operator object() & noexcept(false);
-  simdjson_really_inline operator uint64_t() noexcept(false);
-  simdjson_really_inline operator int64_t() noexcept(false);
-  simdjson_really_inline operator double() noexcept(false);
-  simdjson_really_inline operator std::string_view() noexcept(false);
-  simdjson_really_inline operator raw_json_string() noexcept(false);
-  simdjson_really_inline operator bool() noexcept(false);
-  simdjson_really_inline operator value() noexcept(false);
-#endif
-  simdjson_really_inline simdjson_result<size_t> count_elements() & noexcept;
-  simdjson_really_inline simdjson_result<size_t> count_fields() & noexcept;
-  simdjson_really_inline simdjson_result<value> at(size_t index) & noexcept;
-  simdjson_really_inline simdjson_result<array_iterator> begin() & noexcept;
-  simdjson_really_inline simdjson_result<array_iterator> end() & noexcept;
-  simdjson_really_inline simdjson_result<value> find_field(std::string_view key) & noexcept;
-  simdjson_really_inline simdjson_result<value> find_field(const char *key) & noexcept;
-  simdjson_really_inline simdjson_result<value> operator[](std::string_view key) & noexcept;
-  simdjson_really_inline simdjson_result<value> operator[](const char *key) & noexcept;
-  simdjson_really_inline simdjson_result<value> find_field_unordered(std::string_view key) & noexcept;
-  simdjson_really_inline simdjson_result<value> find_field_unordered(const char *key) & noexcept;
-
-  simdjson_really_inline simdjson_result<json_type> type() noexcept;
-  simdjson_really_inline simdjson_result<bool> is_scalar() noexcept;
-
-  simdjson_really_inline simdjson_result<const char *> current_location() noexcept;
-  simdjson_really_inline bool is_negative() noexcept;
-  simdjson_really_inline simdjson_result<bool> is_integer() noexcept;
-  simdjson_really_inline simdjson_result<number_type> get_number_type() noexcept;
-  simdjson_really_inline simdjson_result<number> get_number() noexcept;
-  simdjson_really_inline simdjson_result<std::string_view> raw_json_token() noexcept;
-  simdjson_really_inline simdjson_result<value> at_pointer(std::string_view json_pointer) noexcept;
-private:
-  document *doc{nullptr};
-};
 } // namespace ondemand
 } // namespace SIMDJSON_IMPLEMENTATION
 } // namespace simdjson
@@ -627,11 +408,9 @@ public:
   simdjson_really_inline simdjson_result<uint64_t> get_uint64() noexcept;
   simdjson_really_inline simdjson_result<int64_t> get_int64() noexcept;
   simdjson_really_inline simdjson_result<double> get_double() noexcept;
-  simdjson_really_inline simdjson_result<double> get_double_from_string() noexcept;
   simdjson_really_inline simdjson_result<std::string_view> get_string() noexcept;
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::raw_json_string> get_raw_json_string() noexcept;
   simdjson_really_inline simdjson_result<bool> get_bool() noexcept;
-  simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> get_value() noexcept;
   simdjson_really_inline bool is_null() noexcept;
 
   template<typename T> simdjson_really_inline simdjson_result<T> get() & noexcept;
@@ -649,11 +428,8 @@ public:
   simdjson_really_inline operator std::string_view() noexcept(false);
   simdjson_really_inline operator SIMDJSON_IMPLEMENTATION::ondemand::raw_json_string() noexcept(false);
   simdjson_really_inline operator bool() noexcept(false);
-  simdjson_really_inline operator SIMDJSON_IMPLEMENTATION::ondemand::value() noexcept(false);
 #endif
   simdjson_really_inline simdjson_result<size_t> count_elements() & noexcept;
-  simdjson_really_inline simdjson_result<size_t> count_fields() & noexcept;
-  simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> at(size_t index) & noexcept;
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> begin() & noexcept;
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> end() & noexcept;
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field(std::string_view key) & noexcept;
@@ -662,78 +438,13 @@ public:
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> operator[](const char *key) & noexcept;
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field_unordered(std::string_view key) & noexcept;
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field_unordered(const char *key) & noexcept;
+
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::json_type> type() noexcept;
-  simdjson_really_inline simdjson_result<bool> is_scalar() noexcept;
-  simdjson_really_inline simdjson_result<const char *> current_location() noexcept;
-  simdjson_really_inline bool is_negative() noexcept;
-  simdjson_really_inline simdjson_result<bool> is_integer() noexcept;
-  simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number_type> get_number_type() noexcept;
-  simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number> get_number() noexcept;
+
   /** @copydoc simdjson_really_inline std::string_view document::raw_json_token() const noexcept */
   simdjson_really_inline simdjson_result<std::string_view> raw_json_token() noexcept;
 
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> at_pointer(std::string_view json_pointer) noexcept;
 };
 
-
 } // namespace simdjson
-
-
-
-namespace simdjson {
-
-template<>
-struct simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference> : public SIMDJSON_IMPLEMENTATION::implementation_simdjson_result_base<SIMDJSON_IMPLEMENTATION::ondemand::document_reference> {
-public:
-  simdjson_really_inline simdjson_result(SIMDJSON_IMPLEMENTATION::ondemand::document_reference value, error_code error) noexcept;
-  simdjson_really_inline simdjson_result() noexcept = default;
-  simdjson_really_inline error_code rewind() noexcept;
-
-  simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array> get_array() & noexcept;
-  simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::object> get_object() & noexcept;
-  simdjson_really_inline simdjson_result<uint64_t> get_uint64() noexcept;
-  simdjson_really_inline simdjson_result<int64_t> get_int64() noexcept;
-  simdjson_really_inline simdjson_result<double> get_double() noexcept;
-  simdjson_really_inline simdjson_result<std::string_view> get_string() noexcept;
-  simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::raw_json_string> get_raw_json_string() noexcept;
-  simdjson_really_inline simdjson_result<bool> get_bool() noexcept;
-  simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> get_value() noexcept;
-  simdjson_really_inline bool is_null() noexcept;
-
-#if SIMDJSON_EXCEPTIONS
-  simdjson_really_inline operator SIMDJSON_IMPLEMENTATION::ondemand::array() & noexcept(false);
-  simdjson_really_inline operator SIMDJSON_IMPLEMENTATION::ondemand::object() & noexcept(false);
-  simdjson_really_inline operator uint64_t() noexcept(false);
-  simdjson_really_inline operator int64_t() noexcept(false);
-  simdjson_really_inline operator double() noexcept(false);
-  simdjson_really_inline operator std::string_view() noexcept(false);
-  simdjson_really_inline operator SIMDJSON_IMPLEMENTATION::ondemand::raw_json_string() noexcept(false);
-  simdjson_really_inline operator bool() noexcept(false);
-  simdjson_really_inline operator SIMDJSON_IMPLEMENTATION::ondemand::value() noexcept(false);
-#endif
-  simdjson_really_inline simdjson_result<size_t> count_elements() & noexcept;
-  simdjson_really_inline simdjson_result<size_t> count_fields() & noexcept;
-  simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> at(size_t index) & noexcept;
-  simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> begin() & noexcept;
-  simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> end() & noexcept;
-  simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field(std::string_view key) & noexcept;
-  simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field(const char *key) & noexcept;
-  simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> operator[](std::string_view key) & noexcept;
-  simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> operator[](const char *key) & noexcept;
-  simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field_unordered(std::string_view key) & noexcept;
-  simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field_unordered(const char *key) & noexcept;
-  simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::json_type> type() noexcept;
-  simdjson_really_inline simdjson_result<bool> is_scalar() noexcept;
-  simdjson_really_inline simdjson_result<const char *> current_location() noexcept;
-  simdjson_really_inline bool is_negative() noexcept;
-  simdjson_really_inline simdjson_result<bool> is_integer() noexcept;
-  simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number_type> get_number_type() noexcept;
-  simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number> get_number() noexcept;
-  /** @copydoc simdjson_really_inline std::string_view document_reference::raw_json_token() const noexcept */
-  simdjson_really_inline simdjson_result<std::string_view> raw_json_token() noexcept;
-
-  simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> at_pointer(std::string_view json_pointer) noexcept;
-};
-
-
-} // namespace simdjson

include/simdjson/generic/ondemand/document_stream-inl.h

@@ -126,7 +126,6 @@ inline size_t document_stream::size_in_bytes() const noexcept {
 }
 
 inline size_t document_stream::truncated_bytes() const noexcept {
-  if(error == CAPACITY) { return len - batch_start; }
   return parser->implementation->structural_indexes[parser->implementation->n_structural_indexes] - parser->implementation->structural_indexes[parser->implementation->n_structural_indexes + 1];
 }
 
@@ -138,9 +137,8 @@ simdjson_really_inline document_stream::iterator::iterator(document_stream* _str
   : stream{_stream}, finished{is_end} {
 }
 
-simdjson_really_inline simdjson_result<ondemand::document_reference> document_stream::iterator::operator*() noexcept {
-  //if(stream->error) { return stream->error; }
-  return simdjson_result<ondemand::document_reference>(stream->doc, stream->error);
+simdjson_really_inline ondemand::document& document_stream::iterator::operator*() noexcept {
+  return stream->doc;
 }
 
 simdjson_really_inline document_stream::iterator& document_stream::iterator::operator++() noexcept {
@@ -347,7 +345,7 @@ simdjson_really_inline std::string_view document_stream::iterator::source() cons
     cur_struct_index++;
   }
 
-  return std::string_view(reinterpret_cast<const char*>(stream->buf) + current_index(), stream->parser->implementation->structural_indexes[cur_struct_index] - current_index() + stream->batch_start + 1);;
+  return std::string_view(reinterpret_cast<const char*>(stream->buf) + current_index(), stream->parser->implementation->structural_indexes[cur_struct_index] - current_index() + stream->batch_start + 1);
 }
 
 inline error_code document_stream::iterator::error() const noexcept {

include/simdjson/generic/ondemand/document_stream.h

@@ -130,7 +130,7 @@ public:
     /**
      * Get the current document (or error).
      */
-    simdjson_really_inline simdjson_result<ondemand::document_reference> operator*() noexcept;
+    simdjson_really_inline ondemand::document& operator*() noexcept;
     /**
      * Advance to the next document (prefix).
      */

include/simdjson/generic/ondemand/field-inl.h

@@ -23,7 +23,7 @@ simdjson_really_inline simdjson_result<field> field::start(const value_iterator
 
 simdjson_really_inline simdjson_warn_unused simdjson_result<std::string_view> field::unescaped_key() noexcept {
   SIMDJSON_ASSUME(first.buf != nullptr); // We would like to call .alive() but Visual Studio won't let us.
-  simdjson_result<std::string_view> answer = first.unescape(second.iter.string_buf_loc());
+  simdjson_result<std::string_view> answer = first.unescape(*second.iter._json_iter);
   first.consume();
   return answer;
 }

include/simdjson/generic/ondemand/json_iterator-inl.h

@@ -35,13 +35,12 @@ simdjson_really_inline json_iterator::json_iterator(const uint8_t *buf, ondemand
 
 {
   logger::log_headers();
-#if SIMDJSON_CHECK_EOF
   assert_more_tokens();
-#endif
 }
 
 inline void json_iterator::rewind() noexcept {
   token.set_position( root_position() );
+  assert_more_tokens(1);
   logger::log_headers(); // We start again
   _string_buf_loc = parser->string_buf.get();
   _depth = 1;
@@ -53,7 +52,19 @@ inline void json_iterator::rewind() noexcept {
 SIMDJSON_PUSH_DISABLE_WARNINGS
 SIMDJSON_DISABLE_STRICT_OVERFLOW_WARNING
 simdjson_warn_unused simdjson_really_inline error_code json_iterator::skip_child(depth_t parent_depth) noexcept {
+  /***
+   * WARNING:
+   * Inside an object, a string value is a depth of +1 compared to the object. Yet a key
+   * is at the same depth as the object.
+   * But json_iterator cannot easily tell whether we are pointing at a key or a string value.
+   * Instead, it assumes that if you are pointing at a string, then it is a value, not a key.
+   * To be clear...
+   * the following code assumes that we are *not* pointing at a key. If we are then a bug
+   * will follow. Unfortunately, it is not possible for the json_iterator its to make this
+   * check.
+   */
   if (depth() <= parent_depth) { return SUCCESS; }
+  SIMDJSON_TRY( require_tokens(1) );
   switch (*return_current_and_advance()) {
     // TODO consider whether matching braces is a requirement: if non-matching braces indicates
     // *missing* braces, then future lookups are not in the object/arrays they think they are,
@@ -77,21 +88,22 @@ simdjson_warn_unused simdjson_really_inline error_code json_iterator::skip_child
       if (depth() <= parent_depth) { return SUCCESS; }
 #if SIMDJSON_CHECK_EOF
       // If there are no more tokens, the parent is incomplete.
-      if (at_end()) { return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "Missing [ or { at start"); }
+      if (at_end_of_input_buffer()) { return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "Missing [ or { at start"); }
 #endif // SIMDJSON_CHECK_EOF
       break;
-    case '"':
+    /*case '"':
       if(*peek() == ':') {
-        // We are at a key!!!
-        // This might happen if you just started an object and you skip it immediately.
-        // Performance note: it would be nice to get rid of this check as it is somewhat
-        // expensive.
-        // https://github.com/simdjson/simdjson/issues/1742
+        // we are at a key!!! This is
+        // only possible if someone searched
+        // for a key in an object and the key
+        // was not found but our code then
+        // decided the consume the separating
+        // comma before returning.
         logger::log_value(*this, "key");
-        return_current_and_advance(); // eat up the ':'
+        advance(); // eat up the ':'
         break; // important!!!
       }
-      simdjson_fallthrough;
+      simdjson_fallthrough;*/
     // Anything else must be a scalar value
     default:
       // For the first scalar, we will have incremented depth already, so we decrement it here.
@@ -102,7 +114,7 @@ simdjson_warn_unused simdjson_really_inline error_code json_iterator::skip_child
   }
 
   // Now that we've considered the first value, we only increment/decrement for arrays/objects
-  while (position() < end_position()) {
+  while (position() < end_of_input_buffer_position()) {
     switch (*return_current_and_advance()) {
       case '[': case '{':
         logger::log_start_value(*this, "skip");
@@ -141,10 +153,6 @@ simdjson_really_inline token_position json_iterator::root_position() const noexc
   return _root;
 }
 
-simdjson_really_inline void json_iterator::assert_at_document_depth() const noexcept {
-  SIMDJSON_ASSUME( _depth == 1 );
-}
-
 simdjson_really_inline void json_iterator::assert_at_root() const noexcept {
   SIMDJSON_ASSUME( _depth == 1 );
 #ifndef SIMDJSON_CLANG_VISUAL_STUDIO
@@ -155,7 +163,15 @@ simdjson_really_inline void json_iterator::assert_at_root() const noexcept {
 }
 
 simdjson_really_inline void json_iterator::assert_more_tokens(uint32_t required_tokens) const noexcept {
-  assert_valid_position(token._position + required_tokens - 1);
+#if SIMDJSON_CHECK_EOF
+  assert_valid_position(token.position() + required_tokens - 1);
+#else
+ // We only check that we are at a valid position when SIMDJSON_CHECK_EOF is true,
+ // but otherwise, we will still end up with the 'required_tokens' parameter and
+ // compilers are bound to complain that it might be 'unused' (rightly so). Hence
+ // the following line is used to silence compiler warnings:
+ (void)required_tokens;
+#endif
 }
 
 simdjson_really_inline void json_iterator::assert_valid_position(token_position position) const noexcept {
@@ -165,36 +181,26 @@ simdjson_really_inline void json_iterator::assert_valid_position(token_position
 #endif
 }
 
-simdjson_really_inline bool json_iterator::at_end() const noexcept {
-  return position() == end_position();
+simdjson_really_inline bool json_iterator::at_end_of_input_buffer() const noexcept {
+  return position() == end_of_input_buffer_position();
 }
-simdjson_really_inline token_position json_iterator::end_position() const noexcept {
+simdjson_really_inline token_position json_iterator::end_of_input_buffer_position() const noexcept {
   uint32_t n_structural_indexes{parser->implementation->n_structural_indexes};
   return &parser->implementation->structural_indexes[n_structural_indexes];
 }
 
-inline std::string json_iterator::to_string() const noexcept {
-  if( !is_alive() ) { return "dead json_iterator instance"; }
-  const char * current_structural = reinterpret_cast<const char *>(token.peek());
-  return std::string("json_iterator [ depth : ") + std::to_string(_depth)
-          + std::string(", structural : '") + std::string(current_structural,1)
-          + std::string("', offset : ") + std::to_string(token.current_offset())
-          + std::string("', error : ") + error_message(error)
-          + std::string(" ]");
+simdjson_really_inline const uint8_t *json_iterator::end_of_input_buffer() const noexcept {
+  return token.buf + parser->implementation->len;
 }
 
-inline simdjson_result<const char *> json_iterator::current_location() noexcept {
-  if (!is_alive()) {    // Unrecoverable error
-    if (!at_root()) {
-      return reinterpret_cast<const char *>(token.peek(-1));
-    } else {
-      return reinterpret_cast<const char *>(token.peek());
-    }
-  }
-  if (at_end()) {
-    return OUT_OF_BOUNDS;
-  }
-  return reinterpret_cast<const char *>(token.peek());
+inline std::string json_iterator::to_string() const noexcept {
+  if( !is_alive() ) { return "\"dead json_iterator instance\""; }
+  const char * current_structural = reinterpret_cast<const char *>(token.peek());
+  return std::string("{\"name\": \"json_iterator\", \"depth\" : ") + std::to_string(_depth)
+          + std::string(", \"structural\" : \"") + std::string(current_structural,1)
+          + std::string("\", \"offset\" : ") + std::to_string(token.current_offset())
+          + std::string(", \"error\" : \"") + error_message(error)
+          + std::string("\"}");
 }
 
 simdjson_really_inline bool json_iterator::is_alive() const noexcept {
@@ -207,44 +213,48 @@ simdjson_really_inline void json_iterator::abandon() noexcept {
 }
 
 simdjson_really_inline const uint8_t *json_iterator::return_current_and_advance() noexcept {
-#if SIMDJSON_CHECK_EOF
   assert_more_tokens();
-#endif // SIMDJSON_CHECK_EOF
   return token.return_current_and_advance();
 }
 
+simdjson_really_inline simdjson_result<const uint8_t *> json_iterator::try_return_current_and_advance(uint32_t required_tokens) noexcept {
+  const uint8_t *json = token.return_current_and_advance();
+  // Check this *after* we get the pointer, since getting the pointer is more time-sensitive than the branch.
+  // Also resolves nicely to 0 in the common case of required_tokens == 1.
+  SIMDJSON_TRY( require_tokens(required_tokens - 1) );
+  return json;
+}
+
+simdjson_really_inline error_code json_iterator::require_tokens(simdjson_unused uint32_t required_tokens) noexcept {
+#if SIMDJSON_CHECK_EOF
+  if (position() + required_tokens > end_of_input_buffer_position()) {
+    return report_error(TAPE_ERROR, "Document ended early");
+  }
+#endif
+  return SUCCESS;
+}
 simdjson_really_inline const uint8_t *json_iterator::unsafe_pointer() const noexcept {
   // deliberately done without safety guard:
   return token.peek(0);
 }
 
 simdjson_really_inline const uint8_t *json_iterator::peek(int32_t delta) const noexcept {
-#if SIMDJSON_CHECK_EOF
   assert_more_tokens(delta+1);
-#endif // SIMDJSON_CHECK_EOF
   return token.peek(delta);
 }
 
 simdjson_really_inline uint32_t json_iterator::peek_length(int32_t delta) const noexcept {
-#if SIMDJSON_CHECK_EOF
   assert_more_tokens(delta+1);
-#endif // #if SIMDJSON_CHECK_EOF
   return token.peek_length(delta);
 }
 
 simdjson_really_inline const uint8_t *json_iterator::peek(token_position position) const noexcept {
-  // todo: currently we require end-of-string buffering, but the following
-  // assert_valid_position should be turned on if/when we lift that condition.
-  // assert_valid_position(position);
-  // This is almost surely related to SIMDJSON_CHECK_EOF but given that SIMDJSON_CHECK_EOF
-  // is ON by default, we have no choice but to disable it for real with a comment.
+  assert_valid_position(position);
   return token.peek(position);
 }
 
 simdjson_really_inline uint32_t json_iterator::peek_length(token_position position) const noexcept {
-#if SIMDJSON_CHECK_EOF
   assert_valid_position(position);
-#endif // SIMDJSON_CHECK_EOF
   return token.peek_length(position);
 }
 
@@ -322,22 +332,6 @@ simdjson_really_inline error_code json_iterator::optional_error(error_code _erro
   return _error;
 }
 
-template<int N>
-simdjson_warn_unused simdjson_really_inline bool json_iterator::copy_to_buffer(const uint8_t *json, uint32_t max_len, uint8_t (&tmpbuf)[N]) noexcept {
-  // Let us guard against silly cases:
-  if((N < max_len) || (N == 0)) { return false; }
-  // Truncate whitespace to fit the buffer.
-  if (max_len > N-1) {
-    // if (jsoncharutils::is_not_structural_or_whitespace(json[N-1])) { return false; }
-    max_len = N-1;
-  }
-
-  // Copy to the buffer.
-  std::memcpy(tmpbuf, json, max_len);
-  tmpbuf[max_len] = ' ';
-  return true;
-}
-
 } // namespace ondemand
 } // namespace SIMDJSON_IMPLEMENTATION
 } // namespace simdjson

include/simdjson/generic/ondemand/json_iterator.h

@@ -88,19 +88,16 @@ public:
    * Get the root value iterator
    */
   simdjson_really_inline token_position root_position() const noexcept;
+
   /**
-   * Assert that we are at the document depth (== 1)
-   */
-  simdjson_really_inline void assert_at_document_depth() const noexcept;
-  /**
-   * Assert that we are at the root of the document
+   * Assert if the iterator is not at the start
    */
   simdjson_really_inline void assert_at_root() const noexcept;
 
   /**
-   * Tell whether the iterator is at the EOF mark
+   * Tell whether the iterator is at the EOF mark (end of the input buffer)
    */
-  simdjson_really_inline bool at_end() const noexcept;
+  simdjson_really_inline bool at_end_of_input_buffer() const noexcept;
 
   /**
    * Tell whether the iterator is live (has not been moved).
@@ -117,6 +114,22 @@ public:
    */
   simdjson_really_inline const uint8_t *return_current_and_advance() noexcept;
 
+  /**
+   * Advance the current token by one, without modifying depth.
+   *
+   * @param requiremed_tokens The number of tokens that must exist. Only advances one token.
+   *
+   * @error TAPE_ERROR If there are not at least required_tokens tokens remaining.
+   */
+  simdjson_really_inline simdjson_result<const uint8_t *> try_return_current_and_advance(uint32_t required_tokens=1) noexcept;
+
+  /**
+   * Return an error unless there are enough tokens left.
+   *
+   * @param required_tokens The number of tokens that must exist.
+   * @error TAPE_ERROR If there are not at least required_tokens tokens remaining.
+   */
+  simdjson_really_inline error_code require_tokens(uint32_t required_tokens=1) noexcept;
   /**
    * Assert that there are at least the given number of tokens left.
    *
@@ -148,6 +161,14 @@ public:
    * @param delta The relative position of the token to retrieve. e.g. 0 = next token, -1 = prev token.
    */
   simdjson_really_inline uint32_t peek_length(int32_t delta=0) const noexcept;
+  /**
+   * Get the JSON text for a given token.
+   *
+   * This is not null-terminated; it is a view into the JSON.
+   *
+   * @param position The position of the token to retrieve.
+   */
+  simdjson_really_inline const uint8_t *peek(token_position position) const noexcept;
   /**
    * Get a pointer to the current location in the input buffer.
    *
@@ -157,17 +178,6 @@ public:
    * safe to derefence this pointer.
    */
   simdjson_really_inline const uint8_t *unsafe_pointer() const noexcept;
-  /**
-   * Get the JSON text for a given token.
-   *
-   * This is not null-terminated; it is a view into the JSON.
-   *
-   * @param position The position of the token to retrieve.
-   *
-   * TODO consider a string_view, assuming the length will get stripped out by the optimizer when
-   * it isn't used ...
-   */
-  simdjson_really_inline const uint8_t *peek(token_position position) const noexcept;
   /**
    * Get the maximum length of the JSON text for the current token (or relative).
    *
@@ -216,7 +226,7 @@ public:
   simdjson_really_inline uint8_t *&string_buf_loc() noexcept;
 
   /**
-   * Report an unrecoverable error, preventing further iteration.
+   * Report an error, preventing further iteration.
    *
    * @param error The error to report. Must not be SUCCESS, UNINITIALIZED, INCORRECT_TYPE, or NO_SUCH_FIELD.
    * @param message An error message to report with the error.
@@ -230,8 +240,6 @@ public:
    */
   simdjson_really_inline error_code optional_error(error_code error, const char *message) noexcept;
 
-  template<int N> simdjson_warn_unused simdjson_really_inline bool copy_to_buffer(const uint8_t *json, uint32_t max_len, uint8_t (&tmpbuf)[N]) noexcept;
-
   simdjson_really_inline token_position position() const noexcept;
   simdjson_really_inline void reenter_child(token_position position, depth_t child_depth) noexcept;
 #ifdef SIMDJSON_DEVELOPMENT_CHECKS
@@ -240,12 +248,6 @@ public:
 #endif
   /* Useful for debugging and logging purposes. */
   inline std::string to_string() const noexcept;
-
-  /**
-   * Returns the current location in the document if in bounds.
-   */
-  inline simdjson_result<const char *> current_location() noexcept;
-
   /**
    * Updates this json iterator so that it is back at the beginning of the document,
    * as if it had just been created.
@@ -256,9 +258,9 @@ protected:
   /// The last token before the end
   simdjson_really_inline token_position last_position() const noexcept;
   /// The token *at* the end. This points at gibberish and should only be used for comparison.
-  simdjson_really_inline token_position end_position() const noexcept;
+  simdjson_really_inline token_position end_of_input_buffer_position() const noexcept;
   /// The end of the buffer.
-  simdjson_really_inline token_position end() const noexcept;
+  simdjson_really_inline const uint8_t *end_of_input_buffer() const noexcept;
 
   friend class document;
   friend class document_stream;

include/simdjson/generic/ondemand/json_type-inl.h

@@ -15,93 +15,12 @@ inline std::ostream& operator<<(std::ostream& out, json_type type) noexcept {
     return out;
 }
 
-inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept {
-    switch (type) {
-        case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break;
-        case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break;
-        case number_type::floating_point_number: out << "floating-point number (binary64)"; break;
-        default: SIMDJSON_UNREACHABLE();
-    }
-    return out;
-}
 #if SIMDJSON_EXCEPTIONS
 inline std::ostream& operator<<(std::ostream& out, simdjson_result<json_type> &type) noexcept(false) {
     return out << type.value();
 }
 #endif
 
-
-
-simdjson_really_inline number_type number::get_number_type() const noexcept {
-  return type;
-}
-
-simdjson_really_inline bool number::is_uint64() const noexcept {
-  return get_number_type() == number_type::unsigned_integer;
-}
-
-simdjson_really_inline uint64_t number::get_uint64() const noexcept {
-  return payload.unsigned_integer;
-}
-
-simdjson_really_inline number::operator uint64_t() const noexcept {
-  return get_uint64();
-}
-
-
-simdjson_really_inline bool number::is_int64() const noexcept {
-  return get_number_type() == number_type::signed_integer;
-}
-
-simdjson_really_inline int64_t number::get_int64() const noexcept {
-  return payload.signed_integer;
-}
-
-simdjson_really_inline number::operator int64_t() const noexcept {
-  return get_int64();
-}
-
-simdjson_really_inline bool number::is_double() const noexcept {
-    return get_number_type() == number_type::floating_point_number;
-}
-
-simdjson_really_inline double number::get_double() const noexcept {
-  return payload.floating_point_number;
-}
-
-simdjson_really_inline number::operator double() const noexcept {
-  return get_double();
-}
-
-simdjson_really_inline double number::as_double() const noexcept {
-  if(is_double()) {
-    return payload.floating_point_number;
-  }
-  if(is_int64()) {
-    return double(payload.signed_integer);
-  }
-  return double(payload.unsigned_integer);
-}
-
-simdjson_really_inline void number::append_s64(int64_t value) noexcept {
-  payload.signed_integer = value;
-  type = number_type::signed_integer;
-}
-
-simdjson_really_inline void number::append_u64(uint64_t value) noexcept {
-  payload.unsigned_integer = value;
-  type = number_type::unsigned_integer;
-}
-
-simdjson_really_inline void number::append_double(double value) noexcept {
-  payload.floating_point_number = value;
-  type = number_type::floating_point_number;
-}
-
-simdjson_really_inline void number::skip_double() noexcept {
-  type = number_type::floating_point_number;
-}
-
 } // namespace ondemand
 } // namespace SIMDJSON_IMPLEMENTATION
 } // namespace simdjson

include/simdjson/generic/ondemand/json_type.h

@@ -1,6 +1,7 @@
 namespace simdjson {
 namespace SIMDJSON_IMPLEMENTATION {
 namespace ondemand {
+
 /**
  * The type of a JSON value.
  */
@@ -14,106 +15,6 @@ enum class json_type {
     null     ///< A JSON null    (null)
 };
 
-class value_iterator;
-
-/**
- * A type representing a JSON number.
- * The design of the struct is deliberately straight-forward. All
- * functions return standard values with no error check.
- */
-struct number {
-
-  /**
-   * return the automatically determined type of
-   * the number: number_type::floating_point_number,
-   * number_type::signed_integer or number_type::unsigned_integer.
-   *
-   *    enum class number_type {
-   *        floating_point_number=1, /// a binary64 number
-   *        signed_integer,          /// a signed integer that fits in a 64-bit word using two's complement
-   *        unsigned_integer         /// a positive integer larger or equal to 1<<63
-   *    };
-   */
-  simdjson_really_inline number_type get_number_type() const noexcept;
-  /**
-   * return true if the automatically determined type of
-   * the number is number_type::unsigned_integer.
-   */
-  simdjson_really_inline bool is_uint64() const noexcept;
-  /**
-   * return the value as a uint64_t, only valid if is_uint64() is true.
-   */
-  simdjson_really_inline uint64_t get_uint64() const noexcept;
-  simdjson_really_inline operator uint64_t() const noexcept;
-
-  /**
-   * return true if the automatically determined type of
-   * the number is number_type::signed_integer.
-   */
-  simdjson_really_inline bool is_int64() const noexcept;
-  /**
-   * return the value as a int64_t, only valid if is_int64() is true.
-   */
-  simdjson_really_inline int64_t get_int64() const noexcept;
-  simdjson_really_inline operator int64_t() const noexcept;
-
-
-  /**
-   * return true if the automatically determined type of
-   * the number is number_type::floating_point_number.
-   */
-  simdjson_really_inline bool is_double() const noexcept;
-  /**
-   * return the value as a double, only valid if is_double() is true.
-   */
-  simdjson_really_inline double get_double() const noexcept;
-  simdjson_really_inline operator double() const noexcept;
-
-  /**
-   * Convert the number to a double. Though it always succeed, the conversion
-   * may be lossy if the number cannot be represented exactly.
-   */
-  simdjson_really_inline double as_double() const noexcept;
-
-
-protected:
-  /**
-   * The next block of declaration is designed so that we can call the number parsing
-   * functions on a number type. They are protected and should never be used outside
-   * of the core simdjson library.
-   */
-  friend class value_iterator;
-  template<typename W>
-  friend error_code numberparsing::write_float(const uint8_t *const src, bool negative, uint64_t i, const uint8_t * start_digits, size_t digit_count, int64_t exponent, W &writer);
-  template<typename W>
-  friend error_code numberparsing::parse_number(const uint8_t *const src, W &writer);
-  template<typename W>
-  friend error_code numberparsing::slow_float_parsing(simdjson_unused const uint8_t * src, W writer);
-  /** Store a signed 64-bit value to the number. */
-  simdjson_really_inline void append_s64(int64_t value) noexcept;
-  /** Store an unsigned 64-bit value to the number. */
-  simdjson_really_inline void append_u64(uint64_t value) noexcept;
-  /** Store a double value to the number. */
-  simdjson_really_inline void append_double(double value) noexcept;
-  /** Specifies that the value is a double, but leave it undefined. */
-  simdjson_really_inline void skip_double() noexcept;
-  /**
-   * End of friend declarations.
-   */
-
-  /**
-   * Our attributes are a union type (size = 64 bits)
-   * followed by a type indicator.
-   */
-  union {
-    double floating_point_number;
-    int64_t signed_integer;
-    uint64_t unsigned_integer;
-  } payload{0};
-  number_type type{number_type::signed_integer};
-  friend class value_iterator;
-};
-
 /**
  * Write the JSON type to the output stream
  *
@@ -121,7 +22,6 @@ protected:
  * @param type The json_type.
  */
 inline std::ostream& operator<<(std::ostream& out, json_type type) noexcept;
-inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept;
 
 #if SIMDJSON_EXCEPTIONS
 /**

include/simdjson/generic/ondemand/logger-inl.h

@@ -124,6 +124,7 @@ inline void log_headers() noexcept {
 inline void log_line(const json_iterator &iter, const char *title_prefix, const char *title, std::string_view detail, int delta, int depth_delta) noexcept {
   log_line(iter, iter.position()+delta, depth_t(iter.depth()+depth_delta), title_prefix, title, detail);
 }
+
 inline void log_line(const json_iterator &iter, token_position index, depth_t depth, const char *title_prefix, const char *title, std::string_view detail) noexcept {
   if (LOG_ENABLED) {
     const int indent = depth*2;

include/simdjson/generic/ondemand/object-inl.h

@@ -42,7 +42,9 @@ simdjson_really_inline simdjson_result<object> object::start_root(value_iterator
   return object(iter);
 }
 simdjson_really_inline error_code object::consume() noexcept {
-  if(iter.is_at_key()) {
+  bool at_key;
+  SIMDJSON_TRY( iter.is_at_key().get(at_key) );
+  if(at_key) {
     /**
      * whenever you are pointing at a key, calling skip_child() is
      * unsafe because you will hit a string and you will assume that
@@ -71,10 +73,12 @@ simdjson_really_inline simdjson_result<std::string_view> object::raw_json() noex
   const uint8_t * starting_point{iter.peek_start()};
   auto error = consume();
   if(error) { return error; }
-  const uint8_t * final_point{iter._json_iter->peek(0)};
+  // After 'consume()', we could be left pointing just beyond the document, but that
+  // is ok because we are not going to dereference the final pointer position, we just
+  // use it to compute the length in bytes.
+  const uint8_t * final_point{iter._json_iter->unsafe_pointer()};
   return std::string_view(reinterpret_cast<const char*>(starting_point), size_t(final_point - starting_point));
 }
-
 simdjson_really_inline simdjson_result<object> object::started(value_iterator &iter) noexcept {
   SIMDJSON_TRY( iter.started_object().error() );
   return object(iter);
@@ -139,29 +143,6 @@ inline simdjson_result<value> object::at_pointer(std::string_view json_pointer)
   return child;
 }
 
-simdjson_really_inline simdjson_result<size_t> object::count_fields() & noexcept {
-  size_t count{0};
-  // Important: we do not consume any of the values.
-  for(simdjson_unused auto v : *this) { count++; }
-  // The above loop will always succeed, but we want to report errors.
-  if(iter.error()) { return iter.error(); }
-  // We need to move back at the start because we expect users to iterate through
-  // the object after counting the number of elements.
-  iter.reset_object();
-  return count;
-}
-
-simdjson_really_inline simdjson_result<bool> object::is_empty() & noexcept {
-  bool is_not_empty;
-  auto error = iter.reset_object().get(is_not_empty);
-  if(error) { return error; }
-  return !is_not_empty;
-}
-
-simdjson_really_inline simdjson_result<bool> object::reset() & noexcept {
-  return iter.reset_object();
-}
-
 } // namespace ondemand
 } // namespace SIMDJSON_IMPLEMENTATION
 } // namespace simdjson
@@ -211,19 +192,4 @@ simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>
   return first.at_pointer(json_pointer);
 }
 
-inline simdjson_result<bool> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::object>::reset() noexcept {
-  if (error()) { return error(); }
-  return first.reset();
-}
-
-inline simdjson_result<bool> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::object>::is_empty() noexcept {
-  if (error()) { return error(); }
-  return first.is_empty();
-}
-
-simdjson_really_inline  simdjson_result<size_t> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::object>::count_fields() & noexcept {
-  if (error()) { return error(); }
-  return first.count_fields();
-}
-
 } // namespace simdjson

include/simdjson/generic/ondemand/object.h

@@ -37,13 +37,6 @@ public:
    * **Raw Keys:** The lookup will be done against the *raw* key, and will not unescape keys.
    * e.g. `object["a"]` will match `{ "a": 1 }`, but will *not* match `{ "\u0061": 1 }`.
    *
-   * You must consume the fields on an object one at a time. A request for a new key
-   * invalidates previous field values: it makes them unsafe. E.g., the array
-   * given by content["bids"].get_array() should not be accessed after you have called
-   * content["asks"].get_array(). You can detect such mistakes by first compiling and running
-   * the code in Debug mode (or with the macro `SIMDJSON_DEVELOPMENT_CHECKS` set to 1): an
-   * OUT_OF_ORDER_ITERATION error is generated.
-   *
    * @param key The key to look up.
    * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object.
    */
@@ -70,13 +63,6 @@ public:
    * If you have multiple fields with a matching key ({"x": 1,  "x": 1}) be mindful
    * that only one field is returned.
    *
-   * You must consume the fields on an object one at a time. A request for a new key
-   * invalidates previous field values: it makes them unsafe. E.g., the array
-   * given by content["bids"].get_array() should not be accessed after you have called
-   * content["asks"].get_array(). You can detect such mistakes by first compiling and running
-   * the code in Debug mode (or with the macro `SIMDJSON_DEVELOPMENT_CHECKS` set to 1): an
-   * OUT_OF_ORDER_ITERATION error is generated.
-   *
    * @param key The key to look up.
    * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object.
    */
@@ -124,41 +110,6 @@ public:
    */
   inline simdjson_result<value> at_pointer(std::string_view json_pointer) noexcept;
 
-  /**
-   * Reset the iterator so that we are pointing back at the
-   * beginning of the object. You should still consume values only once even if you
-   * can iterate through the object more than once. If you unescape a string within
-   * the object more than once, you have unsafe code. Note that rewinding an object
-   * means that you may need to reparse it anew: it is not a free operation.
-   *
-   * @returns true if the object contains some elements (not empty)
-   */
-  inline simdjson_result<bool> reset() & noexcept;
-  /**
-   * This method scans the beginning of the object and checks whether the
-   * object is empty.
-   * The runtime complexity is constant time. After
-   * calling this function, if successful, the object is 'rewinded' at its
-   * beginning as if it had never been accessed. If the JSON is malformed (e.g.,
-   * there is a missing comma), then an error is returned and it is no longer
-   * safe to continue.
-   */
-  inline simdjson_result<bool> is_empty() & noexcept;
-  /**
-   * This method scans the object and counts the number of key-value pairs.
-   * The count_fields method should always be called before you have begun
-   * iterating through the object: it is expected that you are pointing at
-   * the beginning of the object.
-   * The runtime complexity is linear in the size of the object. After
-   * calling this function, if successful, the object is 'rewinded' at its
-   * beginning as if it had never been accessed. If the JSON is malformed (e.g.,
-   * there is a missing comma), then an error is returned and it is no longer
-   * safe to continue.
-   *
-   * To check that an object is empty, it is more performant to use
-   * the is_empty() method.
-   */
-  simdjson_really_inline simdjson_result<size_t> count_fields() & noexcept;
   /**
    * Consumes the object and returns a string_view instance corresponding to the
    * object as represented in JSON. It points inside the original byte array containg
@@ -208,10 +159,6 @@ public:
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> operator[](std::string_view key) & noexcept;
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> operator[](std::string_view key) && noexcept;
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> at_pointer(std::string_view json_pointer) noexcept;
-  inline simdjson_result<bool> reset() noexcept;
-  inline simdjson_result<bool> is_empty() noexcept;
-  inline simdjson_result<size_t> count_fields() & noexcept;
-
 };
 
 } // namespace simdjson

include/simdjson/generic/ondemand/parser-inl.h

@@ -28,17 +28,25 @@ simdjson_warn_unused simdjson_really_inline error_code parser::allocate(size_t n
   return SUCCESS;
 }
 
-simdjson_warn_unused simdjson_really_inline simdjson_result<document> parser::iterate(padded_string_view json) & noexcept {
-  if (json.padding() < SIMDJSON_PADDING) { return INSUFFICIENT_PADDING; }
 
+simdjson_warn_unused simdjson_really_inline simdjson_result<document> parser::iterate(const char *json, size_t len) & noexcept {
   // Allocate if needed
-  if (capacity() < json.length() || !string_buf) {
-    SIMDJSON_TRY( allocate(json.length(), max_depth()) );
+  if (capacity() < len || !string_buf) {
+    SIMDJSON_TRY( allocate(len, max_depth()) );
   }
 
   // Run stage 1.
-  SIMDJSON_TRY( implementation->stage1(reinterpret_cast<const uint8_t *>(json.data()), json.length(), stage1_mode::regular) );
-  return document::start({ reinterpret_cast<const uint8_t *>(json.data()), this });
+  SIMDJSON_TRY( implementation->stage1(reinterpret_cast<const uint8_t *>(json), len, stage1_mode::regular) );
+  return document::start({ reinterpret_cast<const uint8_t *>(json), this });
+}
+
+simdjson_warn_unused simdjson_really_inline simdjson_result<document> parser::iterate(std::string_view view) & noexcept {
+  return iterate(view.data(), view.size());
+}
+
+simdjson_warn_unused simdjson_really_inline simdjson_result<document> parser::iterate(padded_string_view json) & noexcept {
+  // Iterate does not require any padding!!!
+  return iterate(json.data(), json.size());
 }
 
 simdjson_warn_unused simdjson_really_inline simdjson_result<document> parser::iterate(const char *json, size_t len, size_t allocated) & noexcept {
@@ -54,7 +62,7 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<document> parser::it
 }
 
 simdjson_warn_unused simdjson_really_inline simdjson_result<document> parser::iterate(const std::string &json) & noexcept {
-  return iterate(padded_string_view(json));
+  return iterate(json.data(), json.size());
 }
 
 simdjson_warn_unused simdjson_really_inline simdjson_result<document> parser::iterate(const simdjson_result<padded_string_view> &result) & noexcept {
@@ -64,6 +72,11 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<document> parser::it
   return iterate(json);
 }
 
+simdjson_warn_unused simdjson_really_inline simdjson_result<document> parser::iterate(const padded_string & json) & noexcept {
+  // We do not use the padding at all anymore.
+  return iterate(json.data(), json.size());
+}
+
 simdjson_warn_unused simdjson_really_inline simdjson_result<document> parser::iterate(const simdjson_result<padded_string> &result) & noexcept {
   // We don't presently have a way to temporarily get a const T& from a simdjson_result<T> without throwing an exception
   SIMDJSON_TRY( result.error() );
@@ -72,7 +85,8 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<document> parser::it
 }
 
 simdjson_warn_unused simdjson_really_inline simdjson_result<json_iterator> parser::iterate_raw(padded_string_view json) & noexcept {
-  if (json.padding() < SIMDJSON_PADDING) { return INSUFFICIENT_PADDING; }
+  // Iterate does not require any padding!!!
+  // if (json.padding() < SIMDJSON_PADDING) { return INSUFFICIENT_PADDING; }
 
   // Allocate if needed
   if (capacity() < json.length()) {
@@ -97,6 +111,9 @@ inline simdjson_result<document_stream> parser::iterate_many(const std::string &
 inline simdjson_result<document_stream> parser::iterate_many(const padded_string &s, size_t batch_size) noexcept {
   return iterate_many(s.data(), s.length(), batch_size);
 }
+inline simdjson_result<document_stream> parser::iterate_many(std::string_view view, size_t batch_size) noexcept {
+  return iterate_many(view.data(), view.length(), batch_size);
+}
 
 simdjson_really_inline size_t parser::capacity() const noexcept {
   return _capacity;

include/simdjson/generic/ondemand/parser.h

@@ -54,10 +54,7 @@ public:
    *   ondemand::parser parser;
    *   document doc = parser.iterate(json);
    *
-   * It is expected that the content is a valid UTF-8 file, containing a valid JSON document.
-   * Otherwise the iterate method may return an error. In particular, the whole input should be
-   * valid: we do not attempt to tolerate incorrect content either before or after a JSON
-   * document.
+   * The input buffer does not need to be padded.
    *
    * ### IMPORTANT: Validate what you use
    *
@@ -75,17 +72,11 @@ public:
    * iteration to ensure intermediate buffers can be accessed. Any document must be destroyed before
    * you call parse() again or destroy the parser.
    *
-   * ### REQUIRED: Buffer Padding
-   *
-   * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what
-   * those bytes are initialized to, as long as they are allocated.
    *
    * @param json The JSON to parse.
    * @param len The length of the JSON.
-   * @param capacity The number of bytes allocated in the JSON (must be at least len+SIMDJSON_PADDING).
    *
    * @return The document, or an error:
-   *         - INSUFFICIENT_PADDING if the input has less than SIMDJSON_PADDING extra bytes.
    *         - MEMALLOC if realloc_if_needed the parser does not have enough capacity, and memory
    *           allocation fails.
    *         - EMPTY if the document is all whitespace.
@@ -93,59 +84,68 @@ public:
    *         - UNESCAPED_CHARS if a string contains control characters that must be escaped
    *         - UNCLOSED_STRING if there is an unclosed string in the document.
    */
-  simdjson_warn_unused simdjson_result<document> iterate(padded_string_view json) & noexcept;
-  /** @overload simdjson_result<document> iterate(padded_string_view json) & noexcept */
-  simdjson_warn_unused simdjson_result<document> iterate(const char *json, size_t len, size_t capacity) & noexcept;
-  /** @overload simdjson_result<document> iterate(padded_string_view json) & noexcept */
-  simdjson_warn_unused simdjson_result<document> iterate(const uint8_t *json, size_t len, size_t capacity) & noexcept;
-  /** @overload simdjson_result<document> iterate(padded_string_view json) & noexcept */
-  simdjson_warn_unused simdjson_result<document> iterate(std::string_view json, size_t capacity) & noexcept;
-  /** @overload simdjson_result<document> iterate(padded_string_view json) & noexcept */
+  simdjson_warn_unused simdjson_result<document> iterate(const char *json, size_t len) & noexcept;
+  /** @overload simdjson_result<document> iterate(const char *json, size_t len) & noexcept */
+  simdjson_warn_unused simdjson_result<document> iterate(std::string_view view) & noexcept;
+  /** @overload simdjson_result<document> iterate(const char *json, size_t len) & noexcept */
   simdjson_warn_unused simdjson_result<document> iterate(const std::string &json) & noexcept;
-  /** @overload simdjson_result<document> iterate(padded_string_view json) & noexcept */
+
+  /**
+   * @overload simdjson_result<document> iterate(const char *json, size_t len) & noexcept;
+   * iterate does not require padding, this function is kept for backward compatibility and convenience.
+   **/
+  simdjson_warn_unused simdjson_result<document> iterate(padded_string_view json) & noexcept;
+
+  /**
+   * @overload simdjson_result<document> iterate(const char *json, size_t len) & noexcept;
+   * iterate does not require padding, this function is kept for backward compatibility and convenience.
+   **/
+  simdjson_warn_unused simdjson_result<document> iterate(const padded_string & json) & noexcept;
+
+  /**
+   * @overload simdjson_result<document> iterate(const char *json, size_t len) & noexcept;
+   * iterate does not require padding, this function is kept for backward compatibility and convenience.
+   **/
+  simdjson_warn_unused simdjson_result<document> iterate(const char *json, size_t len, size_t capacity) & noexcept;
+
+  /**
+   * @overload simdjson_result<document> iterate(const char *json, size_t len) & noexcept;
+   * iterate does not require padding, this function is kept for backward compatibility and convenience.
+   **/
+  simdjson_warn_unused simdjson_result<document> iterate(const uint8_t *json, size_t len, size_t capacity) & noexcept;
+
+  /**
+   * @overload simdjson_result<document> iterate(const char *json, size_t len) & noexcept;
+   * iterate does not require padding, this function is kept for backward compatibility and convenience.
+   **/
+  simdjson_warn_unused simdjson_result<document> iterate(std::string_view json, size_t capacity) & noexcept;
+
+  /**
+   * @overload simdjson_result<document> iterate(const char *json, size_t len) & noexcept;
+   * iterate does not require padding, this function is kept for backward compatibility and convenience.
+   **/
   simdjson_warn_unused simdjson_result<document> iterate(const simdjson_result<padded_string> &json) & noexcept;
-  /** @overload simdjson_result<document> iterate(padded_string_view json) & noexcept */
+
+  /**
+   * @overload simdjson_result<document> iterate(const char *json, size_t len) & noexcept;
+   * iterate does not require padding, this function is kept for backward compatibility and convenience.
+   **/
   simdjson_warn_unused simdjson_result<document> iterate(const simdjson_result<padded_string_view> &json) & noexcept;
-  /** @overload simdjson_result<document> iterate(padded_string_view json) & noexcept */
+
+  /**
+   * @overload simdjson_result<document> iterate(const char *json, size_t len) & noexcept;
+   * iterate does not require padding, this function is kept for backward compatibility and convenience.
+   **/
   simdjson_warn_unused simdjson_result<document> iterate(padded_string &&json) & noexcept = delete;
+  /**
+   * @overload simdjson_result<document> iterate(const char *json, size_t len) & noexcept;
+   * iterate does not require padding, this function is kept for backward compatibility and convenience.
+   **/
+  simdjson_warn_unused simdjson_result<document> iterate(std::string &&json) & noexcept = delete;
+
 
   /**
    * @private
-   *
-   * Start iterating an on-demand JSON document.
-   *
-   *   ondemand::parser parser;
-   *   json_iterator doc = parser.iterate(json);
-   *
-   * ### IMPORTANT: Buffer Lifetime
-   *
-   * Because parsing is done while you iterate, you *must* keep the JSON buffer around at least as
-   * long as the document iteration.
-   *
-   * ### IMPORTANT: Document Lifetime
-   *
-   * Only one iteration at a time can happen per parser, and the parser *must* be kept alive during
-   * iteration to ensure intermediate buffers can be accessed. Any document must be destroyed before
-   * you call parse() again or destroy the parser.
-   *
-   * The ondemand::document instance holds the iterator. The document must remain in scope
-   * while you are accessing instances of ondemand::value, ondemand::object, ondemand::array.
-   *
-   * ### REQUIRED: Buffer Padding
-   *
-   * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what
-   * those bytes are initialized to, as long as they are allocated.
-   *
-   * @param json The JSON to parse.
-   *
-   * @return The iterator, or an error:
-   *         - INSUFFICIENT_PADDING if the input has less than SIMDJSON_PADDING extra bytes.
-   *         - MEMALLOC if realloc_if_needed the parser does not have enough capacity, and memory
-   *           allocation fails.
-   *         - EMPTY if the document is all whitespace.
-   *         - UTF8_ERROR if the document is not valid UTF-8.
-   *         - UNESCAPED_CHARS if a string contains control characters that must be escaped
-   *         - UNCLOSED_STRING if there is an unclosed string in the document.
    */
   simdjson_warn_unused simdjson_result<json_iterator> iterate_raw(padded_string_view json) & noexcept;
 
@@ -153,7 +153,7 @@ public:
   /**
    * Parse a buffer containing many JSON documents.
    *
-   *   auto json = R"({ "foo": 1 } { "foo": 2 } { "foo": 3 } )"_padded;
+   *   std::string json = R"({ "foo": 1 } { "foo": 2 } { "foo": 3 } )";
    *   ondemand::parser parser;
    *   ondemand::document_stream docs = parser.iterate_many(json);
    *   for (auto & doc : docs) {
@@ -164,32 +164,25 @@ public:
    * No copy of the input buffer is made.
    *
    * The function is lazy: it may be that no more than one JSON document at a time is parsed.
+
    *
    * The caller is responsabile to ensure that the input string data remains unchanged and is
    * not deleted during the loop.
-   *
    * ### Format
    *
    * The buffer must contain a series of one or more JSON documents, concatenated into a single
-   * buffer, separated by ASCII whitespace. It effectively parses until it has a fully valid document,
+   * buffer, separated by whitespace. It effectively parses until it has a fully valid document,
    * then starts parsing the next document at that point. (It does this with more parallelism and
    * lookahead than you might think, though.)
    *
    * documents that consist of an object or array may omit the whitespace between them, concatenating
-   * with no separator. Documents that consist of a single primitive (i.e. documents that are not
-   * arrays or objects) MUST be separated with ASCII whitespace.
-   *
-   * The characters inside a JSON document, and between JSON documents, must be valid Unicode (UTF-8).
+   * with no separator. documents that consist of a single primitive (i.e. documents that are not
+   * arrays or objects) MUST be separated with whitespace.
    *
    * The documents must not exceed batch_size bytes (by default 1MB) or they will fail to parse.
    * Setting batch_size to excessively large or excesively small values may impact negatively the
    * performance.
    *
-   * ### REQUIRED: Buffer Padding
-   *
-   * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what
-   * those bytes are initialized to, as long as they are allocated.
-   *
    * ### Threads
    *
    * When compiled with SIMDJSON_THREADS_ENABLED, this method will use a single thread under the
@@ -212,6 +205,7 @@ public:
    *         - other json errors if parsing fails. You should not rely on these errors to always the same for the
    *           same document: they may vary under runtime dispatch (so they may vary depending on your system and hardware).
    */
+  inline simdjson_result<document_stream> iterate_many(std::string_view view, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept;
   inline simdjson_result<document_stream> iterate_many(const uint8_t *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept;
   /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */
   inline simdjson_result<document_stream> iterate_many(const char *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept;

include/simdjson/generic/ondemand/raw_json_string-inl.h

@@ -6,8 +6,9 @@ namespace ondemand {
 simdjson_really_inline raw_json_string::raw_json_string(const uint8_t * _buf) noexcept : buf{_buf} {}
 
 simdjson_really_inline const char * raw_json_string::raw() const noexcept { return reinterpret_cast<const char *>(buf); }
-simdjson_really_inline simdjson_warn_unused simdjson_result<std::string_view> raw_json_string::unescape(uint8_t *&dst) const noexcept {
-  uint8_t *end = stringparsing::parse_string(buf, dst);
+
+simdjson_really_inline simdjson_warn_unused simdjson_result<std::string_view> raw_json_string::unescape(uint8_t *&dst, const uint8_t *buf_end) const noexcept {
+  uint8_t *end = stringparsing::parse_string(buf, dst, buf_end);
   if (!end) { return STRING_ERROR; }
   std::string_view result(reinterpret_cast<const char *>(dst), end-dst);
   dst = end;
@@ -51,17 +52,19 @@ simdjson_really_inline bool raw_json_string::is_free_from_unescaped_quote(const
 }
 
 
-simdjson_really_inline bool raw_json_string::unsafe_is_equal(size_t length, std::string_view target) const noexcept {
+simdjson_really_inline bool raw_json_string::unsafe_is_equal(size_t max_key_length_including_final_quote, std::string_view target) const noexcept {
   // If we are going to call memcmp, then we must know something about the length of the raw_json_string.
-  return (length >= target.size()) && (raw()[target.size()] == '"') && !memcmp(raw(), target.data(), target.size());
+  if(max_key_length_including_final_quote <= target.size()) { return false; }
+  // It is now safe to read in [0, target.size()].
+  return (raw()[target.size()] == '"') && (memcmp(raw(), target.data(), target.size()) == 0);
 }
 
 simdjson_really_inline bool raw_json_string::unsafe_is_equal(std::string_view target) const noexcept {
   // Assumptions: does not contain unescaped quote characters, and
   // the raw content is quote terminated within a valid JSON string.
-  if(target.size() <= SIMDJSON_PADDING) {
-    return (raw()[target.size()] == '"') && !memcmp(raw(), target.data(), target.size());
-  }
+  // if(target.size() <= SIMDJSON_PADDING) {
+  //  return (raw()[target.size()] == '"') && !memcmp(raw(), target.data(), target.size());
+  // }
   const char * r{raw()};
   size_t pos{0};
   for(;pos < target.size();pos++) {
@@ -150,7 +153,7 @@ simdjson_unused simdjson_really_inline bool operator!=(std::string_view c, const
 
 
 simdjson_really_inline simdjson_warn_unused simdjson_result<std::string_view> raw_json_string::unescape(json_iterator &iter) const noexcept {
-  return unescape(iter.string_buf_loc());
+  return unescape(iter.string_buf_loc(), iter.end_of_input_buffer());
 }
 
 
@@ -183,9 +186,9 @@ simdjson_really_inline simdjson_result<const char *> simdjson_result<SIMDJSON_IM
   if (error()) { return error(); }
   return first.raw();
 }
-simdjson_really_inline simdjson_warn_unused simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::raw_json_string>::unescape(uint8_t *&dst) const noexcept {
+simdjson_really_inline simdjson_warn_unused simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::raw_json_string>::unescape(uint8_t *&dst, const uint8_t *buf_end) const noexcept {
   if (error()) { return error(); }
-  return first.unescape(dst);
+  return first.unescape(dst, buf_end);
 }
 simdjson_really_inline simdjson_warn_unused simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::raw_json_string>::unescape(SIMDJSON_IMPLEMENTATION::ondemand::json_iterator &iter) const noexcept {
   if (error()) { return error(); }

include/simdjson/generic/ondemand/raw_json_string.h

@@ -53,8 +53,11 @@ public:
    * This compares the current instance to the std::string_view target: returns true if
    * they are byte-by-byte equal (no escaping is done) on target.size() characters,
    * and if the raw_json_string instance has a quote character at byte index target.size().
-   * We never read more than length + 1 bytes in the raw_json_string instance.
-   * If length is smaller than target.size(), this will return false.
+   * We never read more than max_key_length_including_final_quote bytes in the raw_json_string instance.
+   * If max_key_length_including_final_quote is smaller than target.size() + 1, this will return false.
+   *
+   * max_key_length_including_final_quote is the maximal key length in bytes, not including
+   * the leading quote.
    *
    * The std::string_view instance may contain any characters. However, the caller
    * is responsible for setting length so that length bytes may be read in the
@@ -63,7 +66,7 @@ public:
    * Performance: the comparison may be done using memcmp which may be efficient
    * for long strings.
    */
-  simdjson_really_inline bool unsafe_is_equal(size_t length, std::string_view target) const noexcept;
+  simdjson_really_inline bool unsafe_is_equal(size_t max_key_length_including_final_quote, std::string_view target) const noexcept;
 
   /**
    * This compares the current instance to the std::string_view target: returns true if
@@ -140,10 +143,13 @@ private:
    * @param dst A pointer to a buffer at least large enough to write this string as well as a \0.
    *            dst will be updated to the next unused location (just after the \0 written out at
    *            the end of this string).
+   * @param buf_end A pointer to the end of the input JSON you passed to iterate(). This is
+   *            used to prevent overruns, since simdjson normally reads (but does not use) past
+   *            the end quote when parsing a string for performance reasons.
    * @return A string_view pointing at the unescaped string in dst
    * @error STRING_ERROR if escapes are incorrect.
    */
-  simdjson_really_inline simdjson_warn_unused simdjson_result<std::string_view> unescape(uint8_t *&dst) const noexcept;
+  simdjson_really_inline simdjson_warn_unused simdjson_result<std::string_view> unescape(uint8_t *&dst, const uint8_t *buf_end) const noexcept;
   /**
    * Unescape this JSON string, replacing \\ with \, \n with newline, etc.
    *
@@ -188,7 +194,7 @@ public:
   simdjson_really_inline ~simdjson_result() noexcept = default; ///< @private
 
   simdjson_really_inline simdjson_result<const char *> raw() const noexcept;
-  simdjson_really_inline simdjson_warn_unused simdjson_result<std::string_view> unescape(uint8_t *&dst) const noexcept;
+  simdjson_really_inline simdjson_warn_unused simdjson_result<std::string_view> unescape(uint8_t *&dst, const uint8_t *buf_end) const noexcept;
   simdjson_really_inline simdjson_warn_unused simdjson_result<std::string_view> unescape(SIMDJSON_IMPLEMENTATION::ondemand::json_iterator &iter) const noexcept;
 };
 

include/simdjson/generic/ondemand/serialization-inl.h

@@ -20,13 +20,6 @@ inline simdjson_result<std::string_view> to_json_string(SIMDJSON_IMPLEMENTATION:
   return trim(v);
 }
 
-inline simdjson_result<std::string_view> to_json_string(SIMDJSON_IMPLEMENTATION::ondemand::document_reference& x) noexcept {
-  std::string_view v;
-  auto error = x.raw_json().get(v);
-  if(error) {return error; }
-  return trim(v);
-}
-
 inline simdjson_result<std::string_view> to_json_string(SIMDJSON_IMPLEMENTATION::ondemand::value& x) noexcept {
   /**
    * If we somehow receive a value that has already been consumed,
@@ -73,30 +66,28 @@ inline simdjson_result<std::string_view> to_json_string(SIMDJSON_IMPLEMENTATION:
   return trim(v);
 }
 
+#if SIMDJSON_EXCEPTIONS
+
 inline simdjson_result<std::string_view> to_json_string(simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document> x) {
   if (x.error()) { return x.error(); }
-  return to_json_string(x.value_unsafe());
-}
-
-inline simdjson_result<std::string_view> to_json_string(simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference> x) {
-  if (x.error()) { return x.error(); }
-  return to_json_string(x.value_unsafe());
+  return to_json_string(x.value());
 }
 
 inline simdjson_result<std::string_view> to_json_string(simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> x) {
   if (x.error()) { return x.error(); }
-  return to_json_string(x.value_unsafe());
+  return to_json_string(x.value());
 }
 
 inline simdjson_result<std::string_view> to_json_string(simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::object> x) {
   if (x.error()) { return x.error(); }
-  return to_json_string(x.value_unsafe());
+  return to_json_string(x.value());
 }
 
 inline simdjson_result<std::string_view> to_json_string(simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array> x) {
   if (x.error()) { return x.error(); }
-  return to_json_string(x.value_unsafe());
+  return to_json_string(x.value());
 }
+#endif
 } // namespace simdjson
 
 
@@ -162,20 +153,7 @@ inline std::ostream& operator<<(std::ostream& out, simdjson::SIMDJSON_IMPLEMENTA
     throw simdjson::simdjson_error(error);
   }
 }
-inline std::ostream& operator<<(std::ostream& out, simdjson::SIMDJSON_IMPLEMENTATION::ondemand::document_reference& value)  {
-  std::string_view v;
-  auto error = simdjson::to_json_string(value).get(v);
-  if(error == simdjson::SUCCESS) {
-    return (out << v);
-  } else {
-    throw simdjson::simdjson_error(error);
-  }
-}
-inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result<simdjson::SIMDJSON_IMPLEMENTATION::ondemand::document>&& x) {
-  if (x.error()) { throw simdjson::simdjson_error(x.error()); }
-  return (out << x.value());
-}
-inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result<simdjson::SIMDJSON_IMPLEMENTATION::ondemand::document_reference>&& x) {
+inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result<simdjson::SIMDJSON_IMPLEMENTATION::ondemand::document> x) {
   if (x.error()) { throw simdjson::simdjson_error(x.error()); }
   return (out << x.value());
 }

include/simdjson/generic/ondemand/serialization.h

@@ -23,10 +23,12 @@ inline simdjson_result<std::string_view> to_json_string(SIMDJSON_IMPLEMENTATION:
  * contains JSON text that is suitable to be parsed as JSON again.
  */
 inline simdjson_result<std::string_view> to_json_string(SIMDJSON_IMPLEMENTATION::ondemand::array& x) noexcept;
+#if SIMDJSON_EXCEPTIONS
 inline simdjson_result<std::string_view> to_json_string(simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document> x);
 inline simdjson_result<std::string_view> to_json_string(simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> x);
 inline simdjson_result<std::string_view> to_json_string(simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::object> x);
 inline simdjson_result<std::string_view> to_json_string(simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array> x);
+#endif
 } // namespace simdjson
 
 
@@ -61,11 +63,7 @@ inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result<sim
  */
 inline std::ostream& operator<<(std::ostream& out, simdjson::SIMDJSON_IMPLEMENTATION::ondemand::document& value);
 #if SIMDJSON_EXCEPTIONS
-inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result<simdjson::SIMDJSON_IMPLEMENTATION::ondemand::document>&& x);
-#endif
-inline std::ostream& operator<<(std::ostream& out, simdjson::SIMDJSON_IMPLEMENTATION::ondemand::document_reference& value);
-#if SIMDJSON_EXCEPTIONS
-inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result<simdjson::SIMDJSON_IMPLEMENTATION::ondemand::document_reference>&& x);
+inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result<simdjson::SIMDJSON_IMPLEMENTATION::ondemand::document> x);
 #endif
 /**
  * Print JSON to an output stream.

include/simdjson/generic/ondemand/value-inl.h

@@ -36,21 +36,12 @@ simdjson_really_inline simdjson_result<std::string_view> value::get_string() noe
 simdjson_really_inline simdjson_result<double> value::get_double() noexcept {
   return iter.get_double();
 }
-simdjson_really_inline simdjson_result<double> value::get_double_in_string() noexcept {
-  return iter.get_double_in_string();
-}
 simdjson_really_inline simdjson_result<uint64_t> value::get_uint64() noexcept {
   return iter.get_uint64();
 }
-simdjson_really_inline simdjson_result<uint64_t> value::get_uint64_in_string() noexcept {
-  return iter.get_uint64_in_string();
-}
 simdjson_really_inline simdjson_result<int64_t> value::get_int64() noexcept {
   return iter.get_int64();
 }
-simdjson_really_inline simdjson_result<int64_t> value::get_int64_in_string() noexcept {
-  return iter.get_int64_in_string();
-}
 simdjson_really_inline simdjson_result<bool> value::get_bool() noexcept {
   return iter.get_bool();
 }
@@ -114,17 +105,6 @@ simdjson_really_inline simdjson_result<size_t> value::count_elements() & noexcep
   iter.move_at_start();
   return answer;
 }
-simdjson_really_inline simdjson_result<size_t> value::count_fields() & noexcept {
-  simdjson_result<size_t> answer;
-  auto a = get_object();
-  answer = a.count_fields();
-  iter.move_at_start();
-  return answer;
-}
-simdjson_really_inline simdjson_result<value> value::at(size_t index) noexcept {
-  auto a = get_array();
-  return a.at(index);
-}
 
 simdjson_really_inline simdjson_result<value> value::find_field(std::string_view key) noexcept {
   return start_or_resume_object().find_field(key);
@@ -151,35 +131,10 @@ simdjson_really_inline simdjson_result<json_type> value::type() noexcept {
   return iter.type();
 }
 
-simdjson_really_inline simdjson_result<bool> value::is_scalar() noexcept {
-  json_type this_type;
-  auto error = type().get(this_type);
-  if(error) { return error; }
-  return ! ((this_type == json_type::array) || (this_type == json_type::object));
-}
-
-simdjson_really_inline bool value::is_negative() noexcept {
-  return iter.is_negative();
-}
-
-simdjson_really_inline simdjson_result<bool> value::is_integer() noexcept {
-  return iter.is_integer();
-}
-simdjson_warn_unused simdjson_really_inline simdjson_result<number_type> value::get_number_type() noexcept {
-  return iter.get_number_type();
-}
-simdjson_warn_unused simdjson_really_inline simdjson_result<number> value::get_number() noexcept {
-  return iter.get_number();
-}
-
 simdjson_really_inline std::string_view value::raw_json_token() noexcept {
   return std::string_view(reinterpret_cast<const char*>(iter.peek_start()), iter.peek_start_length());
 }
 
-simdjson_really_inline simdjson_result<const char *> value::current_location() noexcept {
-  return iter.json_iter().current_location();
-}
-
 simdjson_really_inline simdjson_result<value> value::at_pointer(std::string_view json_pointer) noexcept {
   json_type t;
   SIMDJSON_TRY(type().get(t));
@@ -218,14 +173,6 @@ simdjson_really_inline simdjson_result<size_t> simdjson_result<SIMDJSON_IMPLEMEN
   if (error()) { return error(); }
   return first.count_elements();
 }
-simdjson_really_inline simdjson_result<size_t> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::count_fields() & noexcept {
-  if (error()) { return error(); }
-  return first.count_fields();
-}
-simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::at(size_t index) noexcept {
-  if (error()) { return error(); }
-  return first.at(index);
-}
 simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::begin() & noexcept {
   if (error()) { return error(); }
   return first.begin();
@@ -274,26 +221,14 @@ simdjson_really_inline simdjson_result<uint64_t> simdjson_result<SIMDJSON_IMPLEM
   if (error()) { return error(); }
   return first.get_uint64();
 }
-simdjson_really_inline simdjson_result<uint64_t> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::get_uint64_in_string() noexcept {
-  if (error()) { return error(); }
-  return first.get_uint64_in_string();
-}
 simdjson_really_inline simdjson_result<int64_t> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::get_int64() noexcept {
   if (error()) { return error(); }
   return first.get_int64();
 }
-simdjson_really_inline simdjson_result<int64_t> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::get_int64_in_string() noexcept {
-  if (error()) { return error(); }
-  return first.get_int64_in_string();
-}
 simdjson_really_inline simdjson_result<double> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::get_double() noexcept {
   if (error()) { return error(); }
   return first.get_double();
 }
-simdjson_really_inline simdjson_result<double> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::get_double_in_string() noexcept {
-  if (error()) { return error(); }
-  return first.get_double_in_string();
-}
 simdjson_really_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::get_string() noexcept {
   if (error()) { return error(); }
   return first.get_string();
@@ -334,26 +269,7 @@ simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::json_t
   if (error()) { return error(); }
   return first.type();
 }
-simdjson_really_inline simdjson_result<bool> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::is_scalar() noexcept {
-  if (error()) { return error(); }
-  return first.is_scalar();
-}
-simdjson_really_inline simdjson_result<bool> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::is_negative() noexcept {
-  if (error()) { return error(); }
-  return first.is_negative();
-}
-simdjson_really_inline simdjson_result<bool> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::is_integer() noexcept {
-  if (error()) { return error(); }
-  return first.is_integer();
-}
-simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number_type> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::get_number_type() noexcept {
-  if (error()) { return error(); }
-  return first.get_number_type();
-}
-simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::get_number() noexcept {
-  if (error()) { return error(); }
-  return first.get_number();
-}
+
 #if SIMDJSON_EXCEPTIONS
 simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::operator SIMDJSON_IMPLEMENTATION::ondemand::array() noexcept(false) {
   if (error()) { throw simdjson_error(error()); }
@@ -394,11 +310,6 @@ simdjson_really_inline simdjson_result<std::string_view> simdjson_result<SIMDJSO
   return first.raw_json_token();
 }
 
-simdjson_really_inline simdjson_result<const char *> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::current_location() noexcept {
-  if (error()) { return error(); }
-  return first.current_location();
-}
-
 simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::at_pointer(std::string_view json_pointer) noexcept {
   if (error()) { return error(); }
   return first.at_pointer(json_pointer);

include/simdjson/generic/ondemand/value.h

@@ -69,19 +69,11 @@ public:
   /**
    * Cast this JSON value to an unsigned integer.
    *
-   * @returns A unsigned 64-bit integer.
+   * @returns A signed 64-bit integer.
    * @returns INCORRECT_TYPE If the JSON value is not a 64-bit unsigned integer.
    */
   simdjson_really_inline simdjson_result<uint64_t> get_uint64() noexcept;
 
-  /**
-   * Cast this JSON value (inside string) to a unsigned integer.
-   *
-   * @returns A unsigned 64-bit integer.
-   * @returns INCORRECT_TYPE If the JSON value is not a 64-bit unsigned integer.
-   */
-  simdjson_really_inline simdjson_result<uint64_t> get_uint64_in_string() noexcept;
-
   /**
    * Cast this JSON value to a signed integer.
    *
@@ -90,14 +82,6 @@ public:
    */
   simdjson_really_inline simdjson_result<int64_t> get_int64() noexcept;
 
-  /**
-   * Cast this JSON value (inside string) to a signed integer.
-   *
-   * @returns A signed 64-bit integer.
-   * @returns INCORRECT_TYPE If the JSON value is not a 64-bit integer.
-   */
-  simdjson_really_inline simdjson_result<int64_t> get_int64_in_string() noexcept;
-
   /**
    * Cast this JSON value to a double.
    *
@@ -106,14 +90,6 @@ public:
    */
   simdjson_really_inline simdjson_result<double> get_double() noexcept;
 
-  /**
-   * Cast this JSON value (inside string) to a double
-   *
-   * @returns A double.
-   * @returns INCORRECT_TYPE If the JSON value is not a valid floating-point number.
-   */
-  simdjson_really_inline simdjson_result<double> get_double_in_string() noexcept;
-
   /**
    * Cast this JSON value to a string.
    *
@@ -244,29 +220,6 @@ public:
    * safe to continue.
    */
   simdjson_really_inline simdjson_result<size_t> count_elements() & noexcept;
-  /**
-   * This method scans the object and counts the number of key-value pairs.
-   * The count_fields method should always be called before you have begun
-   * iterating through the object: it is expected that you are pointing at
-   * the beginning of the object.
-   * The runtime complexity is linear in the size of the object. After
-   * calling this function, if successful, the object is 'rewinded' at its
-   * beginning as if it had never been accessed. If the JSON is malformed (e.g.,
-   * there is a missing comma), then an error is returned and it is no longer
-   * safe to continue.
-   *
-   * To check that an object is empty, it is more performant to use
-   * the is_empty() method on the object instance.
-   */
-  simdjson_really_inline simdjson_result<size_t> count_fields() & noexcept;
-  /**
-   * Get the value at the given index in the array. This function has linear-time complexity.
-   * This function should only be called once as the array iterator is not reset between each call.
-   *
-   * @return The value at the given index, or:
-   *         - INDEX_OUT_OF_BOUNDS if the array index is larger than an array length
-   */
-  simdjson_really_inline simdjson_result<value> at(size_t index) noexcept;
   /**
    * Look up a field by name on an object (order-sensitive).
    *
@@ -336,94 +289,15 @@ public:
    */
   simdjson_really_inline simdjson_result<json_type> type() noexcept;
 
-  /**
-   * Checks whether the value is a scalar (string, number, null, Boolean).
-   * Returns false when there it is an array or object.
-   *
-   * @returns true if the type is string, number, null, Boolean
-   * @error TAPE_ERROR when the JSON value is a bad token like "}" "," or "alse".
-   */
-  simdjson_really_inline simdjson_result<bool> is_scalar() noexcept;
-
-  /**
-   * Checks whether the value is a negative number.
-   *
-   * @returns true if the number if negative.
-   */
-  simdjson_really_inline bool is_negative() noexcept;
-  /**
-   * Checks whether the value is an integer number. Note that
-   * this requires to partially parse the number string. If
-   * the value is determined to be an integer, it may still
-   * not parse properly as an integer in subsequent steps
-   * (e.g., it might overflow).
-   *
-   * Performance note: if you call this function systematically
-   * before parsing a number, you may have fallen for a performance
-   * anti-pattern.
-   *
-   * @returns true if the number if negative.
-   */
-  simdjson_really_inline simdjson_result<bool> is_integer() noexcept;
-  /**
-   * Determine the number type (integer or floating-point number).
-   *
-   * get_number_type() is number_type::unsigned_integer if we have
-   * an integer greater or equal to 9223372036854775808
-   * get_number_type() is number_type::signed_integer if we have an
-   * integer that is less than 9223372036854775808
-   * Otherwise, get_number_type() has value number_type::floating_point_number
-   *
-   * This function requires processing the number string, but it is expected
-   * to be faster than get_number().get_number_type() because it is does not
-   * parse the number value.
-   *
-   * @returns the type of the number
-   */
-  simdjson_really_inline simdjson_result<number_type> get_number_type() noexcept;
-
-  /**
-   * Attempt to parse an ondemand::number. An ondemand::number may
-   * contain an integer value or a floating-point value, the simdjson
-   * library will autodetect the type. Thus it is a dynamically typed
-   * number. Before accessing the value, you must determine the detected
-   * type.
-   *
-   * number.get_number_type() is number_type::signed_integer if we have
-   * an integer in [-9223372036854775808,9223372036854775808)
-   * You can recover the value by calling number.get_int64() and you
-   * have that number.is_int64() is true.
-   *
-   * number.get_number_type() is number_type::unsigned_integer if we have
-   * an integer in [9223372036854775808,18446744073709551616)
-   * You can recover the value by calling number.get_uint64() and you
-   * have that number.is_uint64() is true.
-   *
-   * Otherwise, number.get_number_type() has value number_type::floating_point_number
-   * and we have a binary64 number.
-   * You can recover the value by calling number.get_double() and you
-   * have that number.is_double() is true.
-   *
-   * You must check the type before accessing the value: it is an error
-   * to call "get_int64()" when number.get_number_type() is not
-   * number_type::signed_integer and when number.is_int64() is false.
-   *
-   * Performance note: this is designed with performance in mind. When
-   * calling 'get_number()', you scan the number string only once, determining
-   * efficiently the type and storing it in an efficient manner.
-   */
-  simdjson_warn_unused simdjson_really_inline simdjson_result<number> get_number() noexcept;
-
-
   /**
    * Get the raw JSON for this token.
    *
    * The string_view will always point into the input buffer.
    *
    * The string_view will start at the beginning of the token, and include the entire token
-   * *as well as all spaces until the next token (or EOF).* This means, for example, that a
-   * string token always begins with a " and is always terminated by the final ", possibly
-   * followed by a number of spaces.
+   * *as well as all spaces until the next token (or end of the buffer, or EOF).*
+   * This means, for example, that a string token always begins with a " and is always
+   * terminated by the final ", possibly followed by a number of spaces.
    *
    * The string_view is *not* null-terminated. However, if this is a scalar (string, number,
    * boolean, or null), the character after the end of the string_view is guaranteed to be
@@ -440,11 +314,6 @@ public:
    */
   simdjson_really_inline std::string_view raw_json_token() noexcept;
 
-  /**
-   * Returns the current location in the document if in bounds.
-   */
-  simdjson_really_inline simdjson_result<const char *> current_location() noexcept;
-
   /**
    * Get the value associated with the given JSON pointer.  We use the RFC 6901
    * https://tools.ietf.org/html/rfc6901 standard.
@@ -547,11 +416,8 @@ public:
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::object> get_object() noexcept;
 
   simdjson_really_inline simdjson_result<uint64_t> get_uint64() noexcept;
-  simdjson_really_inline simdjson_result<uint64_t> get_uint64_in_string() noexcept;
   simdjson_really_inline simdjson_result<int64_t> get_int64() noexcept;
-  simdjson_really_inline simdjson_result<int64_t> get_int64_in_string() noexcept;
   simdjson_really_inline simdjson_result<double> get_double() noexcept;
-  simdjson_really_inline simdjson_result<double> get_double_in_string() noexcept;
   simdjson_really_inline simdjson_result<std::string_view> get_string() noexcept;
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::raw_json_string> get_raw_json_string() noexcept;
   simdjson_really_inline simdjson_result<bool> get_bool() noexcept;
@@ -572,8 +438,6 @@ public:
   simdjson_really_inline operator bool() noexcept(false);
 #endif
   simdjson_really_inline simdjson_result<size_t> count_elements() & noexcept;
-  simdjson_really_inline simdjson_result<size_t> count_fields() & noexcept;
-  simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> at(size_t index) noexcept;
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> begin() & noexcept;
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> end() & noexcept;
 
@@ -636,18 +500,10 @@ public:
    * let it throw an exception).
    */
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::json_type> type() noexcept;
-  simdjson_really_inline simdjson_result<bool> is_scalar() noexcept;
-  simdjson_really_inline simdjson_result<bool> is_negative() noexcept;
-  simdjson_really_inline simdjson_result<bool> is_integer() noexcept;
-  simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number_type> get_number_type() noexcept;
-  simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number> get_number() noexcept;
 
   /** @copydoc simdjson_really_inline std::string_view value::raw_json_token() const noexcept */
   simdjson_really_inline simdjson_result<std::string_view> raw_json_token() noexcept;
 
-  /** @copydoc simdjson_really_inline simdjson_result<const char *> current_location() noexcept */
-  simdjson_really_inline simdjson_result<const char *> current_location() noexcept;
-
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> at_pointer(std::string_view json_pointer) noexcept;
 };
 

include/simdjson/generic/ondemand/value_iterator-inl.h

@@ -25,12 +25,15 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator
 #ifdef SIMDJSON_DEVELOPMENT_CHECKS
   _json_iter->set_start_position(_depth, start_position());
 #endif
+  SIMDJSON_TRY( _json_iter->require_tokens(1) );
   if (*_json_iter->peek() == '}') {
     logger::log_value(*_json_iter, "empty object");
     _json_iter->return_current_and_advance();
     end_container();
     return false;
   }
+  SIMDJSON_TRY( _json_iter->require_tokens(3) ); // Make sure we have three tokens: "x" : value
+  logger::log_start_value(*_json_iter, "object");
   return true;
 }
 
@@ -40,7 +43,6 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator
   // Note that adding a check for 'streaming' is not expensive since we only have at most
   // one root element.
   if (! _json_iter->streaming() && (*_json_iter->peek_last() != '}')) {
-    _json_iter->abandon();
     return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "missing } at end");
   }
   return started_object();
@@ -48,8 +50,8 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator
 
 simdjson_warn_unused simdjson_really_inline error_code value_iterator::end_container() noexcept {
 #if SIMDJSON_CHECK_EOF
-    if (depth() > 1 && at_end()) { return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "missing parent ] or }"); }
-    // if (depth() <= 1 && !at_end()) { return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "missing [ or { at start"); }
+    if (depth() > 1 && at_end_of_input_buffer()) { return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "missing parent ] or }"); }
+    // if (depth() <= 1 && !at_end_of_input_buffer()) { return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "missing [ or { at start"); }
 #endif // SIMDJSON_CHECK_EOF
     _json_iter->ascend_to(depth()-1);
     return SUCCESS;
@@ -58,6 +60,8 @@ simdjson_warn_unused simdjson_really_inline error_code value_iterator::end_conta
 simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator::has_next_field() noexcept {
   assert_at_next();
 
+  SIMDJSON_TRY( error_unless_more_tokens() );
+
   // It's illegal to call this unless there are more tokens: anything that ends in } or ] is
   // obligated to verify there are more tokens if they are not the top level.
   switch (*_json_iter->return_current_and_advance()) {
@@ -66,6 +70,7 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator
       SIMDJSON_TRY( end_container() );
       return false;
     case ',':
+      SIMDJSON_TRY( error_unless_more_tokens(3) ); // Make sure we have three tokens: "x" : value
       return true;
     default:
       return report_error(TAPE_ERROR, "Missing comma between object fields");
@@ -130,8 +135,9 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator
   while (has_value) {
     // Get the key and colon, stopping at the value.
     raw_json_string actual_key;
-    // size_t max_key_length = _json_iter->peek_length() - 2; // -2 for the two quotes
-    // Note: _json_iter->peek_length() - 2 might overflow if _json_iter->peek_length() < 2.
+    // We know, for sure, that _json_iter->peek_length() is at least '1', but it could
+    // be just one, so subtracting by more than 1 is unsafe.
+    size_t max_key_length_including_final_quote = _json_iter->peek_length() - 1; // -1 for one quote
     // field_key() advances the pointer and checks that '"' is found (corresponding to a key).
     // The depth is left unchanged by field_key().
     if ((error = field_key().get(actual_key) )) { abandon(); return error; };
@@ -139,14 +145,7 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator
     // key and the value. It will also increment the depth by one.
     if ((error = field_value() )) { abandon(); return error; }
     // If it matches, stop and return
-    // We could do it this way if we wanted to allow arbitrary
-    // key content (including escaped quotes).
-    //if (actual_key.unsafe_is_equal(max_key_length, key)) {
-    // Instead we do the following which may trigger buffer overruns if the
-    // user provides an adversarial key (containing a well placed unescaped quote
-    // character and being longer than the number of bytes remaining in the JSON
-    // input).
-    if (actual_key.unsafe_is_equal(key)) {
+    if (actual_key.unsafe_is_equal(max_key_length_including_final_quote, key)) {
       logger::log_event(*this, "match", key, -2);
       // If we return here, then we return while pointing at the ':' that we just checked.
       return true;
@@ -261,8 +260,9 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator
 
     // Get the key and colon, stopping at the value.
     raw_json_string actual_key;
-    // size_t max_key_length = _json_iter->peek_length() - 2; // -2 for the two quotes
-    // Note: _json_iter->peek_length() - 2 might overflow if _json_iter->peek_length() < 2.
+    // We know, for sure, that _json_iter->peek_length() is at least '1', but it could
+    // be just one, so subtracting by more than 1 is unsafe.
+    size_t max_key_length_including_final_quote = _json_iter->peek_length() - 1; // -1 for one quote
     // field_key() advances the pointer and checks that '"' is found (corresponding to a key).
     // The depth is left unchanged by field_key().
     if ((error = field_key().get(actual_key) )) { abandon(); return error; };
@@ -271,14 +271,7 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator
     if ((error = field_value() )) { abandon(); return error; }
 
     // If it matches, stop and return
-    // We could do it this way if we wanted to allow arbitrary
-    // key content (including escaped quotes).
-    // if (actual_key.unsafe_is_equal(max_key_length, key)) {
-    // Instead we do the following which may trigger buffer overruns if the
-    // user provides an adversarial key (containing a well placed unescaped quote
-    // character and being longer than the number of bytes remaining in the JSON
-    // input).
-    if (actual_key.unsafe_is_equal(key)) {
+    if (actual_key.unsafe_is_equal(max_key_length_including_final_quote, key)) {
       logger::log_event(*this, "match", key, -2);
       // If we return here, then we return while pointing at the ':' that we just checked.
       return true;
@@ -311,8 +304,9 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator
 
     // Get the key and colon, stopping at the value.
     raw_json_string actual_key;
-    // size_t max_key_length = _json_iter->peek_length() - 2; // -2 for the two quotes
-    // Note: _json_iter->peek_length() - 2 might overflow if _json_iter->peek_length() < 2.
+    // We know, for sure, that _json_iter->peek_length() is at least '1', but it could
+    // be just one, so subtracting by more than 1 is unsafe.
+    size_t max_key_length_including_final_quote = _json_iter->peek_length() - 1; // -1 for one quote
     // field_key() advances the pointer and checks that '"' is found (corresponding to a key).
     // The depth is left unchanged by field_key().
     error = field_key().get(actual_key); SIMDJSON_ASSUME(!error);
@@ -321,14 +315,7 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator
     error = field_value(); SIMDJSON_ASSUME(!error);
 
     // If it matches, stop and return
-    // We could do it this way if we wanted to allow arbitrary
-    // key content (including escaped quotes).
-    // if (actual_key.unsafe_is_equal(max_key_length, key)) {
-    // Instead we do the following which may trigger buffer overruns if the
-    // user provides an adversarial key (containing a well placed unescaped quote
-    // character and being longer than the number of bytes remaining in the JSON
-    // input).
-    if (actual_key.unsafe_is_equal(key)) {
+    if (actual_key.unsafe_is_equal(max_key_length_including_final_quote, key)) {
       logger::log_event(*this, "match", key, -2);
       // If we return here, then we return while pointing at the ':' that we just checked.
       return true;
@@ -358,6 +345,7 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator
 simdjson_warn_unused simdjson_really_inline simdjson_result<raw_json_string> value_iterator::field_key() noexcept {
   assert_at_next();
 
+  // started_object() and has_next_field() already checked that we have a key
   const uint8_t *key = _json_iter->return_current_and_advance();
   if (*(key++) != '"') { return report_error(TAPE_ERROR, "Object key is not a string"); }
   return raw_json_string(key);
@@ -366,6 +354,7 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<raw_json_string> val
 simdjson_warn_unused simdjson_really_inline error_code value_iterator::field_value() noexcept {
   assert_at_next();
 
+  // started_object() and has_next_field() already checked that we have a : and a value token
   if (*_json_iter->return_current_and_advance() != ':') { return report_error(TAPE_ERROR, "Missing colon in object field"); }
   _json_iter->descend_to(depth()+1);
   return SUCCESS;
@@ -390,6 +379,8 @@ inline std::string value_iterator::to_string() const noexcept {
 
 simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator::started_array() noexcept {
   assert_at_container_start();
+  SIMDJSON_TRY( _json_iter->require_tokens(1) );
+
   if (*_json_iter->peek() == ']') {
     logger::log_value(*_json_iter, "empty array");
     _json_iter->return_current_and_advance();
@@ -409,7 +400,6 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator
   // Note that adding a check for 'streaming' is not expensive since we only have at most
   // one root element.
   if ( ! _json_iter->streaming() && (*_json_iter->peek_last() != ']')) {
-    _json_iter->abandon();
     return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "missing ] at end");
   }
   return started_array();
@@ -418,13 +408,16 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator
 simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator::has_next_element() noexcept {
   assert_at_next();
 
+  const uint8_t *json;
   logger::log_event(*this, "has_next_element");
-  switch (*_json_iter->return_current_and_advance()) {
+  SIMDJSON_TRY( _json_iter->try_return_current_and_advance().get(json) )
+  switch (*json) {
     case ']':
       logger::log_end_value(*_json_iter, "array");
       SIMDJSON_TRY( end_container() );
       return false;
     case ',':
+      SIMDJSON_TRY( _json_iter->require_tokens(1) );
       _json_iter->descend_to(depth()+1);
       return true;
     default:
@@ -432,19 +425,8 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator
   }
 }
 
-simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator::parse_bool(const uint8_t *json) const noexcept {
-  auto not_true = atomparsing::str4ncmp(json, "true");
-  auto not_false = atomparsing::str4ncmp(json, "fals") | (json[4] ^ 'e');
-  bool error = (not_true && not_false) || jsoncharutils::is_not_structural_or_whitespace(json[not_true ? 5 : 4]);
-  if (error) { return incorrect_type_error("Not a boolean"); }
-  return simdjson_result<bool>(!not_true);
-}
-simdjson_really_inline bool value_iterator::parse_null(const uint8_t *json) const noexcept {
-  return !atomparsing::str4ncmp(json, "null") && jsoncharutils::is_structural_or_whitespace(json[4]);
-}
-
 simdjson_warn_unused simdjson_really_inline simdjson_result<std::string_view> value_iterator::get_string() noexcept {
-  return get_raw_json_string().unescape(_json_iter->string_buf_loc());
+  return get_raw_json_string().unescape(*_json_iter);
 }
 simdjson_warn_unused simdjson_really_inline simdjson_result<raw_json_string> value_iterator::get_raw_json_string() noexcept {
   auto json = peek_scalar("string");
@@ -454,103 +436,57 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<raw_json_string> val
 }
 simdjson_warn_unused simdjson_really_inline simdjson_result<uint64_t> value_iterator::get_uint64() noexcept {
   auto result = numberparsing::parse_unsigned(peek_non_root_scalar("uint64"));
-  if(result.error() == SUCCESS) { advance_non_root_scalar("uint64"); }
-  return result;
-}
-simdjson_warn_unused simdjson_really_inline simdjson_result<uint64_t> value_iterator::get_uint64_in_string() noexcept {
-  auto result = numberparsing::parse_unsigned_in_string(peek_non_root_scalar("uint64"));
-  if(result.error() == SUCCESS) { advance_non_root_scalar("uint64"); }
+  if(result.error() != INCORRECT_TYPE) { advance_non_root_scalar("uint64"); }
   return result;
 }
 simdjson_warn_unused simdjson_really_inline simdjson_result<int64_t> value_iterator::get_int64() noexcept {
   auto result = numberparsing::parse_integer(peek_non_root_scalar("int64"));
-  if(result.error() == SUCCESS) { advance_non_root_scalar("int64"); }
-  return result;
-}
-simdjson_warn_unused simdjson_really_inline simdjson_result<int64_t> value_iterator::get_int64_in_string() noexcept {
-  auto result = numberparsing::parse_integer_in_string(peek_non_root_scalar("int64"));
-  if(result.error() == SUCCESS) { advance_non_root_scalar("int64"); }
+  if(result.error() != INCORRECT_TYPE) { advance_non_root_scalar("int64"); }
   return result;
 }
 simdjson_warn_unused simdjson_really_inline simdjson_result<double> value_iterator::get_double() noexcept {
   auto result = numberparsing::parse_double(peek_non_root_scalar("double"));
-  if(result.error() == SUCCESS) { advance_non_root_scalar("double"); }
-  return result;
-}
-simdjson_warn_unused simdjson_really_inline simdjson_result<double> value_iterator::get_double_in_string() noexcept {
-  auto result = numberparsing::parse_double_in_string(peek_non_root_scalar("double"));
-  if(result.error() == SUCCESS) { advance_non_root_scalar("double"); }
+  if(result.error() != INCORRECT_TYPE) { advance_non_root_scalar("double"); }
   return result;
 }
 simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator::get_bool() noexcept {
-  auto result = parse_bool(peek_non_root_scalar("bool"));
-  if(result.error() == SUCCESS) { advance_non_root_scalar("bool"); }
+  auto result = parse_bool(peek_start_length(), peek_non_root_scalar("bool"));
+  if(result.error() != INCORRECT_TYPE) { advance_non_root_scalar("bool"); }
   return result;
 }
+
+simdjson_really_inline simdjson_result<bool> value_iterator::parse_bool(uint32_t max_len, const uint8_t *json) const noexcept {
+  if (max_len >= 4) {
+    if (!atomparsing::str4ncmp_unsafe(json, "true")) {
+      if (max_len == 4 || jsoncharutils::is_structural_or_whitespace(json[4])) {
+        return simdjson_result<bool>(true);
+      }
+    } else if (!atomparsing::str4ncmp_unsafe(json, "fals") && max_len >= 5 && json[4] == 'e') {
+      if (max_len == 5 || jsoncharutils::is_structural_or_whitespace(json[5])) {
+        return simdjson_result<bool>(false);
+      }
+    }
+  }
+  return incorrect_type_error("Not a boolean");
+}
 simdjson_really_inline bool value_iterator::is_null() noexcept {
-  auto result = parse_null(peek_non_root_scalar("null"));
+  auto result = parse_null(peek_start_length(), peek_non_root_scalar("null"));
   if(result) { advance_non_root_scalar("null"); }
   return result;
 }
-simdjson_really_inline bool value_iterator::is_negative() noexcept {
-  return numberparsing::is_negative(peek_non_root_scalar("numbersign"));
-}
-simdjson_really_inline bool value_iterator::is_root_negative() noexcept {
-  return numberparsing::is_negative(peek_root_scalar("numbersign"));
-}
-simdjson_really_inline simdjson_result<bool> value_iterator::is_integer() noexcept {
-  return numberparsing::is_integer(peek_non_root_scalar("integer"));
-}
-simdjson_really_inline simdjson_result<number_type> value_iterator::get_number_type() noexcept {
-  return numberparsing::get_number_type(peek_non_root_scalar("integer"));
-}
-simdjson_really_inline simdjson_result<number> value_iterator::get_number() noexcept {
-  number num;
-  error_code error =  numberparsing::parse_number(peek_non_root_scalar("number"), num);
-  if(error) { return error; }
-  return num;
+
+simdjson_really_inline bool value_iterator::parse_null(uint32_t max_len, const uint8_t *json) const noexcept {
+  if (max_len >= 4) {
+    if (!atomparsing::str4ncmp_unsafe(json, "null")) {
+      if (max_len == 4 || jsoncharutils::is_structural_or_whitespace(json[4])) {
+        return true;
+      }
+    };
+  }
+  return false;
 }
 
-simdjson_really_inline simdjson_result<bool> value_iterator::is_root_integer() noexcept {
-  auto max_len = peek_start_length();
-  auto json = peek_root_scalar("is_root_integer");
-  uint8_t tmpbuf[20+1]; // <20 digits> is the longest possible unsigned integer
-  if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf)) {
-    return false; // if there are more than 20 characters, it cannot be represented as an integer.
-  }
-  return numberparsing::is_integer(tmpbuf);
-}
-
-simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number_type> value_iterator::get_root_number_type() noexcept {
-  auto max_len = peek_start_length();
-  auto json = peek_root_scalar("number");
-  // Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/,
-  // 1074 is the maximum number of significant fractional digits. Add 8 more digits for the biggest
-  // number: -0.<fraction>e-308.
-  uint8_t tmpbuf[1074+8+1];
-  if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf)) {
-    logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters");
-    return NUMBER_ERROR;
-  }
-  return numberparsing::get_number_type(tmpbuf);
-}
-simdjson_really_inline simdjson_result<number> value_iterator::get_root_number() noexcept {
-  auto max_len = peek_start_length();
-  auto json = peek_root_scalar("number");
-  // Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/,
-  // 1074 is the maximum number of significant fractional digits. Add 8 more digits for the biggest
-  // number: -0.<fraction>e-308.
-  uint8_t tmpbuf[1074+8+1];
-  if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf)) {
-    logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters");
-    return NUMBER_ERROR;
-  }
-  number num;
-  error_code error =  numberparsing::parse_number(tmpbuf, num);
-  if(error) { return error; }
-  advance_root_scalar("number");
-  return num;
-}
+constexpr const uint32_t MAX_INT_LENGTH = 1024;
 
 simdjson_warn_unused simdjson_really_inline simdjson_result<std::string_view> value_iterator::get_root_string() noexcept {
   return get_string();
@@ -561,98 +497,31 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<raw_json_string> val
 simdjson_warn_unused simdjson_really_inline simdjson_result<uint64_t> value_iterator::get_root_uint64() noexcept {
   auto max_len = peek_start_length();
   auto json = peek_root_scalar("uint64");
-  uint8_t tmpbuf[20+1]; // <20 digits> is the longest possible unsigned integer
-  if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf)) {
-    logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 20 characters");
-    return NUMBER_ERROR;
-  }
-  auto result = numberparsing::parse_unsigned(tmpbuf);
-  if(result.error() == SUCCESS) { advance_root_scalar("uint64"); }
-  return result;
-}
-simdjson_warn_unused simdjson_really_inline simdjson_result<uint64_t> value_iterator::get_root_uint64_in_string() noexcept {
-  auto max_len = peek_start_length();
-  auto json = peek_root_scalar("uint64");
-  uint8_t tmpbuf[20+1]; // <20 digits> is the longest possible unsigned integer
-  if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf)) {
-    logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 20 characters");
-    return NUMBER_ERROR;
-  }
-  auto result = numberparsing::parse_unsigned_in_string(tmpbuf);
-  if(result.error() == SUCCESS) { advance_root_scalar("uint64"); }
+  auto result = numberparsing::parse_unsigned(json, json+max_len);
+  if(result.error() != INCORRECT_TYPE) { advance_root_scalar("uint64"); }
   return result;
 }
 simdjson_warn_unused simdjson_really_inline simdjson_result<int64_t> value_iterator::get_root_int64() noexcept {
   auto max_len = peek_start_length();
   auto json = peek_root_scalar("int64");
-  uint8_t tmpbuf[20+1]; // -<19 digits> is the longest possible integer
-  if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf)) {
-    logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 20 characters");
-    return NUMBER_ERROR;
-  }
-
-  auto result = numberparsing::parse_integer(tmpbuf);
-  if(result.error() == SUCCESS) { advance_root_scalar("int64"); }
-  return result;
-}
-simdjson_warn_unused simdjson_really_inline simdjson_result<int64_t> value_iterator::get_root_int64_in_string() noexcept {
-  auto max_len = peek_start_length();
-  auto json = peek_root_scalar("int64");
-  uint8_t tmpbuf[20+1]; // -<19 digits> is the longest possible integer
-  if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf)) {
-    logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 20 characters");
-    return NUMBER_ERROR;
-  }
-
-  auto result = numberparsing::parse_integer_in_string(tmpbuf);
-  if(result.error() == SUCCESS) { advance_root_scalar("int64"); }
+  auto result = numberparsing::parse_integer(json, json+max_len);
+  if(result.error() != INCORRECT_TYPE) { advance_root_scalar("int64"); }
   return result;
 }
 simdjson_warn_unused simdjson_really_inline simdjson_result<double> value_iterator::get_root_double() noexcept {
   auto max_len = peek_start_length();
   auto json = peek_root_scalar("double");
-  // Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/,
-  // 1074 is the maximum number of significant fractional digits. Add 8 more digits for the biggest
-  // number: -0.<fraction>e-308.
-  uint8_t tmpbuf[1074+8+1];
-  if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf)) {
-    logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters");
-    return NUMBER_ERROR;
-  }
-  auto result = numberparsing::parse_double(tmpbuf);
-  if(result.error() == SUCCESS) { advance_root_scalar("double"); }
-  return result;
-}
-
-simdjson_warn_unused simdjson_really_inline simdjson_result<double> value_iterator::get_root_double_in_string() noexcept {
-  auto max_len = peek_start_length();
-  auto json = peek_root_scalar("double");
-  // Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/,
-  // 1074 is the maximum number of significant fractional digits. Add 8 more digits for the biggest
-  // number: -0.<fraction>e-308.
-  uint8_t tmpbuf[1074+8+1];
-  if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf)) {
-    logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters");
-    return NUMBER_ERROR;
-  }
-  auto result = numberparsing::parse_double_in_string(tmpbuf);
-  if(result.error() == SUCCESS) { advance_root_scalar("double"); }
+  auto result = numberparsing::parse_double(json, json+max_len);
+  if(result.error() != INCORRECT_TYPE) { advance_root_scalar("double"); }
   return result;
 }
 simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator::get_root_bool() noexcept {
-  auto max_len = peek_start_length();
-  auto json = peek_root_scalar("bool");
-  uint8_t tmpbuf[5+1];
-  if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf)) { return incorrect_type_error("Not a boolean"); }
-  auto result = parse_bool(tmpbuf);
-  if(result.error() == SUCCESS) { advance_root_scalar("bool"); }
+  auto result = parse_bool(peek_start_length(), peek_root_scalar("bool"));
+  if(result.error() != INCORRECT_TYPE) { advance_root_scalar("bool"); }
   return result;
 }
 simdjson_really_inline bool value_iterator::is_root_null() noexcept {
-  auto max_len = peek_start_length();
-  auto json = peek_root_scalar("null");
-  bool result = (max_len >= 4 && !atomparsing::str4ncmp(json, "null") &&
-         (max_len == 4 || jsoncharutils::is_structural_or_whitespace(json[5])));
+  auto result = parse_null(peek_start_length(), peek_root_scalar("null"));
   if(result) { advance_root_scalar("null"); }
   return result;
 }
@@ -679,8 +548,8 @@ simdjson_really_inline bool value_iterator::is_open() const noexcept {
 }
 SIMDJSON_POP_DISABLE_WARNINGS
 
-simdjson_really_inline bool value_iterator::at_end() const noexcept {
-  return _json_iter->at_end();
+simdjson_really_inline bool value_iterator::at_end_of_input_buffer() const noexcept {
+  return _json_iter->at_end_of_input_buffer();
 }
 
 simdjson_really_inline bool value_iterator::at_start() const noexcept {
@@ -713,10 +582,12 @@ simdjson_warn_unused simdjson_really_inline json_iterator &value_iterator::json_
 }
 
 simdjson_really_inline const uint8_t *value_iterator::peek_start() const noexcept {
-  return _json_iter->peek(start_position());
+  auto result = _json_iter->peek(start_position());
+  return result;
 }
 simdjson_really_inline uint32_t value_iterator::peek_start_length() const noexcept {
-  return _json_iter->peek_length(start_position());
+  auto result = _json_iter->peek_length(start_position());
+  return result;
 }
 
 simdjson_really_inline const uint8_t *value_iterator::peek_scalar(const char *type) noexcept {
@@ -804,15 +675,29 @@ simdjson_really_inline error_code value_iterator::incorrect_type_error(const cha
   return INCORRECT_TYPE;
 }
 
+simdjson_really_inline error_code value_iterator::error_unless_more_tokens(uint32_t tokens) const noexcept {
+  if ((position() + tokens) > end_of_input_buffer_position()) {
+    return _json_iter->report_error(TAPE_ERROR, "Document ended early");
+  }
+  return SUCCESS;
+}
+
 simdjson_really_inline bool value_iterator::is_at_start() const noexcept {
   return position() == start_position();
 }
 
-simdjson_really_inline bool value_iterator::is_at_key() const noexcept {
+simdjson_really_inline simdjson_result<bool> value_iterator::is_at_key() const noexcept {
   // Keys are at the same depth as the object.
   // Note here that we could be safer and check that we are within an object,
   // but we do not.
-  return _depth == _json_iter->_depth && *_json_iter->peek() == '"';
+  if(_depth == _json_iter->_depth) {
+    // If it is at a key, we need 3 more tokens, but if it is not at a key, then a single
+    // token would suffice (i.e., '}').
+    SIMDJSON_TRY( _json_iter->require_tokens(1) );
+
+    return *_json_iter->peek() == '"';
+  }
+  return false;
 }
 
 simdjson_really_inline bool value_iterator::is_at_iterator_start() const noexcept {
@@ -912,8 +797,8 @@ simdjson_really_inline token_position value_iterator::position() const noexcept
   return _json_iter->position();
 }
 
-simdjson_really_inline token_position value_iterator::end_position() const noexcept {
-  return _json_iter->end_position();
+simdjson_really_inline token_position value_iterator::end_of_input_buffer_position() const noexcept {
+  return _json_iter->end_of_input_buffer_position();
 }
 
 simdjson_really_inline token_position value_iterator::last_position() const noexcept {

include/simdjson/generic/ondemand/value_iterator.h

@@ -44,9 +44,9 @@ public:
   simdjson_warn_unused simdjson_really_inline error_code skip_child() noexcept;
 
   /**
-   * Tell whether the iterator is at the EOF mark
+   * Tell whether the iterator is at the EOF mark (end of input buffer)
    */
-  simdjson_really_inline bool at_end() const noexcept;
+  simdjson_really_inline bool at_end_of_input_buffer() const noexcept;
 
   /**
    * Tell whether the iterator is at the start of the value
@@ -283,31 +283,17 @@ public:
   simdjson_warn_unused simdjson_really_inline simdjson_result<std::string_view> get_string() noexcept;
   simdjson_warn_unused simdjson_really_inline simdjson_result<raw_json_string> get_raw_json_string() noexcept;
   simdjson_warn_unused simdjson_really_inline simdjson_result<uint64_t> get_uint64() noexcept;
-  simdjson_warn_unused simdjson_really_inline simdjson_result<uint64_t> get_uint64_in_string() noexcept;
   simdjson_warn_unused simdjson_really_inline simdjson_result<int64_t> get_int64() noexcept;
-  simdjson_warn_unused simdjson_really_inline simdjson_result<int64_t> get_int64_in_string() noexcept;
   simdjson_warn_unused simdjson_really_inline simdjson_result<double> get_double() noexcept;
-  simdjson_warn_unused simdjson_really_inline simdjson_result<double> get_double_in_string() noexcept;
   simdjson_warn_unused simdjson_really_inline simdjson_result<bool> get_bool() noexcept;
   simdjson_really_inline bool is_null() noexcept;
-  simdjson_warn_unused simdjson_really_inline bool is_negative() noexcept;
-  simdjson_warn_unused simdjson_really_inline simdjson_result<bool> is_integer() noexcept;
-  simdjson_warn_unused simdjson_really_inline simdjson_result<number_type> get_number_type() noexcept;
-  simdjson_warn_unused simdjson_really_inline simdjson_result<number> get_number() noexcept;
 
   simdjson_warn_unused simdjson_really_inline simdjson_result<std::string_view> get_root_string() noexcept;
   simdjson_warn_unused simdjson_really_inline simdjson_result<raw_json_string> get_root_raw_json_string() noexcept;
   simdjson_warn_unused simdjson_really_inline simdjson_result<uint64_t> get_root_uint64() noexcept;
-  simdjson_warn_unused simdjson_really_inline simdjson_result<uint64_t> get_root_uint64_in_string() noexcept;
   simdjson_warn_unused simdjson_really_inline simdjson_result<int64_t> get_root_int64() noexcept;
-  simdjson_warn_unused simdjson_really_inline simdjson_result<int64_t> get_root_int64_in_string() noexcept;
   simdjson_warn_unused simdjson_really_inline simdjson_result<double> get_root_double() noexcept;
-  simdjson_warn_unused simdjson_really_inline simdjson_result<double> get_root_double_in_string() noexcept;
   simdjson_warn_unused simdjson_really_inline simdjson_result<bool> get_root_bool() noexcept;
-  simdjson_warn_unused simdjson_really_inline bool is_root_negative() noexcept;
-  simdjson_warn_unused simdjson_really_inline simdjson_result<bool> is_root_integer() noexcept;
-  simdjson_warn_unused simdjson_really_inline simdjson_result<number_type> get_root_number_type() noexcept;
-  simdjson_warn_unused simdjson_really_inline simdjson_result<number> get_root_number() noexcept;
   simdjson_really_inline bool is_root_null() noexcept;
 
   simdjson_really_inline error_code error() const noexcept;
@@ -350,8 +336,8 @@ protected:
   inline std::string to_string() const noexcept;
   simdjson_really_inline value_iterator(json_iterator *json_iter, depth_t depth, token_position start_index) noexcept;
 
-  simdjson_really_inline bool parse_null(const uint8_t *json) const noexcept;
-  simdjson_really_inline simdjson_result<bool> parse_bool(const uint8_t *json) const noexcept;
+  simdjson_really_inline bool parse_null(uint32_t max_len, const uint8_t *json) const noexcept;
+  simdjson_really_inline simdjson_result<bool> parse_bool(uint32_t max_len, const uint8_t *json) const noexcept;
   simdjson_really_inline const uint8_t *peek_start() const noexcept;
   simdjson_really_inline uint32_t peek_start_length() const noexcept;
 
@@ -423,7 +409,7 @@ protected:
    * Usage: the skip_child() method should never be used while we are pointing
    * at a key inside an object.
    */
-  simdjson_really_inline bool is_at_key() const noexcept;
+  simdjson_really_inline simdjson_result<bool> is_at_key() const noexcept;
 
   inline void assert_at_start() const noexcept;
   inline void assert_at_container_start() const noexcept;
@@ -437,10 +423,10 @@ protected:
 
   /** @copydoc error_code json_iterator::position() const noexcept; */
   simdjson_really_inline token_position position() const noexcept;
-  /** @copydoc error_code json_iterator::end_position() const noexcept; */
+  /** @copydoc error_code json_iterator::end_of_input_buffer_position() const noexcept; */
   simdjson_really_inline token_position last_position() const noexcept;
-  /** @copydoc error_code json_iterator::end_position() const noexcept; */
-  simdjson_really_inline token_position end_position() const noexcept;
+  /** @copydoc error_code json_iterator::end_of_input_buffer_position() const noexcept; */
+  simdjson_really_inline token_position end_of_input_buffer_position() const noexcept;
   /** @copydoc error_code json_iterator::report_error(error_code error, const char *message) noexcept; */
   simdjson_really_inline error_code report_error(error_code error, const char *message) noexcept;
 
@@ -448,6 +434,7 @@ protected:
   friend class object;
   friend class array;
   friend class value;
+  friend class field;
 }; // value_iterator
 
 } // namespace ondemand

include/simdjson/generic/stringparsing.h

@@ -77,10 +77,11 @@ simdjson_really_inline bool handle_unicode_codepoint(const uint8_t **src_ptr,
  * Unescape a string from src to dst, stopping at a final unescaped quote. E.g., if src points at 'joe"', then
  * dst needs to have four free bytes.
  */
-simdjson_warn_unused simdjson_really_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst) {
+simdjson_warn_unused simdjson_really_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst, const uint8_t *end) {
+  const uint8_t *last_full_buf = end - backslash_and_quote::BYTES_PROCESSED;
   while (1) {
     // Copy the next n bytes, and find the backslash and quote in them.
-    auto bs_quote = backslash_and_quote::copy_and_find(src, dst);
+    auto bs_quote = backslash_and_quote::copy_and_find(src, dst, last_full_buf);
     // If the next thing is the end quote, copy and return
     if (bs_quote.has_quote_first()) {
       // we encountered quotes first. Move dst to point to quotes and exit
@@ -123,15 +124,6 @@ simdjson_warn_unused simdjson_really_inline uint8_t *parse_string(const uint8_t
   return nullptr;
 }
 
-simdjson_unused simdjson_warn_unused simdjson_really_inline error_code parse_string_to_buffer(const uint8_t *src, uint8_t *&current_string_buf_loc, std::string_view &s) {
-  if (*(src++) != '"') { return STRING_ERROR; }
-  auto end = stringparsing::parse_string(src, current_string_buf_loc);
-  if (!end) { return STRING_ERROR; }
-  s = std::string_view(reinterpret_cast<const char *>(current_string_buf_loc), end-current_string_buf_loc);
-  current_string_buf_loc = end;
-  return SUCCESS;
-}
-
 } // namespace stringparsing
 } // unnamed namespace
 } // namespace SIMDJSON_IMPLEMENTATION

include/simdjson/haswell/numberparsing.h

@@ -5,6 +5,33 @@ namespace simdjson {
 namespace SIMDJSON_IMPLEMENTATION {
 namespace {
 
+/**
+ * We could use a scalar version of parse_eight_digits_unrolled that compiles
+ * to something like
+ *
+ *        and     rax, qword ptr [rdi]
+ *        imul    rax, rax, 2561
+ *        shr     rax, 8
+ *        movabs  rcx, 71777214294589695
+ *        and     rcx, rax
+ *        imul    rax, rcx, 6553601
+ *        shr     rax, 16
+ *        movabs  rcx, 281470681808895
+ *        and     rcx, rax
+ *        movabs  rax, 42949672960001
+ *        imul    rax, rcx
+ *        shr     rax, 32
+ *
+ * The vectorized sequence below is favorable, as it compiles to
+ *       vmovq   xmm0, qword ptr [rdi]           # xmm0 = mem[0],zero
+ *       vpaddb  xmm0, xmm0, xmmword ptr [rip + .LCPI0_0]
+ *       vpmaddubsw      xmm0, xmm0, xmmword ptr [rip + .LCPI0_1]
+ *       vpmaddwd        xmm0, xmm0, xmmword ptr [rip + .LCPI0_2]
+ *       vpackusdw       xmm0, xmm0, xmm0
+ *       vpmaddwd        xmm0, xmm0, xmmword ptr [rip + .LCPI0_3]
+ *       vmovd   eax, xmm0
+ * even though it does twice the work and looks far more complicated.
+ */
 static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) {
   // this actually computes *16* values so we are being wasteful.
   const __m128i ascii0 = _mm_set1_epi8('0');
@@ -13,8 +40,10 @@ static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t
   const __m128i mul_1_100 = _mm_setr_epi16(100, 1, 100, 1, 100, 1, 100, 1);
   const __m128i mul_1_10000 =
       _mm_setr_epi16(10000, 1, 10000, 1, 10000, 1, 10000, 1);
+  // _mm_loadu_si128(reinterpret_cast<const __m128i *>(chars)) is faster
+  // but _mm_loadl_epi64(chars) avoids buffer overflows.
   const __m128i input = _mm_sub_epi8(
-      _mm_loadu_si128(reinterpret_cast<const __m128i *>(chars)), ascii0);
+      _mm_loadl_epi64(reinterpret_cast<const __m128i *>(chars)), ascii0);
   const __m128i t1 = _mm_maddubs_epi16(input, mul_1_10);
   const __m128i t2 = _mm_madd_epi16(t1, mul_1_100);
   const __m128i t3 = _mm_packus_epi32(t2, t2);

include/simdjson/haswell/simd.h

@@ -303,12 +303,11 @@ namespace simd {
     simdjson_really_inline simd8x64(const simd8<T> chunk0, const simd8<T> chunk1) : chunks{chunk0, chunk1} {}
     simdjson_really_inline simd8x64(const T ptr[64]) : chunks{simd8<T>::load(ptr), simd8<T>::load(ptr+32)} {}
 
-    simdjson_really_inline uint64_t compress(uint64_t mask, T * output) const {
+    simdjson_really_inline void compress(uint64_t mask, T * output) const {
       uint32_t mask1 = uint32_t(mask);
       uint32_t mask2 = uint32_t(mask >> 32);
       this->chunks[0].compress(mask1, output);
       this->chunks[1].compress(mask2, output + 32 - count_ones(mask1));
-      return 64 - count_ones(mask);
     }
 
     simdjson_really_inline void store(T ptr[64]) const {

include/simdjson/haswell/stringparsing.h

@@ -15,7 +15,10 @@ using namespace simd;
 struct backslash_and_quote {
 public:
   static constexpr uint32_t BYTES_PROCESSED = 32;
-  simdjson_really_inline static backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst);
+  simdjson_really_inline static backslash_and_quote
+  copy_and_find(const uint8_t *src, uint8_t *dst, const uint8_t *last_full_buf);
+  simdjson_really_inline static backslash_and_quote
+  copy_and_find(const uint8_t *src, uint8_t *dst);
 
   simdjson_really_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; }
   simdjson_really_inline bool has_backslash() { return ((quote_bits - 1) & bs_bits) != 0; }
@@ -26,7 +29,21 @@ public:
   uint32_t quote_bits;
 }; // struct backslash_and_quote
 
-simdjson_really_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) {
+simdjson_really_inline backslash_and_quote
+backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst, const uint8_t *last_full_buf) {
+  // If we don't have enough memory left to load a whole simd register, copy it first.
+  uint8_t tmpbuf[BYTES_PROCESSED];
+  if (simdjson_unlikely(src > last_full_buf)) {
+    SIMDJSON_ASSUME(last_full_buf+BYTES_PROCESSED > src);
+    std::memset(tmpbuf, 0, BYTES_PROCESSED);
+    std::memcpy(tmpbuf, src, last_full_buf+BYTES_PROCESSED-src);
+    src = tmpbuf;
+  }
+  return copy_and_find(src, dst);
+}
+
+simdjson_really_inline backslash_and_quote
+backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) {
   // this can read up to 15 bytes beyond the buffer size, but we require
   // SIMDJSON_PADDING of padding
   static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes");

include/simdjson/internal/dom_parser_implementation.h

@@ -34,6 +34,13 @@ inline bool is_streaming(stage1_mode mode) {
 
 namespace internal {
 
+// expectation: sizeof(open_container) = 64/8.
+struct open_container {
+  uint32_t tape_index; // where, on the tape, does the scope ([,{) begins
+  uint32_t count; // how many elements in the scope
+}; // struct open_container
+
+static_assert(sizeof(open_container) == 64/8, "Open container must be 64 bits");
 
 /**
  * An implementation of simdjson's DOM parser for a particular CPU architecture.
@@ -178,6 +185,19 @@ protected:
    */
   size_t _max_depth{0};
 
+public:
+  /** Tape location of each open { or [ */
+  std::unique_ptr<open_container[]> open_containers{};
+  /** Whether each open container is a [ or { */
+  std::unique_ptr<bool[]> is_array{};
+  /** Buffer passed to stage 1 */
+  const uint8_t *buf{};
+  /** Length passed to stage 1 */
+  size_t len{0};
+  /** Document passed to stage 2 */
+  dom::document *doc{};
+
+protected:
   // Declaring these so that subclasses can use them to implement their constructors.
   simdjson_really_inline dom_parser_implementation() noexcept;
   simdjson_really_inline dom_parser_implementation(dom_parser_implementation &&other) noexcept;

include/simdjson/internal/jsonformatutils.h

@@ -2,7 +2,7 @@
 #define SIMDJSON_INTERNAL_JSONFORMATUTILS_H
 
 #include <iomanip>
-#include <ostream>
+#include <iostream>
 #include <sstream>
 
 namespace simdjson {

include/simdjson/nonstd/string_view.hpp

@@ -1,4 +1,4 @@
-// Copyright 2017-2020 by Martin Moene
+// Copyright 2017-2019 by Martin Moene
 //
 // string-view lite, a C++17-like string_view for C++98 and later.
 // For more information see https://github.com/martinmoene/string-view-lite
@@ -12,7 +12,7 @@
 #define NONSTD_SV_LITE_H_INCLUDED
 
 #define string_view_lite_MAJOR  1
-#define string_view_lite_MINOR  6
+#define string_view_lite_MINOR  4
 #define string_view_lite_PATCH  0
 
 #define string_view_lite_VERSION  nssv_STRINGIFY(string_view_lite_MAJOR) "." nssv_STRINGIFY(string_view_lite_MINOR) "." nssv_STRINGIFY(string_view_lite_PATCH)
@@ -26,24 +26,14 @@
 #define nssv_STRING_VIEW_NONSTD   1
 #define nssv_STRING_VIEW_STD      2
 
-// tweak header support:
-
-#ifdef __has_include
-# if __has_include(<nonstd/string_view.tweak.hpp>)
-#  include <nonstd/string_view.tweak.hpp>
-# endif
-#define nssv_HAVE_TWEAK_HEADER  1
-#else
-#define nssv_HAVE_TWEAK_HEADER  0
-//# pragma message("string_view.hpp: Note: Tweak header not supported.")
-#endif
-
-// string_view selection and configuration:
-
 #if !defined( nssv_CONFIG_SELECT_STRING_VIEW )
 # define nssv_CONFIG_SELECT_STRING_VIEW  ( nssv_HAVE_STD_STRING_VIEW ? nssv_STRING_VIEW_STD : nssv_STRING_VIEW_NONSTD )
 #endif
 
+#if defined( nssv_CONFIG_SELECT_STD_STRING_VIEW ) || defined( nssv_CONFIG_SELECT_NONSTD_STRING_VIEW )
+# error nssv_CONFIG_SELECT_STD_STRING_VIEW and nssv_CONFIG_SELECT_NONSTD_STRING_VIEW are deprecated and removed, please use nssv_CONFIG_SELECT_STRING_VIEW=nssv_STRING_VIEW_...
+#endif
+
 #ifndef  nssv_CONFIG_STD_SV_OPERATOR
 # define nssv_CONFIG_STD_SV_OPERATOR  0
 #endif
@@ -65,17 +55,10 @@
 # define nssv_CONFIG_CONVERSION_STD_STRING_FREE_FUNCTIONS  1
 #endif
 
-#ifndef  nssv_CONFIG_NO_STREAM_INSERTION
-# define nssv_CONFIG_NO_STREAM_INSERTION  0
-#endif
-
 // Control presence of exception handling (try and auto discover):
 
 #ifndef nssv_CONFIG_NO_EXCEPTIONS
-# if _MSC_VER
-#  include <cstddef>    // for _HAS_EXCEPTIONS
-# endif
-# if defined(__cpp_exceptions) || defined(__EXCEPTIONS) || (_HAS_EXCEPTIONS)
+# if defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)
 #  define nssv_CONFIG_NO_EXCEPTIONS  0
 # else
 #  define nssv_CONFIG_NO_EXCEPTIONS  1
@@ -238,21 +221,16 @@ using std::operator<<;
 
 #define nssv_COMPILER_VERSION( major, minor, patch )  ( 10 * ( 10 * (major) + (minor) ) + (patch) )
 
-#if defined( __apple_build_version__ )
-# define nssv_COMPILER_APPLECLANG_VERSION  nssv_COMPILER_VERSION(__clang_major__, __clang_minor__, __clang_patchlevel__)
-# define nssv_COMPILER_CLANG_VERSION       0
-#elif defined( __clang__ )
-# define nssv_COMPILER_APPLECLANG_VERSION  0
-# define nssv_COMPILER_CLANG_VERSION       nssv_COMPILER_VERSION(__clang_major__, __clang_minor__, __clang_patchlevel__)
+#if defined(__clang__)
+# define nssv_COMPILER_CLANG_VERSION  nssv_COMPILER_VERSION(__clang_major__, __clang_minor__, __clang_patchlevel__)
 #else
-# define nssv_COMPILER_APPLECLANG_VERSION  0
-# define nssv_COMPILER_CLANG_VERSION       0
+# define nssv_COMPILER_CLANG_VERSION    0
 #endif
 
 #if defined(__GNUC__) && !defined(__clang__)
 # define nssv_COMPILER_GNUC_VERSION  nssv_COMPILER_VERSION(__GNUC__, __GNUC_MINOR__, __GNUC_PATCHLEVEL__)
 #else
-# define nssv_COMPILER_GNUC_VERSION  0
+# define nssv_COMPILER_GNUC_VERSION    0
 #endif
 
 // half-open range [lo..hi):
@@ -314,45 +292,6 @@ using std::operator<<;
 
 #define nssv_HAVE_STD_HASH              nssv_CPP11_120
 
-// Presence of compiler intrinsics:
-
-// Providing char-type specializations for compare() and length() that
-// use compiler intrinsics can improve compile- and run-time performance.
-//
-// The challenge is in using the right combinations of builtin availability
-// and its constexpr-ness.
-//
-// | compiler | __builtin_memcmp (constexpr) | memcmp  (constexpr) |
-// |----------|------------------------------|---------------------|
-// | clang    | 4.0              (>= 4.0   ) | any     (?        ) |
-// | clang-a  | 9.0              (>= 9.0   ) | any     (?        ) |
-// | gcc      | any              (constexpr) | any     (?        ) |
-// | msvc     | >= 14.2 C++17    (>= 14.2  ) | any     (?        ) |
-
-#define nssv_HAVE_BUILTIN_VER     ( (nssv_CPP17_000 && nssv_COMPILER_MSVC_VERSION >= 142) || nssv_COMPILER_GNUC_VERSION > 0 || nssv_COMPILER_CLANG_VERSION >= 400 || nssv_COMPILER_APPLECLANG_VERSION >= 900 )
-#define nssv_HAVE_BUILTIN_CE      (  nssv_HAVE_BUILTIN_VER )
-
-#define nssv_HAVE_BUILTIN_MEMCMP  ( (nssv_HAVE_CONSTEXPR_14 && nssv_HAVE_BUILTIN_CE) || !nssv_HAVE_CONSTEXPR_14 )
-#define nssv_HAVE_BUILTIN_STRLEN  ( (nssv_HAVE_CONSTEXPR_11 && nssv_HAVE_BUILTIN_CE) || !nssv_HAVE_CONSTEXPR_11 )
-
-#ifdef __has_builtin
-# define nssv_HAVE_BUILTIN( x )  __has_builtin( x )
-#else
-# define nssv_HAVE_BUILTIN( x )  0
-#endif
-
-#if nssv_HAVE_BUILTIN(__builtin_memcmp) || nssv_HAVE_BUILTIN_VER
-# define nssv_BUILTIN_MEMCMP  __builtin_memcmp
-#else
-# define nssv_BUILTIN_MEMCMP  memcmp
-#endif
-
-#if nssv_HAVE_BUILTIN(__builtin_strlen) || nssv_HAVE_BUILTIN_VER
-# define nssv_BUILTIN_STRLEN  __builtin_strlen
-#else
-# define nssv_BUILTIN_STRLEN  strlen
-#endif
-
 // C++ feature usage:
 
 #if nssv_HAVE_CONSTEXPR_11
@@ -411,12 +350,9 @@ using std::operator<<;
 #include <cassert>
 #include <iterator>
 #include <limits>
+#include <ostream>
 #include <string>   // std::char_traits<>
 
-#if ! nssv_CONFIG_NO_STREAM_INSERTION
-# include <ostream>
-#endif
-
 #if ! nssv_CONFIG_NO_EXCEPTIONS
 # include <stdexcept>
 #endif
@@ -469,75 +405,40 @@ nssv_DISABLE_MSVC_WARNINGS( 4455 26481 26472 )
 
 namespace nonstd { namespace sv_lite {
 
+#if nssv_CPP11_OR_GREATER
+
 namespace detail {
 
-// support constexpr comparison in C++14;
-// for C++17 and later, use provided traits:
+#if nssv_CPP14_OR_GREATER
 
 template< typename CharT >
-inline nssv_constexpr14 int compare( CharT const * s1, CharT const * s2, std::size_t count )
+inline constexpr std::size_t length( CharT * s, std::size_t result = 0 )
 {
-    while ( count-- != 0 )
-    {
-        if ( *s1 < *s2 ) return -1;
-        if ( *s1 > *s2 ) return +1;
-        ++s1; ++s2;
+    CharT * v = s;
+    std::size_t r = result;
+    while ( *v != '\0' ) {
+       ++v;
+       ++r;
     }
-    return 0;
+    return r;
 }
 
-#if nssv_HAVE_BUILTIN_MEMCMP
+#else // nssv_CPP14_OR_GREATER
 
-// specialization of compare() for char, see also generic compare() above:
-
-inline nssv_constexpr14 int compare( char const * s1, char const * s2, std::size_t count )
-{
-    return nssv_BUILTIN_MEMCMP( s1, s2, count );
-}
-
-#endif
-
-#if nssv_HAVE_BUILTIN_STRLEN
-
-// specialization of length() for char, see also generic length() further below:
-
-inline nssv_constexpr std::size_t length( char const * s )
-{
-    return nssv_BUILTIN_STRLEN( s );
-}
-
-#endif
-
-#if defined(__OPTIMIZE__)
-
-// gcc, clang provide __OPTIMIZE__
 // Expect tail call optimization to make length() non-recursive:
 
 template< typename CharT >
-inline nssv_constexpr std::size_t length( CharT * s, std::size_t result = 0 )
+inline constexpr std::size_t length( CharT * s, std::size_t result = 0 )
 {
     return *s == '\0' ? result : length( s + 1, result + 1 );
 }
 
-#else // OPTIMIZE
-
-// non-recursive:
-
-template< typename CharT >
-inline nssv_constexpr14 std::size_t length( CharT * s )
-{
-    std::size_t result = 0;
-    while ( *s++ != '\0' )
-    {
-       ++result;
-    }
-    return result;
-}
-
-#endif // OPTIMIZE
+#endif // nssv_CPP14_OR_GREATER
 
 } // namespace detail
 
+#endif // nssv_CPP11_OR_GREATER
+
 template
 <
     class CharT,
@@ -688,9 +589,9 @@ public:
 
     nssv_constexpr14 void swap( basic_string_view & other ) nssv_noexcept
     {
-        const basic_string_view tmp(other);
-        other = *this;
-        *this = tmp;
+        using std::swap;
+        swap( data_, other.data_ );
+        swap( size_, other.size_ );
     }
 
     // 24.4.2.6 String operations:
@@ -729,11 +630,7 @@ public:
 
     nssv_constexpr14 int compare( basic_string_view other ) const nssv_noexcept // (1)
     {
-#if nssv_CPP17_OR_GREATER
         if ( const int result = Traits::compare( data(), other.data(), (std::min)( size(), other.size() ) ) )
-#else
-        if ( const int result = detail::compare( data(), other.data(), (std::min)( size(), other.size() ) ) )
-#endif
         {
             return result;
         }
@@ -977,7 +874,7 @@ private:
     {
         const basic_string_view v;
 
-        nssv_constexpr explicit not_in_view( basic_string_view v_ ) : v( v_ ) {}
+        nssv_constexpr explicit not_in_view( basic_string_view v ) : v( v ) {}
 
         nssv_constexpr bool operator()( CharT c ) const
         {
@@ -1067,37 +964,37 @@ template< class CharT, class Traits >
 nssv_constexpr bool operator== (
     basic_string_view <CharT, Traits> lhs,
     basic_string_view <CharT, Traits> rhs ) nssv_noexcept
-{ return lhs.size() == rhs.size() && lhs.compare( rhs ) == 0; }
+{ return lhs.compare( rhs ) == 0 ; }
 
 template< class CharT, class Traits >
 nssv_constexpr bool operator!= (
     basic_string_view <CharT, Traits> lhs,
     basic_string_view <CharT, Traits> rhs ) nssv_noexcept
-{ return !( lhs == rhs ); }
+{ return lhs.compare( rhs ) != 0 ; }
 
 template< class CharT, class Traits >
 nssv_constexpr bool operator< (
     basic_string_view <CharT, Traits> lhs,
     basic_string_view <CharT, Traits> rhs ) nssv_noexcept
-{ return lhs.compare( rhs ) < 0; }
+{ return lhs.compare( rhs ) < 0 ; }
 
 template< class CharT, class Traits >
 nssv_constexpr bool operator<= (
     basic_string_view <CharT, Traits> lhs,
     basic_string_view <CharT, Traits> rhs ) nssv_noexcept
-{ return lhs.compare( rhs ) <= 0; }
+{ return lhs.compare( rhs ) <= 0 ; }
 
 template< class CharT, class Traits >
 nssv_constexpr bool operator> (
     basic_string_view <CharT, Traits> lhs,
     basic_string_view <CharT, Traits> rhs ) nssv_noexcept
-{ return lhs.compare( rhs ) > 0; }
+{ return lhs.compare( rhs ) > 0 ; }
 
 template< class CharT, class Traits >
 nssv_constexpr bool operator>= (
     basic_string_view <CharT, Traits> lhs,
     basic_string_view <CharT, Traits> rhs ) nssv_noexcept
-{ return lhs.compare( rhs ) >= 0; }
+{ return lhs.compare( rhs ) >= 0 ; }
 
 // Let S be basic_string_view<CharT, Traits>, and sv be an instance of S.
 // Implementations shall provide sufficient additional overloads marked
@@ -1106,21 +1003,21 @@ nssv_constexpr bool operator>= (
 
 #if ! nssv_CPP11_OR_GREATER || nssv_BETWEEN( nssv_COMPILER_MSVC_VERSION, 100, 141 )
 
-// accommodate for older compilers:
+// accomodate for older compilers:
 
 // ==
 
 template< class CharT, class Traits>
 nssv_constexpr bool operator==(
     basic_string_view<CharT, Traits> lhs,
-    CharT const * rhs ) nssv_noexcept
-{ return lhs.size() == detail::length( rhs ) && lhs.compare( rhs ) == 0; }
+    char const * rhs ) nssv_noexcept
+{ return lhs.compare( rhs ) == 0; }
 
 template< class CharT, class Traits>
 nssv_constexpr bool operator==(
-    CharT const * lhs,
+    char const * lhs,
     basic_string_view<CharT, Traits> rhs ) nssv_noexcept
-{ return detail::length( lhs ) == rhs.size() && rhs.compare( lhs ) == 0; }
+{ return rhs.compare( lhs ) == 0; }
 
 template< class CharT, class Traits>
 nssv_constexpr bool operator==(
@@ -1139,38 +1036,38 @@ nssv_constexpr bool operator==(
 template< class CharT, class Traits>
 nssv_constexpr bool operator!=(
     basic_string_view<CharT, Traits> lhs,
-    CharT const * rhs ) nssv_noexcept
-{ return !( lhs == rhs ); }
+    char const * rhs ) nssv_noexcept
+{ return lhs.compare( rhs ) != 0; }
 
 template< class CharT, class Traits>
 nssv_constexpr bool operator!=(
-    CharT const * lhs,
+    char const * lhs,
     basic_string_view<CharT, Traits> rhs ) nssv_noexcept
-{ return !( lhs == rhs ); }
+{ return rhs.compare( lhs ) != 0; }
 
 template< class CharT, class Traits>
 nssv_constexpr bool operator!=(
     basic_string_view<CharT, Traits> lhs,
     std::basic_string<CharT, Traits> rhs ) nssv_noexcept
-{ return !( lhs == rhs ); }
+{ return lhs.size() != rhs.size() && lhs.compare( rhs ) != 0; }
 
 template< class CharT, class Traits>
 nssv_constexpr bool operator!=(
     std::basic_string<CharT, Traits> rhs,
     basic_string_view<CharT, Traits> lhs ) nssv_noexcept
-{ return !( lhs == rhs ); }
+{ return lhs.size() != rhs.size() || rhs.compare( lhs ) != 0; }
 
 // <
 
 template< class CharT, class Traits>
 nssv_constexpr bool operator<(
     basic_string_view<CharT, Traits> lhs,
-    CharT const * rhs ) nssv_noexcept
+    char const * rhs ) nssv_noexcept
 { return lhs.compare( rhs ) < 0; }
 
 template< class CharT, class Traits>
 nssv_constexpr bool operator<(
-    CharT const * lhs,
+    char const * lhs,
     basic_string_view<CharT, Traits> rhs ) nssv_noexcept
 { return rhs.compare( lhs ) > 0; }
 
@@ -1191,12 +1088,12 @@ nssv_constexpr bool operator<(
 template< class CharT, class Traits>
 nssv_constexpr bool operator<=(
     basic_string_view<CharT, Traits> lhs,
-    CharT const * rhs ) nssv_noexcept
+    char const * rhs ) nssv_noexcept
 { return lhs.compare( rhs ) <= 0; }
 
 template< class CharT, class Traits>
 nssv_constexpr bool operator<=(
-    CharT const * lhs,
+    char const * lhs,
     basic_string_view<CharT, Traits> rhs ) nssv_noexcept
 { return rhs.compare( lhs ) >= 0; }
 
@@ -1217,12 +1114,12 @@ nssv_constexpr bool operator<=(
 template< class CharT, class Traits>
 nssv_constexpr bool operator>(
     basic_string_view<CharT, Traits> lhs,
-    CharT const * rhs ) nssv_noexcept
+    char const * rhs ) nssv_noexcept
 { return lhs.compare( rhs ) > 0; }
 
 template< class CharT, class Traits>
 nssv_constexpr bool operator>(
-    CharT const * lhs,
+    char const * lhs,
     basic_string_view<CharT, Traits> rhs ) nssv_noexcept
 { return rhs.compare( lhs ) < 0; }
 
@@ -1243,12 +1140,12 @@ nssv_constexpr bool operator>(
 template< class CharT, class Traits>
 nssv_constexpr bool operator>=(
     basic_string_view<CharT, Traits> lhs,
-    CharT const * rhs ) nssv_noexcept
+    char const * rhs ) nssv_noexcept
 { return lhs.compare( rhs ) >= 0; }
 
 template< class CharT, class Traits>
 nssv_constexpr bool operator>=(
-    CharT const * lhs,
+    char const * lhs,
     basic_string_view<CharT, Traits> rhs ) nssv_noexcept
 { return rhs.compare( lhs ) <= 0; }
 
@@ -1268,7 +1165,7 @@ nssv_constexpr bool operator>=(
 
 #define nssv_BASIC_STRING_VIEW_I(T,U)  typename std::decay< basic_string_view<T,U> >::type
 
-#if defined(_MSC_VER)       // issue 40
+#if nssv_BETWEEN( nssv_COMPILER_MSVC_VERSION, 140, 150 )
 # define nssv_MSVC_ORDER(x)  , int=x
 #else
 # define nssv_MSVC_ORDER(x)  /*, int=x*/
@@ -1280,7 +1177,7 @@ template< class CharT, class Traits  nssv_MSVC_ORDER(1) >
 nssv_constexpr bool operator==(
          basic_string_view  <CharT, Traits> lhs,
     nssv_BASIC_STRING_VIEW_I(CharT, Traits) rhs ) nssv_noexcept
-{ return lhs.size() == rhs.size() && lhs.compare( rhs ) == 0; }
+{ return lhs.compare( rhs ) == 0; }
 
 template< class CharT, class Traits  nssv_MSVC_ORDER(2) >
 nssv_constexpr bool operator==(
@@ -1294,13 +1191,13 @@ template< class CharT, class Traits  nssv_MSVC_ORDER(1) >
 nssv_constexpr bool operator!= (
          basic_string_view  < CharT, Traits > lhs,
     nssv_BASIC_STRING_VIEW_I( CharT, Traits ) rhs ) nssv_noexcept
-{ return !( lhs == rhs ); }
+{ return lhs.size() != rhs.size() || lhs.compare( rhs ) != 0 ; }
 
 template< class CharT, class Traits  nssv_MSVC_ORDER(2) >
 nssv_constexpr bool operator!= (
     nssv_BASIC_STRING_VIEW_I( CharT, Traits ) lhs,
          basic_string_view  < CharT, Traits > rhs ) nssv_noexcept
-{ return !( lhs == rhs ); }
+{ return lhs.compare( rhs ) != 0 ; }
 
 // <
 
@@ -1308,13 +1205,13 @@ template< class CharT, class Traits  nssv_MSVC_ORDER(1) >
 nssv_constexpr bool operator< (
          basic_string_view  < CharT, Traits > lhs,
     nssv_BASIC_STRING_VIEW_I( CharT, Traits ) rhs ) nssv_noexcept
-{ return lhs.compare( rhs ) < 0; }
+{ return lhs.compare( rhs ) < 0 ; }
 
 template< class CharT, class Traits  nssv_MSVC_ORDER(2) >
 nssv_constexpr bool operator< (
     nssv_BASIC_STRING_VIEW_I( CharT, Traits ) lhs,
          basic_string_view  < CharT, Traits > rhs ) nssv_noexcept
-{ return lhs.compare( rhs ) < 0; }
+{ return lhs.compare( rhs ) < 0 ; }
 
 // <=
 
@@ -1322,13 +1219,13 @@ template< class CharT, class Traits  nssv_MSVC_ORDER(1) >
 nssv_constexpr bool operator<= (
          basic_string_view  < CharT, Traits > lhs,
     nssv_BASIC_STRING_VIEW_I( CharT, Traits ) rhs ) nssv_noexcept
-{ return lhs.compare( rhs ) <= 0; }
+{ return lhs.compare( rhs ) <= 0 ; }
 
 template< class CharT, class Traits  nssv_MSVC_ORDER(2) >
 nssv_constexpr bool operator<= (
     nssv_BASIC_STRING_VIEW_I( CharT, Traits ) lhs,
          basic_string_view  < CharT, Traits > rhs ) nssv_noexcept
-{ return lhs.compare( rhs ) <= 0; }
+{ return lhs.compare( rhs ) <= 0 ; }
 
 // >
 
@@ -1336,13 +1233,13 @@ template< class CharT, class Traits  nssv_MSVC_ORDER(1) >
 nssv_constexpr bool operator> (
          basic_string_view  < CharT, Traits > lhs,
     nssv_BASIC_STRING_VIEW_I( CharT, Traits ) rhs ) nssv_noexcept
-{ return lhs.compare( rhs ) > 0; }
+{ return lhs.compare( rhs ) > 0 ; }
 
 template< class CharT, class Traits  nssv_MSVC_ORDER(2) >
 nssv_constexpr bool operator> (
     nssv_BASIC_STRING_VIEW_I( CharT, Traits ) lhs,
          basic_string_view  < CharT, Traits > rhs ) nssv_noexcept
-{ return lhs.compare( rhs ) > 0; }
+{ return lhs.compare( rhs ) > 0 ; }
 
 // >=
 
@@ -1350,13 +1247,13 @@ template< class CharT, class Traits  nssv_MSVC_ORDER(1) >
 nssv_constexpr bool operator>= (
          basic_string_view  < CharT, Traits > lhs,
     nssv_BASIC_STRING_VIEW_I( CharT, Traits ) rhs ) nssv_noexcept
-{ return lhs.compare( rhs ) >= 0; }
+{ return lhs.compare( rhs ) >= 0 ; }
 
 template< class CharT, class Traits  nssv_MSVC_ORDER(2) >
 nssv_constexpr bool operator>= (
     nssv_BASIC_STRING_VIEW_I( CharT, Traits ) lhs,
          basic_string_view  < CharT, Traits > rhs ) nssv_noexcept
-{ return lhs.compare( rhs ) >= 0; }
+{ return lhs.compare( rhs ) >= 0 ; }
 
 #undef nssv_MSVC_ORDER
 #undef nssv_BASIC_STRING_VIEW_I
@@ -1365,8 +1262,6 @@ nssv_constexpr bool operator>= (
 
 // 24.4.4 Inserters and extractors:
 
-#if ! nssv_CONFIG_NO_STREAM_INSERTION
-
 namespace detail {
 
 template< class Stream >
@@ -1416,8 +1311,6 @@ operator<<(
     return detail::write_to_stream( os, sv );
 }
 
-#endif // nssv_CONFIG_NO_STREAM_INSERTION
-
 // Several typedefs for common character types are provided:
 
 typedef basic_string_view<char>      string_view;
@@ -1566,9 +1459,7 @@ using sv_lite::operator<=;
 using sv_lite::operator>;
 using sv_lite::operator>=;
 
-#if ! nssv_CONFIG_NO_STREAM_INSERTION
 using sv_lite::operator<<;
-#endif
 
 #if nssv_CONFIG_CONVERSION_STD_STRING_FREE_FUNCTIONS
 using sv_lite::to_string;
@@ -1635,4 +1526,4 @@ public:
 nssv_RESTORE_WARNINGS()
 
 #endif // nssv_HAVE_STD_STRING_VIEW
-#endif // NONSTD_SV_LITE_H_INCLUDED
+#endif // NONSTD_SV_LITE_H_INCLUDED

include/simdjson/padded_string-inl.h

@@ -110,6 +110,7 @@ inline const char *padded_string::data() const noexcept { return data_ptr; }
 inline char *padded_string::data() noexcept { return data_ptr; }
 
 inline padded_string::operator std::string_view() const { return std::string_view(data(), length()); }
+inline std::string padded_string::to_string() const { return std::string(data(), length()); }
 
 inline padded_string::operator padded_string_view() const noexcept {
   return padded_string_view(data(), length(), length() + SIMDJSON_PADDING);

include/simdjson/padded_string.h

@@ -98,6 +98,11 @@ struct padded_string final {
    */
   operator std::string_view() const;
 
+  /**
+   * Create a std::string with the same content (and no padding).
+   */
+  std::string to_string() const;
+
   /**
    * Create a padded_string_view with the same content.
    */

include/simdjson/padded_string_view.h

@@ -50,6 +50,8 @@ public:
    * @param capacity The allocated length of the string, including padding.
    */
   explicit inline padded_string_view(std::string_view s, size_t capacity) noexcept;
+  inline padded_string_view(const char *s, size_t capacity) noexcept = delete;
+  inline padded_string_view(const uint8_t *s, size_t capacity) noexcept = delete;
 
   /** The number of allocated bytes. */
   inline size_t capacity() const noexcept;

include/simdjson/ppc64/simd.h

@@ -422,7 +422,7 @@ template <typename T> struct simd8x64 {
            (this->chunks[2] | this->chunks[3]);
   }
 
-  simdjson_really_inline uint64_t compress(uint64_t mask, T *output) const {
+  simdjson_really_inline void compress(uint64_t mask, T *output) const {
     this->chunks[0].compress(uint16_t(mask), output);
     this->chunks[1].compress(uint16_t(mask >> 16),
                              output + 16 - count_ones(mask & 0xFFFF));
@@ -430,7 +430,6 @@ template <typename T> struct simd8x64 {
                              output + 32 - count_ones(mask & 0xFFFFFFFF));
     this->chunks[3].compress(uint16_t(mask >> 48),
                              output + 48 - count_ones(mask & 0xFFFFFFFFFFFF));
-    return 64 - count_ones(mask);
   }
 
   simdjson_really_inline uint64_t to_bitmask() const {

include/simdjson/ppc64/stringparsing.h

@@ -16,6 +16,8 @@ struct backslash_and_quote {
 public:
   static constexpr uint32_t BYTES_PROCESSED = 32;
   simdjson_really_inline static backslash_and_quote
+  copy_and_find(const uint8_t *src, uint8_t *dst, const uint8_t *last_full_buf);
+  simdjson_really_inline static backslash_and_quote
   copy_and_find(const uint8_t *src, uint8_t *dst);
 
   simdjson_really_inline bool has_quote_first() {
@@ -33,6 +35,19 @@ public:
   uint32_t quote_bits;
 }; // struct backslash_and_quote
 
+simdjson_really_inline backslash_and_quote
+backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst, const uint8_t *last_full_buf) {
+  // If we don't have enough memory left to load a whole simd register, copy it first.
+  uint8_t tmpbuf[BYTES_PROCESSED];
+  if (simdjson_unlikely(src > last_full_buf)) {
+    SIMDJSON_ASSUME(last_full_buf+BYTES_PROCESSED > src);
+    std::memset(tmpbuf, 0, BYTES_PROCESSED);
+    std::memcpy(tmpbuf, src, last_full_buf+BYTES_PROCESSED-src);
+    src = tmpbuf;
+  }
+  return copy_and_find(src, dst);
+}
+
 simdjson_really_inline backslash_and_quote
 backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) {
   // this can read up to 31 bytes beyond the buffer size, but we require

include/simdjson/simdjson_version.h

@@ -4,22 +4,22 @@
 #define SIMDJSON_SIMDJSON_VERSION_H
 
 /** The version of simdjson being used (major.minor.revision) */
-#define SIMDJSON_VERSION 1.0.2
+#define SIMDJSON_VERSION 0.9.1
 
 namespace simdjson {
 enum {
   /**
    * The major version (MAJOR.minor.revision) of simdjson being used.
    */
-  SIMDJSON_VERSION_MAJOR = 1,
+  SIMDJSON_VERSION_MAJOR = 0,
   /**
    * The minor version (major.MINOR.revision) of simdjson being used.
    */
-  SIMDJSON_VERSION_MINOR = 0,
+  SIMDJSON_VERSION_MINOR = 9,
   /**
    * The revision (major.minor.REVISION) of simdjson being used.
    */
-  SIMDJSON_VERSION_REVISION = 2
+  SIMDJSON_VERSION_REVISION = 1
 };
 } // namespace simdjson
 

include/simdjson/westmere/numberparsing.h

@@ -4,7 +4,33 @@
 namespace simdjson {
 namespace SIMDJSON_IMPLEMENTATION {
 namespace {
-
+/**
+ * We could use a scalar version of parse_eight_digits_unrolled that compiles
+ * to something like
+ *
+ *        and     rax, qword ptr [rdi]
+ *        imul    rax, rax, 2561
+ *        shr     rax, 8
+ *        movabs  rcx, 71777214294589695
+ *        and     rcx, rax
+ *        imul    rax, rcx, 6553601
+ *        shr     rax, 16
+ *        movabs  rcx, 281470681808895
+ *        and     rcx, rax
+ *        movabs  rax, 42949672960001
+ *        imul    rax, rcx
+ *        shr     rax, 32
+ *
+ * The vectorized sequence below is favorable, as it compiles to
+ *       vmovq   xmm0, qword ptr [rdi]           # xmm0 = mem[0],zero
+ *       vpaddb  xmm0, xmm0, xmmword ptr [rip + .LCPI0_0]
+ *       vpmaddubsw      xmm0, xmm0, xmmword ptr [rip + .LCPI0_1]
+ *       vpmaddwd        xmm0, xmm0, xmmword ptr [rip + .LCPI0_2]
+ *       vpackusdw       xmm0, xmm0, xmm0
+ *       vpmaddwd        xmm0, xmm0, xmmword ptr [rip + .LCPI0_3]
+ *       vmovd   eax, xmm0
+ * even though it does twice the work and looks far more complicated.
+ */
 static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) {
   // this actually computes *16* values so we are being wasteful.
   const __m128i ascii0 = _mm_set1_epi8('0');
@@ -13,8 +39,10 @@ static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t
   const __m128i mul_1_100 = _mm_setr_epi16(100, 1, 100, 1, 100, 1, 100, 1);
   const __m128i mul_1_10000 =
       _mm_setr_epi16(10000, 1, 10000, 1, 10000, 1, 10000, 1);
+  // _mm_loadu_si128(reinterpret_cast<const __m128i *>(chars)) is faster
+  // but _mm_loadl_epi64(chars) avoids buffer overflows.
   const __m128i input = _mm_sub_epi8(
-      _mm_loadu_si128(reinterpret_cast<const __m128i *>(chars)), ascii0);
+      _mm_loadl_epi64(reinterpret_cast<const __m128i *>(chars)), ascii0);
   const __m128i t1 = _mm_maddubs_epi16(input, mul_1_10);
   const __m128i t2 = _mm_madd_epi16(t1, mul_1_100);
   const __m128i t3 = _mm_packus_epi32(t2, t2);
@@ -23,6 +51,7 @@ static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t
       t4); // only captures the sum of the first 8 digits, drop the rest
 }
 
+
 } // unnamed namespace
 } // namespace SIMDJSON_IMPLEMENTATION
 } // namespace simdjson

include/simdjson/westmere/simd.h

@@ -284,12 +284,11 @@ namespace simd {
       return (this->chunks[0] | this->chunks[1]) | (this->chunks[2] | this->chunks[3]);
     }
 
-    simdjson_really_inline uint64_t compress(uint64_t mask, T * output) const {
+    simdjson_really_inline void compress(uint64_t mask, T * output) const {
       this->chunks[0].compress(uint16_t(mask), output);
       this->chunks[1].compress(uint16_t(mask >> 16), output + 16 - count_ones(mask & 0xFFFF));
       this->chunks[2].compress(uint16_t(mask >> 32), output + 32 - count_ones(mask & 0xFFFFFFFF));
       this->chunks[3].compress(uint16_t(mask >> 48), output + 48 - count_ones(mask & 0xFFFFFFFFFFFF));
-      return 64 - count_ones(mask);
     }
 
     simdjson_really_inline uint64_t to_bitmask() const {

include/simdjson/westmere/stringparsing.h

@@ -11,7 +11,10 @@ using namespace simd;
 struct backslash_and_quote {
 public:
   static constexpr uint32_t BYTES_PROCESSED = 32;
-  simdjson_really_inline static backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst);
+  simdjson_really_inline static backslash_and_quote
+  copy_and_find(const uint8_t *src, uint8_t *dst, const uint8_t *last_full_buf);
+  simdjson_really_inline static backslash_and_quote
+  copy_and_find(const uint8_t *src, uint8_t *dst);
 
   simdjson_really_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; }
   simdjson_really_inline bool has_backslash() { return bs_bits != 0; }
@@ -22,7 +25,21 @@ public:
   uint32_t quote_bits;
 }; // struct backslash_and_quote
 
-simdjson_really_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) {
+simdjson_really_inline backslash_and_quote
+backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst, const uint8_t *last_full_buf) {
+  // If we don't have enough memory left to load a whole simd register, copy it first.
+  uint8_t tmpbuf[BYTES_PROCESSED];
+  if (simdjson_unlikely(src > last_full_buf)) {
+    SIMDJSON_ASSUME(last_full_buf+BYTES_PROCESSED > src);
+    std::memset(tmpbuf, 0, BYTES_PROCESSED);
+    std::memcpy(tmpbuf, src, last_full_buf+BYTES_PROCESSED-src);
+    src = tmpbuf;
+  }
+  return copy_and_find(src, dst);
+}
+
+simdjson_really_inline backslash_and_quote
+backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) {
   // this can read up to 31 bytes beyond the buffer size, but we require
   // SIMDJSON_PADDING of padding
   static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes");

singleheader/amalgamate_demo.cpp

@@ -1,70 +1,35 @@
-#include "simdjson.cpp"
-#include "simdjson.h"
 #include <iostream>
+#include "simdjson.h"
+#include "simdjson.cpp"
 
 int main(int argc, char *argv[]) {
-  if (argc < 2) {
-    std::cerr << "Please specify at least one file name and" << std::endl;
-    std::cerr << "up to two files." << std::endl;
-    std::cerr << "The first file should be a JSON document." << std::endl;
-    std::cerr << "The second file should container many JSON documents."
-              << std::endl;
-    std::cerr << "Try the test files: jsonexamples/twitter.json "
-                 "jsonexamples/amazon_cellphones.ndjson"
-              << std::endl;
+  if(argc < 2) {
+    std::cerr << "Please specify at least one file name. " << std::endl;
     return EXIT_FAILURE;
   }
-  const char *filename = argv[1];
-
-  simdjson::padded_string json;
-  std::cout << "loading: " << filename << std::endl;
-  auto error = simdjson::padded_string::load(filename).get(json);
+  const char * filename = argv[1];
+  simdjson::dom::parser parser;
+  simdjson::dom::element elem;
+  auto error = parser.load(filename).get(elem); // do the parsing
   if (error) {
-    std::cout << "could not load the file " << filename << std::endl;
+    std::cout << "parse failed" << std::endl;
     std::cout << "error code: " << error << std::endl;
+    std::cout << error << std::endl;
     return EXIT_FAILURE;
   } else {
-    std::cout << "loaded: " << json.size() << " bytes." << std::endl;
+    std::cout << "parse valid: " << elem << std::endl;
   }
-  simdjson::ondemand::parser parser;
-  simdjson::ondemand::document doc;
-  error = parser.iterate(json).get(doc);
-  if (error) {
-    std::cout << error << std::endl;
-    return EXIT_FAILURE;
-  }
-  simdjson::ondemand::json_type type;
-  error = doc.type().get(type);
-  if (error) {
-    std::cout << error << std::endl;
-    return EXIT_FAILURE;
-  }
-  std::cout << "if valid, the document has the following type at the root: " << type
-            << std::endl;
-
-  if (argc == 2) {
+  if(argc == 2) {
     return EXIT_SUCCESS;
   }
 
-  // iterate_many
-  const char *filename2 = argv[2];
-  std::cout << "loading: " << filename2 << std::endl;
-  simdjson::padded_string json2;
-  error = simdjson::padded_string::load(filename2).get(json2);
-  if (error) {
-    std::cout << "could not load the file " << filename2 << std::endl;
-    std::cout << "error code: " << error << std::endl;
-    return EXIT_FAILURE;
-  } else {
-    std::cout << "loaded: " << json2.size() << " bytes." << std::endl;
-  }
-  simdjson::ondemand::document_stream stream;
-  error = parser.iterate_many(json2).get(stream);
-  size_t counter{0};
+  // parse_many
+  const char * filename2 = argv[2];
+  simdjson::dom::document_stream stream;
+  error = parser.load_many(filename2).get(stream);
   if (!error) {
     for (auto result : stream) {
       error = result.error();
-      counter++;
     }
   }
   if (error) {
@@ -73,9 +38,8 @@ int main(int argc, char *argv[]) {
     std::cout << error << std::endl;
     return EXIT_FAILURE;
   } else {
-    std::cout << "I found " << counter << " potential JSON documents." << std::endl;
+    std::cout << "parse_many valid" << std::endl;
   }
-  std::cout << "For more information on how simdjson works, please refer to our documentation." << std::endl;
-  std::cout << "https://github.com/simdjson/simdjson/blob/master/doc/basics.md" << std::endl;
   return EXIT_SUCCESS;
 }
+

singleheader/simdjson.cpp

@@ -1,4 +1,4 @@
-/* auto-generated on 2021-10-27 19:25:23 -0400. Do not edit! */
+/* auto-generated on 2021-07-23 23:06:32 -0400. Do not edit! */
 /* begin file src/simdjson.cpp */
 #include "simdjson.h"
 
@@ -1558,38 +1558,43 @@ double from_chars(const char *first, const char *end) noexcept {
 
 namespace simdjson {
 namespace internal {
+  /**
+   * We include in the error messages the name of the enum
+   * as a quality-of-life feature for programmers who receive
+   * the error. If they get a text error, they sometimes want
+   * to trace back the error code, and it is handy if it is
+   * right there.
+   */
 
   SIMDJSON_DLLIMPORTEXPORT const error_code_info error_codes[] {
-    { SUCCESS, "No error" },
-    { CAPACITY, "This parser can't support a document that big" },
-    { MEMALLOC, "Error allocating memory, we're most likely out of memory" },
-    { TAPE_ERROR, "The JSON document has an improper structure: missing or superfluous commas, braces, missing keys, etc." },
-    { DEPTH_ERROR, "The JSON document was too deep (too many nested objects and arrays)" },
-    { STRING_ERROR, "Problem while parsing a string" },
-    { T_ATOM_ERROR, "Problem while parsing an atom starting with the letter 't'" },
-    { F_ATOM_ERROR, "Problem while parsing an atom starting with the letter 'f'" },
-    { N_ATOM_ERROR, "Problem while parsing an atom starting with the letter 'n'" },
-    { NUMBER_ERROR, "Problem while parsing a number" },
-    { UTF8_ERROR, "The input is not valid UTF-8" },
-    { UNINITIALIZED, "Uninitialized" },
-    { EMPTY, "Empty: no JSON found" },
-    { UNESCAPED_CHARS, "Within strings, some characters must be escaped, we found unescaped characters" },
-    { UNCLOSED_STRING, "A string is opened, but never closed." },
-    { UNSUPPORTED_ARCHITECTURE, "simdjson does not have an implementation supported by this CPU architecture (perhaps it's a non-SIMD CPU?)." },
-    { INCORRECT_TYPE, "The JSON element does not have the requested type." },
-    { NUMBER_OUT_OF_RANGE, "The JSON number is too large or too small to fit within the requested type." },
-    { INDEX_OUT_OF_BOUNDS, "Attempted to access an element of a JSON array that is beyond its length." },
-    { NO_SUCH_FIELD, "The JSON field referenced does not exist in this object." },
-    { IO_ERROR, "Error reading the file." },
-    { INVALID_JSON_POINTER, "Invalid JSON pointer syntax." },
-    { INVALID_URI_FRAGMENT, "Invalid URI fragment syntax." },
-    { UNEXPECTED_ERROR, "Unexpected error, consider reporting this problem as you may have found a bug in simdjson" },
-    { PARSER_IN_USE, "Cannot parse a new document while a document is still in use." },
-    { OUT_OF_ORDER_ITERATION, "Objects and arrays can only be iterated when they are first encountered." },
-    { INSUFFICIENT_PADDING, "simdjson requires the input JSON string to have at least SIMDJSON_PADDING extra bytes allocated, beyond the string's length. Consider using the simdjson::padded_string class if needed." },
-    { INCOMPLETE_ARRAY_OR_OBJECT, "JSON document ended early in the middle of an object or array." },
-    { SCALAR_DOCUMENT_AS_VALUE, "A JSON document made of a scalar (number, Boolean, null or string) is treated as a value. Use get_bool(), get_double(), etc. on the document instead. "},
-    { OUT_OF_BOUNDS, "Attempted to access location outside of document."}
+    { SUCCESS, "No error (SUCCESS)" },
+    { CAPACITY, "This parser can't support a document that big (CAPACITY)" },
+    { MEMALLOC, "Error allocating memory, we're most likely out of memory (MEMALLOC)" },
+    { TAPE_ERROR, "The JSON document has an improper structure: missing or superfluous commas, braces, missing keys, etc. (TAPE_ERROR)" },
+    { DEPTH_ERROR, "The JSON document was too deep (too many nested objects and arrays) (DEPTH_ERROR)" },
+    { STRING_ERROR, "Problem while parsing a string (STRING_ERROR)" },
+    { T_ATOM_ERROR, "Problem while parsing an atom starting with the letter 't' (T_ATOM_ERROR)" },
+    { F_ATOM_ERROR, "Problem while parsing an atom starting with the letter 'f' (F_ATOM_ERROR)" },
+    { N_ATOM_ERROR, "Problem while parsing an atom starting with the letter 'n' (N_ATOM_ERROR)" },
+    { NUMBER_ERROR, "Problem while parsing a number (NUMBER_ERROR)" },
+    { UTF8_ERROR, "The input is not valid UTF-8 (UTF8_ERROR)" },
+    { UNINITIALIZED, "Uninitialized (UNINITIALIZED)" },
+    { EMPTY, "Empty: no JSON found (EMPTY)" },
+    { UNESCAPED_CHARS, "Within strings, some characters must be escaped, we found unescaped characters (UNESCAPED_CHARS)" },
+    { UNCLOSED_STRING, "A string is opened, but never closed (UNCLOSED_STRING)" },
+    { UNSUPPORTED_ARCHITECTURE, "simdjson does not have an implementation supported by this CPU architecture (UNSUPPORTED_ARCHITECTURE)" },
+    { INCORRECT_TYPE, "The JSON element does not have the requested type (INCORRECT_TYPE)" },
+    { NUMBER_OUT_OF_RANGE, "The JSON number is too large or too small to fit within the requested type (NUMBER_OUT_OF_RANGE)" },
+    { INDEX_OUT_OF_BOUNDS, "Attempted to access an element of a JSON array that is beyond its length (INDEX_OUT_OF_BOUNDS)" },
+    { NO_SUCH_FIELD, "The JSON field referenced does not exist in this object (NO_SUCH_FIELD)" },
+    { IO_ERROR, "Error reading the file (IO_ERROR)" },
+    { INVALID_JSON_POINTER, "Invalid JSON pointer syntax (INVALID_JSON_POINTER)" },
+    { INVALID_URI_FRAGMENT, "Invalid URI fragment syntax (INVALID_URI_FRAGMENT)" },
+    { UNEXPECTED_ERROR, "Unexpected error, consider reporting this problem as you may have found a bug in simdjson (UNEXPECTED_ERROR)" },
+    { PARSER_IN_USE, "Cannot parse a new document while a document is still in use (PARSER_IN_USE)" },
+    { OUT_OF_ORDER_ITERATION, "Objects and arrays can only be iterated when they are first encountered (OUT_OF_ORDER_ITERATION)" },
+    { INSUFFICIENT_PADDING, "simdjson requires the input JSON string to have at least SIMDJSON_PADDING extra bytes allocated, beyond the string's length (INSUFFICIENT_PADDING)" },
+    { INCOMPLETE_ARRAY_OR_OBJECT, "JSON document ended early in the middle of an object or array (INCOMPLETE_ARRAY_OR_OBJECT)" }
   }; // error_messages[]
 
 } // namespace internal
@@ -3535,7 +3540,8 @@ private:
 
 simdjson_really_inline void json_minifier::next(const simd::simd8x64<uint8_t>& in, const json_block& block) {
   uint64_t mask = block.whitespace();
-  dst += in.compress(mask, dst);
+  in.compress(mask, dst);
+  dst += 64 - count_ones(mask);
 }
 
 simdjson_really_inline error_code json_minifier::finish(uint8_t *dst_start, size_t &dst_len) {
@@ -4734,7 +4740,7 @@ simdjson_really_inline tape_builder::tape_builder(dom::document &doc) noexcept :
 simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept {
   iter.log_value(key ? "key" : "string");
   uint8_t *dst = on_start_string(iter);
-  dst = stringparsing::parse_string(value+1, dst);
+  dst = stringparsing::parse_string(value+1, dst, iter.buf+iter.dom_parser.len);
   if (dst == nullptr) {
     iter.log_error("Invalid escape in string");
     return STRING_ERROR;
@@ -4749,34 +4755,23 @@ simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_
 
 simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept {
   iter.log_value("number");
-  return numberparsing::parse_number(value, tape);
+#ifdef SIMDJSON_SWAR_NUMBER_PARSING
+  return numberparsing::parse_number(value, tape, value + iter.remaining_len());
+#else
+  return numberparsing::parse_number(value, tape, nullptr);
+#endif
 }
 
 simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept {
-  //
-  // We need to make a copy to make sure that the string is space terminated.
-  // This is not about padding the input, which should already padded up
-  // to len + SIMDJSON_PADDING. However, we have no control at this stage
-  // on how the padding was done. What if the input string was padded with nulls?
-  // It is quite common for an input string to have an extra null character (C string).
-  // We do not want to allow 9\0 (where \0 is the null character) inside a JSON
-  // document, but the string "9\0" by itself is fine. So we make a copy and
-  // pad the input with spaces when we know that there is just one input element.
-  // This copy is relatively expensive, but it will almost never be called in
-  // practice unless you are in the strange scenario where you have many JSON
-  // documents made of single atoms.
-  //
-  std::unique_ptr<uint8_t[]>copy(new (std::nothrow) uint8_t[iter.remaining_len() + SIMDJSON_PADDING]);
-  if (copy.get() == nullptr) { return MEMALLOC; }
-  std::memcpy(copy.get(), value, iter.remaining_len());
-  std::memset(copy.get() + iter.remaining_len(), ' ', SIMDJSON_PADDING);
-  error_code error = visit_number(iter, copy.get());
-  return error;
+  // Root numbers will not have another token after them, so we need to be careful not to step
+  // off the edge of the buffer.
+  iter.log_value("number");
+  return numberparsing::parse_number(value, tape, value + iter.remaining_len());
 }
 
 simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept {
   iter.log_value("true");
-  if (!atomparsing::is_valid_true_atom(value)) { return T_ATOM_ERROR; }
+  if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; }
   tape.append(0, internal::tape_type::TRUE_VALUE);
   return SUCCESS;
 }
@@ -4790,7 +4785,7 @@ simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_
 
 simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept {
   iter.log_value("false");
-  if (!atomparsing::is_valid_false_atom(value)) { return F_ATOM_ERROR; }
+  if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; }
   tape.append(0, internal::tape_type::FALSE_VALUE);
   return SUCCESS;
 }
@@ -4804,7 +4799,7 @@ simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_
 
 simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept {
   iter.log_value("null");
-  if (!atomparsing::is_valid_null_atom(value)) { return N_ATOM_ERROR; }
+  if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; }
   tape.append(0, internal::tape_type::NULL_VALUE);
   return SUCCESS;
 }
@@ -6062,7 +6057,7 @@ simdjson_really_inline tape_builder::tape_builder(dom::document &doc) noexcept :
 simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept {
   iter.log_value(key ? "key" : "string");
   uint8_t *dst = on_start_string(iter);
-  dst = stringparsing::parse_string(value+1, dst);
+  dst = stringparsing::parse_string(value+1, dst, iter.buf+iter.dom_parser.len);
   if (dst == nullptr) {
     iter.log_error("Invalid escape in string");
     return STRING_ERROR;
@@ -6077,34 +6072,23 @@ simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_
 
 simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept {
   iter.log_value("number");
-  return numberparsing::parse_number(value, tape);
+#ifdef SIMDJSON_SWAR_NUMBER_PARSING
+  return numberparsing::parse_number(value, tape, value + iter.remaining_len());
+#else
+  return numberparsing::parse_number(value, tape, nullptr);
+#endif
 }
 
 simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept {
-  //
-  // We need to make a copy to make sure that the string is space terminated.
-  // This is not about padding the input, which should already padded up
-  // to len + SIMDJSON_PADDING. However, we have no control at this stage
-  // on how the padding was done. What if the input string was padded with nulls?
-  // It is quite common for an input string to have an extra null character (C string).
-  // We do not want to allow 9\0 (where \0 is the null character) inside a JSON
-  // document, but the string "9\0" by itself is fine. So we make a copy and
-  // pad the input with spaces when we know that there is just one input element.
-  // This copy is relatively expensive, but it will almost never be called in
-  // practice unless you are in the strange scenario where you have many JSON
-  // documents made of single atoms.
-  //
-  std::unique_ptr<uint8_t[]>copy(new (std::nothrow) uint8_t[iter.remaining_len() + SIMDJSON_PADDING]);
-  if (copy.get() == nullptr) { return MEMALLOC; }
-  std::memcpy(copy.get(), value, iter.remaining_len());
-  std::memset(copy.get() + iter.remaining_len(), ' ', SIMDJSON_PADDING);
-  error_code error = visit_number(iter, copy.get());
-  return error;
+  // Root numbers will not have another token after them, so we need to be careful not to step
+  // off the edge of the buffer.
+  iter.log_value("number");
+  return numberparsing::parse_number(value, tape, value + iter.remaining_len());
 }
 
 simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept {
   iter.log_value("true");
-  if (!atomparsing::is_valid_true_atom(value)) { return T_ATOM_ERROR; }
+  if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; }
   tape.append(0, internal::tape_type::TRUE_VALUE);
   return SUCCESS;
 }
@@ -6118,7 +6102,7 @@ simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_
 
 simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept {
   iter.log_value("false");
-  if (!atomparsing::is_valid_false_atom(value)) { return F_ATOM_ERROR; }
+  if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; }
   tape.append(0, internal::tape_type::FALSE_VALUE);
   return SUCCESS;
 }
@@ -6132,7 +6116,7 @@ simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_
 
 simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept {
   iter.log_value("null");
-  if (!atomparsing::is_valid_null_atom(value)) { return N_ATOM_ERROR; }
+  if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; }
   tape.append(0, internal::tape_type::NULL_VALUE);
   return SUCCESS;
 }
@@ -6999,7 +6983,8 @@ private:
 
 simdjson_really_inline void json_minifier::next(const simd::simd8x64<uint8_t>& in, const json_block& block) {
   uint64_t mask = block.whitespace();
-  dst += in.compress(mask, dst);
+  in.compress(mask, dst);
+  dst += 64 - count_ones(mask);
 }
 
 simdjson_really_inline error_code json_minifier::finish(uint8_t *dst_start, size_t &dst_len) {
@@ -8197,7 +8182,7 @@ simdjson_really_inline tape_builder::tape_builder(dom::document &doc) noexcept :
 simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept {
   iter.log_value(key ? "key" : "string");
   uint8_t *dst = on_start_string(iter);
-  dst = stringparsing::parse_string(value+1, dst);
+  dst = stringparsing::parse_string(value+1, dst, iter.buf+iter.dom_parser.len);
   if (dst == nullptr) {
     iter.log_error("Invalid escape in string");
     return STRING_ERROR;
@@ -8212,34 +8197,23 @@ simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_
 
 simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept {
   iter.log_value("number");
-  return numberparsing::parse_number(value, tape);
+#ifdef SIMDJSON_SWAR_NUMBER_PARSING
+  return numberparsing::parse_number(value, tape, value + iter.remaining_len());
+#else
+  return numberparsing::parse_number(value, tape, nullptr);
+#endif
 }
 
 simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept {
-  //
-  // We need to make a copy to make sure that the string is space terminated.
-  // This is not about padding the input, which should already padded up
-  // to len + SIMDJSON_PADDING. However, we have no control at this stage
-  // on how the padding was done. What if the input string was padded with nulls?
-  // It is quite common for an input string to have an extra null character (C string).
-  // We do not want to allow 9\0 (where \0 is the null character) inside a JSON
-  // document, but the string "9\0" by itself is fine. So we make a copy and
-  // pad the input with spaces when we know that there is just one input element.
-  // This copy is relatively expensive, but it will almost never be called in
-  // practice unless you are in the strange scenario where you have many JSON
-  // documents made of single atoms.
-  //
-  std::unique_ptr<uint8_t[]>copy(new (std::nothrow) uint8_t[iter.remaining_len() + SIMDJSON_PADDING]);
-  if (copy.get() == nullptr) { return MEMALLOC; }
-  std::memcpy(copy.get(), value, iter.remaining_len());
-  std::memset(copy.get() + iter.remaining_len(), ' ', SIMDJSON_PADDING);
-  error_code error = visit_number(iter, copy.get());
-  return error;
+  // Root numbers will not have another token after them, so we need to be careful not to step
+  // off the edge of the buffer.
+  iter.log_value("number");
+  return numberparsing::parse_number(value, tape, value + iter.remaining_len());
 }
 
 simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept {
   iter.log_value("true");
-  if (!atomparsing::is_valid_true_atom(value)) { return T_ATOM_ERROR; }
+  if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; }
   tape.append(0, internal::tape_type::TRUE_VALUE);
   return SUCCESS;
 }
@@ -8253,7 +8227,7 @@ simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_
 
 simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept {
   iter.log_value("false");
-  if (!atomparsing::is_valid_false_atom(value)) { return F_ATOM_ERROR; }
+  if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; }
   tape.append(0, internal::tape_type::FALSE_VALUE);
   return SUCCESS;
 }
@@ -8267,7 +8241,7 @@ simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_
 
 simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept {
   iter.log_value("null");
-  if (!atomparsing::is_valid_null_atom(value)) { return N_ATOM_ERROR; }
+  if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; }
   tape.append(0, internal::tape_type::NULL_VALUE);
   return SUCCESS;
 }
@@ -9125,7 +9099,8 @@ private:
 
 simdjson_really_inline void json_minifier::next(const simd::simd8x64<uint8_t>& in, const json_block& block) {
   uint64_t mask = block.whitespace();
-  dst += in.compress(mask, dst);
+  in.compress(mask, dst);
+  dst += 64 - count_ones(mask);
 }
 
 simdjson_really_inline error_code json_minifier::finish(uint8_t *dst_start, size_t &dst_len) {
@@ -10324,7 +10299,7 @@ simdjson_really_inline tape_builder::tape_builder(dom::document &doc) noexcept :
 simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept {
   iter.log_value(key ? "key" : "string");
   uint8_t *dst = on_start_string(iter);
-  dst = stringparsing::parse_string(value+1, dst);
+  dst = stringparsing::parse_string(value+1, dst, iter.buf+iter.dom_parser.len);
   if (dst == nullptr) {
     iter.log_error("Invalid escape in string");
     return STRING_ERROR;
@@ -10339,34 +10314,23 @@ simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_
 
 simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept {
   iter.log_value("number");
-  return numberparsing::parse_number(value, tape);
+#ifdef SIMDJSON_SWAR_NUMBER_PARSING
+  return numberparsing::parse_number(value, tape, value + iter.remaining_len());
+#else
+  return numberparsing::parse_number(value, tape, nullptr);
+#endif
 }
 
 simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept {
-  //
-  // We need to make a copy to make sure that the string is space terminated.
-  // This is not about padding the input, which should already padded up
-  // to len + SIMDJSON_PADDING. However, we have no control at this stage
-  // on how the padding was done. What if the input string was padded with nulls?
-  // It is quite common for an input string to have an extra null character (C string).
-  // We do not want to allow 9\0 (where \0 is the null character) inside a JSON
-  // document, but the string "9\0" by itself is fine. So we make a copy and
-  // pad the input with spaces when we know that there is just one input element.
-  // This copy is relatively expensive, but it will almost never be called in
-  // practice unless you are in the strange scenario where you have many JSON
-  // documents made of single atoms.
-  //
-  std::unique_ptr<uint8_t[]>copy(new (std::nothrow) uint8_t[iter.remaining_len() + SIMDJSON_PADDING]);
-  if (copy.get() == nullptr) { return MEMALLOC; }
-  std::memcpy(copy.get(), value, iter.remaining_len());
-  std::memset(copy.get() + iter.remaining_len(), ' ', SIMDJSON_PADDING);
-  error_code error = visit_number(iter, copy.get());
-  return error;
+  // Root numbers will not have another token after them, so we need to be careful not to step
+  // off the edge of the buffer.
+  iter.log_value("number");
+  return numberparsing::parse_number(value, tape, value + iter.remaining_len());
 }
 
 simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept {
   iter.log_value("true");
-  if (!atomparsing::is_valid_true_atom(value)) { return T_ATOM_ERROR; }
+  if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; }
   tape.append(0, internal::tape_type::TRUE_VALUE);
   return SUCCESS;
 }
@@ -10380,7 +10344,7 @@ simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_
 
 simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept {
   iter.log_value("false");
-  if (!atomparsing::is_valid_false_atom(value)) { return F_ATOM_ERROR; }
+  if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; }
   tape.append(0, internal::tape_type::FALSE_VALUE);
   return SUCCESS;
 }
@@ -10394,7 +10358,7 @@ simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_
 
 simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept {
   iter.log_value("null");
-  if (!atomparsing::is_valid_null_atom(value)) { return N_ATOM_ERROR; }
+  if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; }
   tape.append(0, internal::tape_type::NULL_VALUE);
   return SUCCESS;
 }
@@ -11287,7 +11251,8 @@ private:
 
 simdjson_really_inline void json_minifier::next(const simd::simd8x64<uint8_t>& in, const json_block& block) {
   uint64_t mask = block.whitespace();
-  dst += in.compress(mask, dst);
+  in.compress(mask, dst);
+  dst += 64 - count_ones(mask);
 }
 
 simdjson_really_inline error_code json_minifier::finish(uint8_t *dst_start, size_t &dst_len) {
@@ -12485,7 +12450,7 @@ simdjson_really_inline tape_builder::tape_builder(dom::document &doc) noexcept :
 simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept {
   iter.log_value(key ? "key" : "string");
   uint8_t *dst = on_start_string(iter);
-  dst = stringparsing::parse_string(value+1, dst);
+  dst = stringparsing::parse_string(value+1, dst, iter.buf+iter.dom_parser.len);
   if (dst == nullptr) {
     iter.log_error("Invalid escape in string");
     return STRING_ERROR;
@@ -12500,34 +12465,23 @@ simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_
 
 simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept {
   iter.log_value("number");
-  return numberparsing::parse_number(value, tape);
+#ifdef SIMDJSON_SWAR_NUMBER_PARSING
+  return numberparsing::parse_number(value, tape, value + iter.remaining_len());
+#else
+  return numberparsing::parse_number(value, tape, nullptr);
+#endif
 }
 
 simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept {
-  //
-  // We need to make a copy to make sure that the string is space terminated.
-  // This is not about padding the input, which should already padded up
-  // to len + SIMDJSON_PADDING. However, we have no control at this stage
-  // on how the padding was done. What if the input string was padded with nulls?
-  // It is quite common for an input string to have an extra null character (C string).
-  // We do not want to allow 9\0 (where \0 is the null character) inside a JSON
-  // document, but the string "9\0" by itself is fine. So we make a copy and
-  // pad the input with spaces when we know that there is just one input element.
-  // This copy is relatively expensive, but it will almost never be called in
-  // practice unless you are in the strange scenario where you have many JSON
-  // documents made of single atoms.
-  //
-  std::unique_ptr<uint8_t[]>copy(new (std::nothrow) uint8_t[iter.remaining_len() + SIMDJSON_PADDING]);
-  if (copy.get() == nullptr) { return MEMALLOC; }
-  std::memcpy(copy.get(), value, iter.remaining_len());
-  std::memset(copy.get() + iter.remaining_len(), ' ', SIMDJSON_PADDING);
-  error_code error = visit_number(iter, copy.get());
-  return error;
+  // Root numbers will not have another token after them, so we need to be careful not to step
+  // off the edge of the buffer.
+  iter.log_value("number");
+  return numberparsing::parse_number(value, tape, value + iter.remaining_len());
 }
 
 simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept {
   iter.log_value("true");
-  if (!atomparsing::is_valid_true_atom(value)) { return T_ATOM_ERROR; }
+  if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; }
   tape.append(0, internal::tape_type::TRUE_VALUE);
   return SUCCESS;
 }
@@ -12541,7 +12495,7 @@ simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_
 
 simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept {
   iter.log_value("false");
-  if (!atomparsing::is_valid_false_atom(value)) { return F_ATOM_ERROR; }
+  if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; }
   tape.append(0, internal::tape_type::FALSE_VALUE);
   return SUCCESS;
 }
@@ -12555,7 +12509,7 @@ simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_
 
 simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept {
   iter.log_value("null");
-  if (!atomparsing::is_valid_null_atom(value)) { return N_ATOM_ERROR; }
+  if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; }
   tape.append(0, internal::tape_type::NULL_VALUE);
   return SUCCESS;
 }

src/generic/stage1/json_minifier.h

@@ -27,7 +27,8 @@ private:
 
 simdjson_really_inline void json_minifier::next(const simd::simd8x64<uint8_t>& in, const json_block& block) {
   uint64_t mask = block.whitespace();
-  dst += in.compress(mask, dst);
+  in.compress(mask, dst);
+  dst += 64 - count_ones(mask);
 }
 
 simdjson_really_inline error_code json_minifier::finish(uint8_t *dst_start, size_t &dst_len) {

src/generic/stage2/json_iterator.h

@@ -241,9 +241,11 @@ simdjson_really_inline json_iterator::json_iterator(dom_parser_implementation &_
 }
 
 simdjson_really_inline const uint8_t *json_iterator::peek() const noexcept {
+  if(at_eof()) { return reinterpret_cast<const uint8_t *>(" ");}
   return &buf[*(next_structural)];
 }
 simdjson_really_inline const uint8_t *json_iterator::advance() noexcept {
+  if(at_eof()) { return reinterpret_cast<const uint8_t *>(" ");}
   return &buf[*(next_structural++)];
 }
 simdjson_really_inline size_t json_iterator::remaining_len() const noexcept {