This cleans a bit the current code, especially with respect to EOF guards. (#1669)

* Upgrading the GitHub Actions.

* Upgrading appveyor

* Upgrading circle ci.

* Cleaning.

.appveyor.yml

@@ -15,22 +15,22 @@ environment:
     - job_name: VS2019
       CMAKE_ARGS: -A %Platform%
     - job_name: VS2019ARM
-      CMAKE_ARGS: -A ARM64 -DCMAKE_CROSSCOMPILING=1 -D SIMDJSON_GOOGLE_BENCHMARKS=OFF # Does Google Benchmark builds under VS ARM?
+      CMAKE_ARGS: -A ARM64   -DSIMDJSON_DEVELOPER_MODE=ON -DCMAKE_CROSSCOMPILING=1 -D SIMDJSON_GOOGLE_BENCHMARKS=OFF # Does Google Benchmark builds under VS ARM?
     - job_name: VS2017 (Static, No Threads)
       image: Visual Studio 2017
-      CMAKE_ARGS: -A %Platform% -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_ENABLE_THREADS=OFF
+      CMAKE_ARGS: -A %Platform%   -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_ENABLE_THREADS=OFF
       CTEST_ARGS: -LE explicitonly
     - job_name: VS2019 (Win32)
       platform: Win32
-      CMAKE_ARGS: -A %Platform% -DBUILD_SHARED_LIBS=ON -DSIMDJSON_ENABLE_THREADS=ON # This should be the default. Testing anyway.
+      CMAKE_ARGS: -A %Platform%   -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=ON -DSIMDJSON_ENABLE_THREADS=ON # This should be the default. Testing anyway.
       CTEST_ARGS: -LE explicitonly
     - job_name: VS2019 (Win32, No Exceptions)
       platform: Win32
-      CMAKE_ARGS: -A %Platform% -DBUILD_SHARED_LIBS=ON -DSIMDJSON_ENABLE_THREADS=ON -DSIMDJSON_EXCEPTIONS=OFF
+      CMAKE_ARGS: -A %Platform%   -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=ON -DSIMDJSON_ENABLE_THREADS=ON -DSIMDJSON_EXCEPTIONS=OFF
       CTEST_ARGS: -LE explicitonly
     - job_name: VS2015
       image: Visual Studio 2015
-      CMAKE_ARGS: -A %Platform% -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_ENABLE_THREADS=OFF
+      CMAKE_ARGS: -A %Platform%   -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_ENABLE_THREADS=OFF
       CTEST_ARGS: -LE explicitonly
 
 build_script:

.circleci/config.yml

@@ -1,7 +1,7 @@
 version: 2.1
 
 
-# We constantly run out of memory so please do not use parallelism (-j, -j4). 
+# We constantly run out of memory so please do not use parallelism (-j, -j4).
 
 # Reusable image / compiler definitions
 executors:
@@ -48,7 +48,7 @@ executors:
           CXX: clang++-9
           CC: clang-9
           CMAKE_BUILD_FLAGS:
-          CTEST_FLAGS: --output-on-failure 
+          CTEST_FLAGS: --output-on-failure
 
   clang6:
     docker:
@@ -88,13 +88,13 @@ commands:
     steps:
       - cmake_prep
       - dependency_restore
-      - run: cmake $CMAKE_FLAGS -DCMAKE_INSTALL_PREFIX:PATH=destination -B build . 
+      - run: cmake  -DSIMDJSON_DEVELOPER_MODE=ON $CMAKE_FLAGS -DCMAKE_INSTALL_PREFIX:PATH=destination -B build .
       - dependency_cache # dependencies are produced in the configure step
 
   cmake_build:
     steps:
       - cmake_build_cache
-      - run: cmake --build build
+      - run: cmake  --build build
 
   cmake_test:
     steps:
@@ -130,7 +130,7 @@ commands:
       - cmake_build_cache
       - run: |
           cmake -DSIMDJSON_ENABLE_DOM_CHECKPERF=ON --build  build --target checkperf &&
-          cd build && 
+          cd build &&
           ctest --output-on-failure -R checkperf
 
   # we not only want cmake to build and run tests, but we want also a successful installation from which we can build, link and run programs
@@ -191,12 +191,12 @@ jobs:
   sanitize-gcc10:
     description: Build and run tests on GCC 10 and AVX 2 with a cmake sanitize build
     executor: gcc10
-    environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly }
+    environment: { CMAKE_FLAGS: -DCMAKE_BUILD_TYPE=Debug -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly }
     steps: [ cmake_test ]
   sanitize-clang10:
     description: Build and run tests on clang 10 and AVX 2 with a cmake sanitize build
     executor: clang10
-    environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly }
+    environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_NO_FORCE_INLINING=ON -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly }
     steps: [ cmake_test ]
   threadsanitize-gcc10:
     description: Build and run tests on GCC 10 and AVX 2 with a cmake sanitize build
@@ -206,15 +206,7 @@ jobs:
   threadsanitize-clang10:
     description: Build and run tests on clang 10 and AVX 2 with a cmake sanitize build
     executor: clang10
-    environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE_THREADS=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly }
-    steps: [ cmake_test ]
-  nocheckeof-clang10:
-    description: Validate that when __SIMDJSON_CHECK_EOF=0, everything still succeeds
-    environment:
-      CXXFLAGS: -D__SIMDJSON_CHECK_EOF=0
-      CMAKE_BUILD_FLAGS: --target ondemand_tests
-      CTEST_FLAGS: --output-on-failure -R ondemand_
-    executor: clang10
+    environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON  -DSIMDJSON_NO_FORCE_INLINING=ON -DSIMDJSON_SANITIZE_THREADS=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly }
     steps: [ cmake_test ]
   # dynamic
   dynamic-gcc10:
@@ -270,12 +262,12 @@ jobs:
   sanitize-haswell-gcc10:
     description: Build and run tests on GCC 10 and AVX 2 with a cmake sanitize build
     executor: gcc10
-    environment: {  CXXFLAGS: -march=haswell, CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly }
+    environment: {  CXXFLAGS: -march=haswell, CMAKE_FLAGS: -DCMAKE_BUILD_TYPE=Debug -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly }
     steps: [ cmake_test ]
   sanitize-haswell-clang10:
     description: Build and run tests on clang 10 and AVX 2 with a cmake sanitize build
     executor: clang10
-    environment: { CXXFLAGS: -march=haswell, CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly }
+    environment: { CXXFLAGS: -march=haswell, CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON  -DSIMDJSON_NO_FORCE_INLINING=ON -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly }
     steps: [ cmake_test ]
 
 workflows:

.github/workflows/alpine.yml

@@ -1,12 +1,6 @@
 name: Alpine Linux
 
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
+on: [push, pull_request]
 
 jobs:
   ubuntu-build:

.github/workflows/fix-trailing-whitespace.yml

@@ -1,9 +1,6 @@
 name: Detect trailing whitespace
 
-on:
-  pull_request:
-    branches:
-      - master
+on: [push, pull_request]
 
 jobs:
   whitespace:

.github/workflows/mingw-ci.yml

@@ -1,12 +1,7 @@
 name: MinGW32-CI
 
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
+on: [push, pull_request]
+
 
 # Important: scoop will either install 32-bit GCC or 64-bit GCC, not both.
 
@@ -47,7 +42,7 @@ jobs:
         run: |
           Invoke-Expression (New-Object System.Net.WebClient).DownloadString('https://get.scoop.sh')
           scoop install sudo --global
-          sudo scoop install git --global 
+          sudo scoop install git --global
           sudo scoop install ninja --global
           sudo scoop install cmake --global
           sudo scoop install gcc --arch 32bit --global

.github/workflows/mingw64-ci.yml

@@ -1,12 +1,7 @@
 name: MinGW64-CI
 
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
+on: [push, pull_request]
+
 
 # Important: scoop will either install 32-bit GCC or 64-bit GCC, not both.
 

.github/workflows/msys2-clang.yml

@@ -1,12 +1,7 @@
 name: MSYS2-CLANG-CI
 
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
+on: [push, pull_request]
+
 
 jobs:
   windows-mingw:

.github/workflows/msys2.yml

@@ -1,12 +1,6 @@
 name: MSYS2-CI
 
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
+on: [push, pull_request]
 
 jobs:
   windows-mingw:

.github/workflows/ubuntu18-threadsani.yml

@@ -1,12 +1,7 @@
 name: Ubuntu 18.04 CI (GCC 7) with Thread Sanitizer
 
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
+on: [push, pull_request]
+
 
 jobs:
   ubuntu-build:

.github/workflows/ubuntu18.yml

@@ -1,12 +1,6 @@
 name: Ubuntu 18.04 CI (GCC 7)
 
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
+on: [push, pull_request]
 
 jobs:
   ubuntu-build:

.github/workflows/ubuntu20-noexcept.yml

@@ -1,12 +1,6 @@
 name: Ubuntu 20.04 CI (GCC 9) without exceptions
 
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
+on: [push, pull_request]
 
 jobs:
   ubuntu-build:

.github/workflows/ubuntu20-nothread.yml

@@ -1,12 +1,6 @@
-name: Ubuntu 20.04 CI (GCC 9) without threads
+name: Ubuntu 20.04 CI (GCC 9) Without Threads
 
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
+on: [push, pull_request]
 
 jobs:
   ubuntu-build:

.github/workflows/ubuntu20-sani.yml

@@ -0,0 +1,23 @@
+name: Ubuntu 20.04 CI (GCC 9) With Memory Sanitizer
+
+on: [push, pull_request]
+
+jobs:
+  ubuntu-build:
+    if: >-
+      ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
+      ! contains(toJSON(github.event.commits.*.message), '[skip github]')
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/cache@v2
+        with:
+          path: dependencies/.cache
+          key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
+      - name: Use cmake
+        run: |
+          mkdir builddebug &&
+          cd builddebug &&
+          cmake -DSIMDJSON_SANITIZE=ON -DCMAKE_BUILD_TYPE=Debug -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF ..  &&
+          cmake --build .   &&
+          ctest -j --output-on-failure -LE explicitonly

.github/workflows/ubuntu20-threadsani.yml

@@ -1,12 +1,6 @@
 name: Ubuntu 20.04 CI (GCC 9) with Thread Sanitizer
 
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
+on: [push, pull_request]
 
 jobs:
   ubuntu-build:

.github/workflows/ubuntu20.yml

@@ -1,12 +1,6 @@
 name: Ubuntu 20.04 CI (GCC 9)
 
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
+on: [push, pull_request]
 
 jobs:
   ubuntu-build:

.github/workflows/vs15-ci.yml

@@ -1,12 +1,6 @@
 name: VS15-CI
 
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
+on: [push, pull_request]
 
 jobs:
   ci:

.github/workflows/vs16-ci.yml

@@ -1,12 +1,6 @@
 name: VS16-CI
 
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
+on: [push, pull_request]
 
 jobs:
   ci:

.github/workflows/vs16-clang-ci.yml

@@ -1,12 +1,6 @@
 name: VS16-CLANG-CI
 
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
+on: [push, pull_request]
 
 jobs:
   ci:
@@ -31,7 +25,7 @@ jobs:
         buildWithCMake: true
         cmakeAppendedArgs:  -T ClangCL -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF
         buildWithCMakeArgs: --config Release
-    
+
     - name: 'Run CTest'
       run: ctest -C Release -LE explicitonly  --output-on-failure
       working-directory: "${{ github.workspace }}/../../_temp/windows"

.github/workflows/vs16-ninja-ci.yml

@@ -1,12 +1,6 @@
 name: VS16-Ninja-CI
 
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
+on: [push, pull_request]
 
 jobs:
   ci:

.github/workflows/vs16-noexcept-ci.yml

@@ -1,12 +1,6 @@
 name: VS16-NoExcept-CI
 
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
+on: [push, pull_request]
 
 jobs:
   ci:

cmake/developer-options.cmake

@@ -3,6 +3,12 @@
 #
 add_library(simdjson-internal-flags INTERFACE)
 
+
+option(SIMDJSON_CHECK_EOF "Check for the end of the input buffer. The setting is unnecessary since we require padding of the inputs. You should expect tests to fail with this option turned on." OFF)
+if(SIMDJSON_CHECK_EOF)
+  add_compile_definitions(SIMDJSON_CHECK_EOF=1)
+endif()
+
 option(SIMDJSON_SANITIZE_UNDEFINED "Sanitize undefined behavior" OFF)
 if(SIMDJSON_SANITIZE_UNDEFINED)
   add_compile_options(-fsanitize=undefined -fno-sanitize-recover=all)

doc/basics.md

@@ -1010,7 +1010,7 @@ Unlike `parser.iterate`, `parser.iterate_many` may parse "on demand" (lazily). T
 As with `parser.iterate`, when calling  `parser.iterate_many(string)`, no copy is made of the provided string input. The provided memory buffer may be accessed each time a JSON document is parsed.  Calling `parser.iterate_many(string)` on a  temporary string buffer (e.g., `docs = parser.parse_many("[1,2,3]"_padded)`) is unsafe (and will not compile) because the  `document_stream` instance needs access to the buffer to return the JSON documents.
 
 
-`iterate_many` can also take an optional parameter `size_t batch_size` which defines the window processing size. It is set by default to a large value (`1000000` corresponding to 1 MB). None of your JSON documents should exceed this window size, or else you will get  the error `simdjson::CAPACITY`. You cannot set this window size larger than 4 GB: you will get  the error `simdjson::CAPACITY`. The smaller the window size is, the less memory the function will use. Setting the window size too small (e.g., less than 100 kB) may also impact performance negatively. Leaving it to 1 MB is expected to be a good choice, unless you have some larger documents.
+The `iterate_many` function can also take an optional parameter `size_t batch_size` which defines the window processing size. It is set by default to a large value (`1000000` corresponding to 1 MB). None of your JSON documents should exceed this window size, or else you will get  the error `simdjson::CAPACITY`. You cannot set this window size larger than 4 GB: you will get  the error `simdjson::CAPACITY`. The smaller the window size is, the less memory the function will use. Setting the window size too small (e.g., less than 100 kB) may also impact performance negatively. Leaving it to 1 MB is expected to be a good choice, unless you have some larger documents.
 
 If your documents are large (e.g., larger than a megabyte), then the `iterate_many` function is maybe ill-suited. It is really meant to support reading efficiently streams of relatively small documents (e.g., a few kilobytes each). If you have larger documents, you should use other functions like `iterate`.
 

include/simdjson/arm64/numberparsing.h

@@ -19,7 +19,7 @@ static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t
 } // namespace SIMDJSON_IMPLEMENTATION
 } // namespace simdjson
 
-#define SWAR_NUMBER_PARSING
+#define SIMDJSON_SWAR_NUMBER_PARSING 1
 
 #include "simdjson/generic/numberparsing.h"
 

include/simdjson/common_defs.h

@@ -253,11 +253,8 @@ namespace std {
 #endif
 #endif
 
-// Feature flag for partially-implemented "don't require padding" feature
-// TODO remove once feature complete.
-#ifndef __SIMDJSON_CHECK_EOF
-# define __SIMDJSON_CHECK_EOF 1
-#endif
+// The SIMDJSON_CHECK_EOF macro is a feature flag for the "don't require padding"
+// feature.
 
 #if SIMDJSON_CPLUSPLUS17
 // if we have C++, then fallthrough is a default attribute
@@ -268,11 +265,11 @@ namespace std {
 #if __has_attribute(__fallthrough__)
 // we are good to go:
 # define simdjson_fallthrough                    __attribute__((__fallthrough__))
-#endif
-#endif
+#endif // __has_attribute(__fallthrough__)
+#endif // SIMDJSON_CPLUSPLUS17
 // on some systems, we simply do not have support for fallthrough, so use a default:
 #ifndef simdjson_fallthrough
 # define simdjson_fallthrough do {} while (0)  /* fallthrough */
-#endif
+#endif // simdjson_fallthrough
 
 #endif // SIMDJSON_COMMON_DEFS_H

include/simdjson/fallback/numberparsing.h

@@ -27,7 +27,8 @@ static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t
 } // namespace SIMDJSON_IMPLEMENTATION
 } // namespace simdjson
 
-#define SWAR_NUMBER_PARSING
+#define SIMDJSON_SWAR_NUMBER_PARSING 1
+
 #include "simdjson/generic/numberparsing.h"
 
 #endif // SIMDJSON_FALLBACK_NUMBERPARSING_H

include/simdjson/generic/numberparsing.h

@@ -367,14 +367,16 @@ simdjson_really_inline error_code parse_decimal(simdjson_unused const uint8_t *c
   // the integer into a float in a lossless manner.
   const uint8_t *const first_after_period = p;
 
-#ifdef SWAR_NUMBER_PARSING
+#ifdef SIMDJSON_SWAR_NUMBER_PARSING
+#if SIMDJSON_SWAR_NUMBER_PARSING
   // this helps if we have lots of decimals!
   // this turns out to be frequent enough.
   if (is_made_of_eight_digits_fast(p)) {
     i = i * 100000000 + parse_eight_digits_unrolled(p);
     p += 8;
   }
-#endif
+#endif // SIMDJSON_SWAR_NUMBER_PARSING
+#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING
   // Unrolling the first digit makes a small difference on some implementations (e.g. westmere)
   if (parse_digit(*p, i)) { ++p; }
   while (parse_digit(*p, i)) { p++; }
@@ -441,9 +443,7 @@ simdjson_really_inline size_t significant_digits(const uint8_t * start_digits, s
   // It is possible that the integer had an overflow.
   // We have to handle the case where we have 0.0000somenumber.
   const uint8_t *start = start_digits;
-  while ((*start == '0') || (*start == '.')) {
-    start++;
-  }
+  while ((*start == '0') || (*start == '.')) { ++start; }
   // we over-decrement by one when there is a '.'
   return digit_count - size_t(start - start_digits);
 }

include/simdjson/generic/ondemand/array-inl.h

@@ -82,7 +82,10 @@ simdjson_really_inline simdjson_result<std::string_view> array::raw_json() noexc
   const uint8_t * starting_point{iter.peek_start()};
   auto error = consume();
   if(error) { return error; }
-  const uint8_t * final_point{iter._json_iter->peek(0)};
+  // After 'consume()', we could be left pointing just beyond the document, but that
+  // is ok because we are not going to dereference the final pointer position, we just
+  // use it to compute the length in bytes.
+  const uint8_t * final_point{iter._json_iter->unsafe_pointer()};
   return std::string_view(reinterpret_cast<const char*>(starting_point), size_t(final_point - starting_point));
 }
 

include/simdjson/generic/ondemand/document-inl.h

@@ -156,12 +156,14 @@ simdjson_really_inline error_code document::consume() noexcept {
 }
 
 simdjson_really_inline simdjson_result<std::string_view> document::raw_json() noexcept {
-  printf("document::raw_json()\n");
   auto _iter = get_root_value_iterator();
   const uint8_t * starting_point{_iter.peek_start()};
   auto error = consume();
   if(error) { return error; }
-  const uint8_t * final_point{iter.peek(0)};
+  // After 'consume()', we could be left pointing just beyond the document, but that
+  // is ok because we are not going to dereference the final pointer position, we just
+  // use it to compute the length in bytes.
+  const uint8_t * final_point{iter.unsafe_pointer()};
   return std::string_view(reinterpret_cast<const char*>(starting_point), size_t(final_point - starting_point));
 }
 

include/simdjson/generic/ondemand/json_iterator-inl.h

@@ -35,7 +35,9 @@ simdjson_really_inline json_iterator::json_iterator(const uint8_t *buf, ondemand
 
 {
   logger::log_headers();
+#if SIMDJSON_CHECK_EOF
   assert_more_tokens();
+#endif
 }
 
 inline void json_iterator::rewind() noexcept {
@@ -84,10 +86,10 @@ simdjson_warn_unused simdjson_really_inline error_code json_iterator::skip_child
       logger::log_end_value(*this, "skip");
       _depth--;
       if (depth() <= parent_depth) { return SUCCESS; }
-#if __SIMDJSON_CHECK_EOF
+#if SIMDJSON_CHECK_EOF
       // If there are no more tokens, the parent is incomplete.
       if (at_end()) { return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "Missing [ or { at start"); }
-#endif // __SIMDJSON_CHECK_EOF
+#endif // SIMDJSON_CHECK_EOF
       break;
     /*case '"':
       if(*peek() == ':') {
@@ -199,23 +201,28 @@ simdjson_really_inline void json_iterator::abandon() noexcept {
 }
 
 simdjson_really_inline const uint8_t *json_iterator::return_current_and_advance() noexcept {
-  // The following assert_more_tokens is currently disabled because rely on end-of-file buffering.
-  // assert_more_tokens();
-  // This is almost surely related to __SIMDJSON_CHECK_EOF but given that __SIMDJSON_CHECK_EOF
-  // is ON by default, we have no choice but to disable it for real with a comment.
+#if SIMDJSON_CHECK_EOF
+  assert_more_tokens();
+#endif // SIMDJSON_CHECK_EOF
   return token.return_current_and_advance();
 }
 
+simdjson_really_inline const uint8_t *json_iterator::unsafe_pointer() const noexcept {
+  // deliberately done without safety guard:
+  return token.peek(0);
+}
+
 simdjson_really_inline const uint8_t *json_iterator::peek(int32_t delta) const noexcept {
-  // The following assert_more_tokens is currently disabled because rely on end-of-file buffering.
-  // assert_more_tokens(delta+1);
-  // This is almost surely related to __SIMDJSON_CHECK_EOF but given that __SIMDJSON_CHECK_EOF
-  // is ON by default, we have no choice but to disable it for real with a comment.
+#if SIMDJSON_CHECK_EOF
+  assert_more_tokens(delta+1);
+#endif // SIMDJSON_CHECK_EOF
   return token.peek(delta);
 }
 
 simdjson_really_inline uint32_t json_iterator::peek_length(int32_t delta) const noexcept {
+#if SIMDJSON_CHECK_EOF
   assert_more_tokens(delta+1);
+#endif // #if SIMDJSON_CHECK_EOF
   return token.peek_length(delta);
 }
 
@@ -223,17 +230,15 @@ simdjson_really_inline const uint8_t *json_iterator::peek(token_position positio
   // todo: currently we require end-of-string buffering, but the following
   // assert_valid_position should be turned on if/when we lift that condition.
   // assert_valid_position(position);
-  // This is almost surely related to __SIMDJSON_CHECK_EOF but given that __SIMDJSON_CHECK_EOF
+  // This is almost surely related to SIMDJSON_CHECK_EOF but given that SIMDJSON_CHECK_EOF
   // is ON by default, we have no choice but to disable it for real with a comment.
   return token.peek(position);
 }
 
 simdjson_really_inline uint32_t json_iterator::peek_length(token_position position) const noexcept {
-  // todo: currently we require end-of-string buffering, but the following
-  // assert_valid_position should be turned on if/when we lift that condition.
-  // assert_valid_position(position);
-  // This is almost surely related to __SIMDJSON_CHECK_EOF but given that __SIMDJSON_CHECK_EOF
-  // is ON by default, we have no choice but to disable it for real with a comment.
+#if SIMDJSON_CHECK_EOF
+  assert_valid_position(position);
+#endif // SIMDJSON_CHECK_EOF
   return token.peek_length(position);
 }
 

include/simdjson/generic/ondemand/json_iterator.h

@@ -145,12 +145,21 @@ public:
    * @param delta The relative position of the token to retrieve. e.g. 0 = next token, -1 = prev token.
    */
   simdjson_really_inline uint32_t peek_length(int32_t delta=0) const noexcept;
+  /**
+   * Get a pointer to the current location in the input buffer.
+   *
+   * This is not null-terminated; it is a view into the JSON.
+   *
+   * You may be pointing outside of the input buffer: it is not generally
+   * safe to derefence this pointer.
+   */
+  simdjson_really_inline const uint8_t *unsafe_pointer() const noexcept;
   /**
    * Get the JSON text for a given token.
    *
    * This is not null-terminated; it is a view into the JSON.
    *
-   * @param index The position of the token to retrieve.
+   * @param position The position of the token to retrieve.
    *
    * TODO consider a string_view, assuming the length will get stripped out by the optimizer when
    * it isn't used ...
@@ -161,7 +170,7 @@ public:
    *
    * The length will include any whitespace at the end of the token.
    *
-   * @param index The position of the token to retrieve.
+   * @param position The position of the token to retrieve.
    */
   simdjson_really_inline uint32_t peek_length(token_position position) const noexcept;
   /**

include/simdjson/generic/ondemand/parser.h

@@ -129,8 +129,6 @@ public:
    * those bytes are initialized to, as long as they are allocated.
    *
    * @param json The JSON to parse.
-   * @param len The length of the JSON.
-   * @param allocated The number of bytes allocated in the JSON (must be at least len+SIMDJSON_PADDING).
    *
    * @return The iterator, or an error:
    *         - INSUFFICIENT_PADDING if the input has less than SIMDJSON_PADDING extra bytes.
@@ -143,6 +141,67 @@ public:
    */
   simdjson_warn_unused simdjson_result<json_iterator> iterate_raw(padded_string_view json) & noexcept;
 
+
+  /**
+   * Parse a buffer containing many JSON documents.
+   *
+   *   auto json = R"({ "foo": 1 } { "foo": 2 } { "foo": 3 } )"_padded;
+   *   ondemand::parser parser;
+   *   ondemand::document_stream docs = parser.iterate_many(json);
+   *   for (auto & doc : docs) {
+   *     std::cout << doc["foo"] << std::endl;
+   *   }
+   *   // Prints 1 2 3
+   *
+   * No copy of the input buffer is made.
+   *
+   * The function is lazy: it may be that no more than one JSON document at a time is parsed.
+   *
+   * The caller is responsabile to ensure that the input string data remains unchanged and is
+   * not deleted during the loop.
+   *
+   * ### Format
+   *
+   * The buffer must contain a series of one or more JSON documents, concatenated into a single
+   * buffer, separated by whitespace. It effectively parses until it has a fully valid document,
+   * then starts parsing the next document at that point. (It does this with more parallelism and
+   * lookahead than you might think, though.)
+   *
+   * documents that consist of an object or array may omit the whitespace between them, concatenating
+   * with no separator. documents that consist of a single primitive (i.e. documents that are not
+   * arrays or objects) MUST be separated with whitespace.
+   *
+   * The documents must not exceed batch_size bytes (by default 1MB) or they will fail to parse.
+   * Setting batch_size to excessively large or excesively small values may impact negatively the
+   * performance.
+   *
+   * ### REQUIRED: Buffer Padding
+   *
+   * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what
+   * those bytes are initialized to, as long as they are allocated.
+   *
+   * ### Threads
+   *
+   * When compiled with SIMDJSON_THREADS_ENABLED, this method will use a single thread under the
+   * hood to do some lookahead.
+   *
+   * ### Parser Capacity
+   *
+   * If the parser's current capacity is less than batch_size, it will allocate enough capacity
+   * to handle it (up to max_capacity).
+   *
+   * @param buf The concatenated JSON to parse.
+   * @param len The length of the concatenated JSON.
+   * @param batch_size The batch size to use. MUST be larger than the largest document. The sweet
+   *                   spot is cache-related: small enough to fit in cache, yet big enough to
+   *                   parse as many documents as possible in one tight loop.
+   *                   Defaults to 10MB, which has been a reasonable sweet spot in our tests.
+   * @return The stream, or an error. An empty input will yield 0 documents rather than an EMPTY error. Errors:
+   *         - MEMALLOC if the parser does not have enough capacity and memory allocation fails
+   *         - CAPACITY if the parser does not have enough capacity and batch_size > max_capacity.
+   *         - other json errors if parsing fails. You should not rely on these errors to always the same for the
+   *           same document: they may vary under runtime dispatch (so they may vary depending on your system and hardware).
+   */
   inline simdjson_result<document_stream> iterate_many(const uint8_t *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept;
   /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */
   inline simdjson_result<document_stream> iterate_many(const char *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept;

include/simdjson/generic/ondemand/value_iterator-inl.h

@@ -46,10 +46,10 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator
 }
 
 simdjson_warn_unused simdjson_really_inline error_code value_iterator::end_container() noexcept {
-#if __SIMDJSON_CHECK_EOF
+#if SIMDJSON_CHECK_EOF
     if (depth() > 1 && at_end()) { return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "missing parent ] or }"); }
     // if (depth() <= 1 && !at_end()) { return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "missing [ or { at start"); }
-#endif // __SIMDJSON_CHECK_EOF
+#endif // SIMDJSON_CHECK_EOF
     _json_iter->ascend_to(depth()-1);
     return SUCCESS;
 }
@@ -130,6 +130,7 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator
     // Get the key and colon, stopping at the value.
     raw_json_string actual_key;
     // size_t max_key_length = _json_iter->peek_length() - 2; // -2 for the two quotes
+    // Note: _json_iter->peek_length() - 2 might overflow if _json_iter->peek_length() < 2.
     // field_key() advances the pointer and checks that '"' is found (corresponding to a key).
     // The depth is left unchanged by field_key().
     if ((error = field_key().get(actual_key) )) { abandon(); return error; };
@@ -260,6 +261,7 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator
     // Get the key and colon, stopping at the value.
     raw_json_string actual_key;
     // size_t max_key_length = _json_iter->peek_length() - 2; // -2 for the two quotes
+    // Note: _json_iter->peek_length() - 2 might overflow if _json_iter->peek_length() < 2.
     // field_key() advances the pointer and checks that '"' is found (corresponding to a key).
     // The depth is left unchanged by field_key().
     if ((error = field_key().get(actual_key) )) { abandon(); return error; };
@@ -309,6 +311,7 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator
     // Get the key and colon, stopping at the value.
     raw_json_string actual_key;
     // size_t max_key_length = _json_iter->peek_length() - 2; // -2 for the two quotes
+    // Note: _json_iter->peek_length() - 2 might overflow if _json_iter->peek_length() < 2.
     // field_key() advances the pointer and checks that '"' is found (corresponding to a key).
     // The depth is left unchanged by field_key().
     error = field_key().get(actual_key); SIMDJSON_ASSUME(!error);

include/simdjson/haswell/numberparsing.h

@@ -27,7 +27,7 @@ static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t
 } // namespace SIMDJSON_IMPLEMENTATION
 } // namespace simdjson
 
-#define SWAR_NUMBER_PARSING
+#define SIMDJSON_SWAR_NUMBER_PARSING 1
 
 #include "simdjson/generic/numberparsing.h"
 

include/simdjson/ppc64/numberparsing.h

@@ -25,7 +25,7 @@ parse_eight_digits_unrolled(const uint8_t *chars) {
 } // namespace SIMDJSON_IMPLEMENTATION
 } // namespace simdjson
 
-#define SWAR_NUMBER_PARSING
+#define SIMDJSON_SWAR_NUMBER_PARSING 1
 
 #include "simdjson/generic/numberparsing.h"
 

include/simdjson/westmere/numberparsing.h

@@ -27,7 +27,7 @@ static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t
 } // namespace SIMDJSON_IMPLEMENTATION
 } // namespace simdjson
 
-#define SWAR_NUMBER_PARSING
+#define SIMDJSON_SWAR_NUMBER_PARSING 1
 
 #include "simdjson/generic/numberparsing.h"
 

singleheader/simdjson.h

@@ -10386,7 +10386,7 @@ static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t
 } // namespace arm64
 } // namespace simdjson
 
-#define SWAR_NUMBER_PARSING
+#define SIMDJSON_SWAR_NUMBER_PARSING
 
 /* begin file include/simdjson/generic/numberparsing.h */
 #include <limits>
@@ -10743,7 +10743,7 @@ simdjson_really_inline error_code parse_decimal(simdjson_unused const uint8_t *c
   // the integer into a float in a lossless manner.
   const uint8_t *const first_after_period = p;
 
-#ifdef SWAR_NUMBER_PARSING
+#ifdef SIMDJSON_SWAR_NUMBER_PARSING
   // this helps if we have lots of decimals!
   // this turns out to be frequent enough.
   if (is_made_of_eight_digits_fast(p)) {
@@ -11826,7 +11826,7 @@ static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t
 } // namespace fallback
 } // namespace simdjson
 
-#define SWAR_NUMBER_PARSING
+#define SIMDJSON_SWAR_NUMBER_PARSING
 /* begin file include/simdjson/generic/numberparsing.h */
 #include <limits>
 
@@ -12182,7 +12182,7 @@ simdjson_really_inline error_code parse_decimal(simdjson_unused const uint8_t *c
   // the integer into a float in a lossless manner.
   const uint8_t *const first_after_period = p;
 
-#ifdef SWAR_NUMBER_PARSING
+#ifdef SIMDJSON_SWAR_NUMBER_PARSING
   // this helps if we have lots of decimals!
   // this turns out to be frequent enough.
   if (is_made_of_eight_digits_fast(p)) {
@@ -13749,7 +13749,7 @@ static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t
 } // namespace haswell
 } // namespace simdjson
 
-#define SWAR_NUMBER_PARSING
+#define SIMDJSON_SWAR_NUMBER_PARSING
 
 /* begin file include/simdjson/generic/numberparsing.h */
 #include <limits>
@@ -14106,7 +14106,7 @@ simdjson_really_inline error_code parse_decimal(simdjson_unused const uint8_t *c
   // the integer into a float in a lossless manner.
   const uint8_t *const first_after_period = p;
 
-#ifdef SWAR_NUMBER_PARSING
+#ifdef SIMDJSON_SWAR_NUMBER_PARSING
   // this helps if we have lots of decimals!
   // this turns out to be frequent enough.
   if (is_made_of_eight_digits_fast(p)) {
@@ -15772,7 +15772,7 @@ parse_eight_digits_unrolled(const uint8_t *chars) {
 } // namespace ppc64
 } // namespace simdjson
 
-#define SWAR_NUMBER_PARSING
+#define SIMDJSON_SWAR_NUMBER_PARSING
 
 /* begin file include/simdjson/generic/numberparsing.h */
 #include <limits>
@@ -16129,7 +16129,7 @@ simdjson_really_inline error_code parse_decimal(simdjson_unused const uint8_t *c
   // the integer into a float in a lossless manner.
   const uint8_t *const first_after_period = p;
 
-#ifdef SWAR_NUMBER_PARSING
+#ifdef SIMDJSON_SWAR_NUMBER_PARSING
   // this helps if we have lots of decimals!
   // this turns out to be frequent enough.
   if (is_made_of_eight_digits_fast(p)) {
@@ -17653,7 +17653,7 @@ static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t
 } // namespace westmere
 } // namespace simdjson
 
-#define SWAR_NUMBER_PARSING
+#define SIMDJSON_SWAR_NUMBER_PARSING
 
 /* begin file include/simdjson/generic/numberparsing.h */
 #include <limits>
@@ -18010,7 +18010,7 @@ simdjson_really_inline error_code parse_decimal(simdjson_unused const uint8_t *c
   // the integer into a float in a lossless manner.
   const uint8_t *const first_after_period = p;
 
-#ifdef SWAR_NUMBER_PARSING
+#ifdef SIMDJSON_SWAR_NUMBER_PARSING
   // this helps if we have lots of decimals!
   // this turns out to be frequent enough.
   if (is_made_of_eight_digits_fast(p)) {

tests/dom/basictests.cpp

@@ -2116,7 +2116,20 @@ namespace to_string_tests {
   }
 }
 
-
+bool simple_overflows() {
+  std::cout << "Running " << __func__ << std::endl;
+  simdjson::dom::parser parser;
+  simdjson::dom::element doc;
+  ASSERT_ERROR( parser.parse(std::string("[f]")).get(doc), simdjson::F_ATOM_ERROR);
+  ASSERT_ERROR( parser.parse(std::string("[t]")).get(doc), simdjson::T_ATOM_ERROR);
+  ASSERT_ERROR( parser.parse(std::string("[n]")).get(doc), simdjson::N_ATOM_ERROR);
+  ASSERT_ERROR( parser.parse(std::string("[-]")).get(doc), simdjson::NUMBER_ERROR);
+  ASSERT_ERROR( parser.parse(std::string("{\"a\":f}")).get(doc), simdjson::F_ATOM_ERROR);
+  ASSERT_ERROR( parser.parse(std::string("{\"a\":t}")).get(doc), simdjson::T_ATOM_ERROR);
+  ASSERT_ERROR( parser.parse(std::string("{\"a\":n}")).get(doc), simdjson::N_ATOM_ERROR);
+  ASSERT_ERROR( parser.parse(std::string("{\"a\":-}")).get(doc), simdjson::NUMBER_ERROR);
+  return true;
+}
 
 int main(int argc, char *argv[]) {
   std::cout << std::unitbuf;
@@ -2141,7 +2154,6 @@ int main(int argc, char *argv[]) {
       return EXIT_FAILURE;
     }
   }
-
   // this is put here deliberately to check that the documentation is correct (README),
   // should this fail to compile, you should update the documentation:
   if (simdjson::active_implementation->name() == "unsupported") {
@@ -2153,7 +2165,8 @@ int main(int argc, char *argv[]) {
   std::cout << "------------------------------------------------------------" << std::endl;
 
   std::cout << "Running basic tests." << std::endl;
-  if (to_string_tests::run() &&
+  if (simple_overflows() &&
+      to_string_tests::run() &&
       validate_tests::run() &&
       minify_tests::run() &&
       parse_api_tests::run() &&

tests/dom/document_stream_tests.cpp

@@ -735,7 +735,7 @@ namespace document_stream_tests {
   bool document_stream_test() {
     std::cout << "Running " << __func__ << std::endl;
     fflush(NULL);
-    const size_t n_records = 10000;
+    const size_t n_records = 100;
     std::string data;
     std::vector<char> buf(1024);
     for (size_t i = 0; i < n_records; ++i) {
@@ -775,7 +775,7 @@ namespace document_stream_tests {
   bool document_stream_utf8_test() {
     std::cout << "Running " << __func__ << std::endl;
     fflush(NULL);
-    const size_t n_records = 10000;
+    const size_t n_records = 100;
     std::string data;
     std::vector<char> buf(1024);
     for (size_t i = 0; i < n_records; ++i) {

tests/dom/document_tests.cpp

@@ -108,7 +108,7 @@ namespace document_tests {
   // returns true if successful
   bool skyprophet_test() {
     std::cout << "Running " << __func__ << std::endl;
-    const size_t n_records = 100000;
+    const size_t n_records = 100;
     std::vector<std::string> data;
     std::vector<char> buf(1024);
     for (size_t i = 0; i < n_records; ++i) {

tests/ondemand/ondemand_array_error_tests.cpp

@@ -100,10 +100,16 @@ namespace array_error_tests {
   }
   bool array_iterate_incomplete_error() {
     TEST_START();
+#if SIMDJSON_CHECK_EOF
     ONDEMAND_SUBTEST("unclosed after array", R"([ [1] )", assert_iterate(doc.get_array().at(0), { int64_t(1) }, { INCOMPLETE_ARRAY_OR_OBJECT }));
     ONDEMAND_SUBTEST("unclosed after array", R"([ [1,])", assert_iterate(doc.get_array().at(0), { int64_t(1) }, { INCORRECT_TYPE, TAPE_ERROR }));
     ONDEMAND_SUBTEST("unclosed after array", R"([ [1])",  assert_iterate(doc.get_array().at(0), { int64_t(1) }, { INCOMPLETE_ARRAY_OR_OBJECT }));
     ONDEMAND_SUBTEST("unclosed after array", R"([ [])",   assert_iterate(doc.get_array().at(0),                 { INCOMPLETE_ARRAY_OR_OBJECT }));
+#else
+    ONDEMAND_SUBTEST("unclosed after array", R"([ [1] )", assert_iterate(doc.get_array().at(0), { int64_t(1) }));
+    ONDEMAND_SUBTEST("unclosed after array", R"([ [1,])", assert_iterate(doc.get_array().at(0), { int64_t(1) }, { INCORRECT_TYPE, TAPE_ERROR }));
+    ONDEMAND_SUBTEST("unclosed after array", R"([ [1])",  assert_iterate(doc.get_array().at(0), { int64_t(1) }));
+#endif
     TEST_SUCCEED();
   }
 

tests/ondemand/ondemand_document_stream_tests.cpp

@@ -315,7 +315,7 @@ namespace document_stream_tests {
     bool document_stream_test() {
         TEST_START();
         fflush(NULL);
-        const size_t n_records = 10000;
+        const size_t n_records = 100;
         std::string data;
         std::vector<char> buf(1024);
         // Generating data
@@ -352,7 +352,7 @@ namespace document_stream_tests {
     bool document_stream_utf8_test() {
         TEST_START();
         fflush(NULL);
-        const size_t n_records = 10000;
+        const size_t n_records = 100;
         std::string data;
         std::vector<char> buf(1024);
         // Generating data

tests/ondemand/ondemand_object_error_tests.cpp

@@ -88,10 +88,10 @@ namespace object_error_tests {
   }
   bool object_lookup_unclosed_error() {
     TEST_START();
-  #if __SIMDJSON_CHECK_EOF
+  #if SIMDJSON_CHECK_EOF
     ONDEMAND_SUBTEST("unclosed", R"({ "a":            )",    assert_error(doc["a"], INCOMPLETE_ARRAY_OR_OBJECT));
   #else
-    ONDEMAND_SUBTEST("unclosed", R"({ "a":            )",    assert_success(doc["a"]));
+    ONDEMAND_SUBTEST("unclosed", R"({ "a":            )",    assert_error(doc["a"], INCOMPLETE_ARRAY_OR_OBJECT));
   #endif
     ONDEMAND_SUBTEST("unclosed", R"({ "a"             )",    assert_error(doc["a"], INCOMPLETE_ARRAY_OR_OBJECT));
     ONDEMAND_SUBTEST("unclosed", R"({                 )",    assert_error(doc["a"], INCOMPLETE_ARRAY_OR_OBJECT));