Release

* Undoing PR 2004 due to performance concerns.

Fixes https://github.com/simdjson/simdjson/issues/2028
Fixes https://github.com/simdjson/simdjson/issues/2021

* Adding final lines.

---------

Co-authored-by: Daniel Lemire <dlemire@lemire.me>

.clangd

@@ -0,0 +1,2 @@
+CompileFlags:
+    CompilationDatabase: build

.github/workflows/fuzzers.yml

@@ -124,7 +124,7 @@ jobs:
         done
 
     - name: Save the corpus as a github artifact
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v3
       with:
         name: corpus
         path: corpus.tar
@@ -147,7 +147,7 @@ jobs:
       run: tar cf valgrind.tar valgrind-*.txt
 
     - name: Save valgrind output as a github artifact
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v3
       if: always()
       with:
         name: valgrindresults
@@ -155,7 +155,7 @@ jobs:
         if-no-files-found: ignore
 
     - name: Archive any crashes as an artifact
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v3
       if: always()
       with:
         name: crashes

.github/workflows/power-fuzz.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-20.04
     name: Build on ubuntu-20.04 ppc64le
     steps:
-      - uses: actions/checkout@v2.1.0
+      - uses: actions/checkout@v3
       - uses: uraimo/run-on-arch-action@v2.0.5
         name: Run commands
         id: runcmd

.github/workflows/ppc64.yml

@@ -0,0 +1,29 @@
+name: Ubuntu ppc64le (GCC 11)
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: uraimo/run-on-arch-action@v2
+        name: Test
+        id: runcmd
+        with:
+          arch: ppc64le
+          distro: ubuntu_latest
+          githubToken: ${{ github.token }}
+          install: |
+            apt-get update -q -y
+            apt-get install -y cmake make g++
+          run: |
+            cmake -DCMAKE_BUILD_TYPE=Release -B build
+            cmake --build build -j=2
+            ctest --output-on-failure --test-dir build

.github/workflows/ubuntu22-cxx20.yml

@@ -0,0 +1,33 @@
+name: Ubuntu 22.04 CI (GCC 12, CXX 20)
+
+on: [push, pull_request]
+
+jobs:
+  ubuntu-build:
+    if: >-
+      ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
+      ! contains(toJSON(github.event.commits.*.message), '[skip github]')
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
+        with:
+          path: dependencies/.cache
+          key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
+      - name: Use cmake
+        run: |
+          mkdir builddebug &&
+          cd builddebug &&
+          CXX=g++-12 cmake -DSIMDJSON_CXX_STANDARD=20 -DCMAKE_BUILD_TYPE=Debug -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF ..  &&
+          cmake --build .   &&
+          ctest -j --output-on-failure -LE explicitonly   &&
+          cd .. &&
+          mkdir build &&
+          cd build &&
+          CXX=g++-12 cmake -DSIMDJSON_CXX_STANDARD=20  -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination ..  &&
+          cmake --build .   &&
+          ctest -j --output-on-failure -LE explicitonly   &&
+          cmake --install . &&
+          echo -e '#include <simdjson.h>\nint main(int argc,char**argv) {simdjson::dom::parser parser;simdjson::dom::element tweets = parser.load(argv[1]); }' > tmp.cpp && c++ -Idestination/include -Ldestination/lib -std=c++17 -Wl,-rpath,destination/lib -o linkandrun tmp.cpp -lsimdjson && ./linkandrun jsonexamples/twitter.json &&
+          cd ../tests/installation_tests/find &&
+          mkdir build && cd build && cmake -DCMAKE_INSTALL_PREFIX:PATH=../../../build/destination .. &&  cmake --build .

.github/workflows/vs17-ci-cxx20.yml

@@ -0,0 +1,44 @@
+name: VS17-CI CXX20
+
+on: [push, pull_request]
+
+jobs:
+  ci:
+    if: >-
+      ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
+      ! contains(toJSON(github.event.commits.*.message), '[skip github]')
+    name: windows-vs17
+    runs-on: windows-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - {gen: Visual Studio 17 2022, arch: Win32, shared: ON}
+          - {gen: Visual Studio 17 2022, arch: Win32, shared: OFF}
+          - {gen: Visual Studio 17 2022, arch: x64, shared: ON}
+          - {gen: Visual Studio 17 2022, arch: x64, shared: OFF}
+    steps:
+    - name: checkout
+      uses: actions/checkout@v3
+    - name: Configure
+      run: |
+        cmake -DSIMDJSON_CXX_STANDARD=20 -G "${{matrix.gen}}" -A ${{matrix.arch}} -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -DBUILD_SHARED_LIBS=${{matrix.shared}} -B build
+    - name: Build Debug
+      run: cmake --build build --config Debug --verbose
+    - name: Build Release
+      run: cmake --build build --config Release --verbose
+    - name: Run Release tests
+      run: |
+        cd build
+        ctest -C Release  -LE explicitonly  --output-on-failure
+    - name: Run Debug tests
+      run: |
+        cd build
+        ctest -C Debug  -LE explicitonly  --output-on-failure
+    - name: Install
+      run: |
+        cmake --install build --config Release
+    - name: Test Installation
+      run: |
+        cmake -G "${{matrix.gen}}" -A ${{matrix.arch}}  -B build_install_test tests/installation_tests/find
+        cmake --build build_install_test --config Release

CMakeLists.txt

@@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.14)
 project(
     simdjson
     # The version number is modified by tools/release.py
-    VERSION 3.1.7
+    VERSION 3.2.1
     DESCRIPTION "Parsing gigabytes of JSON per second"
     HOMEPAGE_URL "https://simdjson.org/"
     LANGUAGES CXX C
@@ -20,8 +20,8 @@ string(
 # ---- Options, variables ----
 
 # These version numbers are modified by tools/release.py
-set(SIMDJSON_LIB_VERSION "15.0.0" CACHE STRING "simdjson library version")
+set(SIMDJSON_LIB_VERSION "16.0.0" CACHE STRING "simdjson library version")
-set(SIMDJSON_LIB_SOVERSION "15" CACHE STRING "simdjson library soversion")
+set(SIMDJSON_LIB_SOVERSION "16" CACHE STRING "simdjson library soversion")
 
 option(SIMDJSON_ENABLE_THREADS "Link with thread support" ON)
 

Doxyfile

@@ -38,7 +38,7 @@ PROJECT_NAME           = simdjson
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = "3.1.7"
+PROJECT_NUMBER         = "3.2.1"
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a

HACKING.md

@@ -6,16 +6,26 @@ Here is wisdom about how to build, test and run simdjson from within the reposit
 
 If you plan to contribute to simdjson, please read our [CONTRIBUTING](https://github.com/simdjson/simdjson/blob/master/CONTRIBUTING.md) guide.
 
-
+- [Hacking simdjson](#hacking-simdjson)
+  - [Build Quickstart](#build-quickstart)
   - [Design notes](#design-notes)
   - [Developer mode](#developer-mode)
   - [Directory Structure and Source](#directory-structure-and-source)
   - [Runtime Dispatching](#runtime-dispatching)
   - [Regenerating Single-Header Files](#regenerating-single-header-files)
   - [Usage (CMake on 64-bit platforms like Linux, FreeBSD or macOS)](#usage-cmake-on-64-bit-platforms-like-linux-freebsd-or-macos)
-  - [Usage (CMake on 64-bit Windows using Visual Studio 2019)](#usage-cmake-on-64-bit-windows-using-visual-studio-2019)
+  - [Usage (CMake on 64-bit Windows using Visual Studio 2019 or better)](#usage-cmake-on-64-bit-windows-using-visual-studio-2019-or-better)
   - [Various References](#various-references)
 
+Build Quickstart
+------------------------------
+
+```bash
+mkdir build
+cd build
+cmake -D SIMDJSON_DEVELOPER_MODE=ON ..
+cmake --build .
+```
 
 Design notes
 ------------------------------

README.md

@@ -28,16 +28,39 @@ This library is part of the [Awesome Modern C++](https://awesomecpp.com) list.
 Table of Contents
 -----------------
 
+* [Real-world usage](#real-world-usage)
 * [Quick Start](#quick-start)
 * [Documentation](#documentation)
 * [Performance results](#performance-results)
-* [Real-world usage](#real-world-usage)
 * [Bindings and Ports of simdjson](#bindings-and-ports-of-simdjson)
 * [About simdjson](#about-simdjson)
 * [Funding](#funding)
 * [Contributing to simdjson](#contributing-to-simdjson)
 * [License](#license)
 
+
+Real-world usage
+----------------
+
+- [ClickHouse](https://github.com/ClickHouse/ClickHouse)
+- [Facebook/Meta Velox](https://velox-lib.io)
+- [milvus](https://github.com/milvus-io/milvus)
+- [Clang Build Analyzer](https://github.com/aras-p/ClangBuildAnalyzer)
+- [Shopify HeapProfiler](https://github.com/Shopify/heap-profiler)
+- [StarRocks](https://github.com/StarRocks/starrocks)
+- [Microsoft FishStore](https://github.com/microsoft/FishStore)
+- [Intel PCM](https://github.com/intel/pcm)
+- [WatermelonDB](https://github.com/Nozbe/WatermelonDB)
+- [Apache Doris](https://github.com/apache/doris)
+- [Dgraph](https://github.com/dgraph-io/dgraph)
+- [UJRPC](https://github.com/unum-cloud/ujrpc)
+- [fastgltf](https://github.com/spnda/fastgltf)
+- [vast](https://github.com/tenzir/vast)
+- [ada-url](https://github.com/ada-url/ada)
+- [fastgron](https://github.com/adamritter/fastgron)
+
+If you are planning to use simdjson in a product, please work from one of our releases.
+
 Quick Start
 -----------
 
@@ -64,13 +87,13 @@ int main(void) {
     ondemand::document tweets = parser.iterate(json);
     std::cout << uint64_t(tweets["search_metadata"]["count"]) << " results." << std::endl;
 }
-
+```
-   ```
 3. `c++ -o quickstart quickstart.cpp simdjson.cpp`
 4. `./quickstart`
-   ```
+
+  ```
    100 results.
-   ```
+  ```
 
 Documentation
 -------------
@@ -81,7 +104,13 @@ Usage documentation is available:
 * [Performance](doc/performance.md) shows some more advanced scenarios and how to tune for them.
 * [Implementation Selection](doc/implementation-selection.md) describes runtime CPU detection and
   how you can work with it.
-* [API](https://simdjson.org/api/1.0.0/annotated.html) contains the automatically generated API documentation.
+* [API](https://simdjson.github.io/simdjson/) contains the automatically generated API documentation.
+
+
+Some users may want to browse code along with the compiled assembly. You want to check out the following lists of examples:
+
+* [simdjson examples with errors handled through exceptions](https://godbolt.org/z/98Kx9Kqjn)
+* [simdjson examples with errors without exceptions](https://godbolt.org/z/PKG7GdbPo)
 
 Performance results
 -------------------
@@ -110,24 +139,6 @@ For NDJSON files, we can exceed 3 GB/s with [our  multithreaded parsing function
 
 
 
-Real-world usage
-----------------
-
-- [ClickHouse](https://github.com/ClickHouse/ClickHouse)
-- [Clang Build Analyzer](https://github.com/aras-p/ClangBuildAnalyzer)
-- [Shopify HeapProfiler](https://github.com/Shopify/heap-profiler)
-- [StarRocks](https://github.com/StarRocks/starrocks)
-- [Microsoft FishStore](https://github.com/microsoft/FishStore)
-- [Intel PCM](https://github.com/intel/pcm)
-- [WatermelonDB](https://github.com/Nozbe/WatermelonDB)
-- [Apache Doris](https://github.com/apache/doris)
-- [Dgraph](https://github.com/dgraph-io/dgraph)
-- [UJRPC](https://github.com/unum-cloud/ujrpc)
-- [fastgltf](https://github.com/spnda/fastgltf)
-- [Clang Build Analyzer](https://github.com/aras-p/ClangBuildAnalyzer)
-- [vast](https://github.com/tenzir/vast)
-
-If you are planning to use simdjson in a product, please work from one of our releases.
 
 Bindings and Ports of simdjson
 ------------------------------

benchmark/bench_dom_api.cpp

@@ -521,7 +521,7 @@ static void twitter_image_sizes(State& state) {
     set<tuple<uint64_t, uint64_t>> image_sizes;
     for (dom::object tweet : doc["statuses"]) {
       dom::array media;
-      if (not (error = tweet["entities"]["media"].get(media))) {
+      if (! (error = tweet["entities"]["media"].get(media))) {
         for (dom::object image : media) {
           for (auto size : image["sizes"].get_object()) {
              image_sizes.emplace(size.value["w"], size.value["h"]);
@@ -638,7 +638,7 @@ static void error_code_twitter_image_sizes(State& state) noexcept {
     if ((error = doc["statuses"].get(statuses))) { return; }
     for (dom::element tweet : statuses) {
       dom::array images;
-      if (not (error = tweet["entities"]["media"].get(images))) {
+      if (! (error = tweet["entities"]["media"].get(images))) {
         for (dom::element image : images) {
           dom::object sizes;
           if ((error = image["sizes"].get(sizes))) { return; }

benchmark/benchmarker.h

@@ -520,7 +520,7 @@ struct benchmarker {
         double freqall = (all_stages_without_allocation.best.cycles() / all_stages_without_allocation.best.elapsed_sec()) / 1000000000.0;
         double freqmin = min(freq1, freq2);
         double freqmax = max(freq1, freq2);
-        if((freqall < 0.95 * freqmin) or (freqall > 1.05 * freqmax)) {
+        if((freqall < 0.95 * freqmin) || (freqall > 1.05 * freqmax)) {
           printf("\nWarning: The processor frequency fluctuates in an expected way!!!\n"
           "Range for stage 1 and stage 2 : [%.3f GHz, %.3f GHz], overall: %.3f GHz.\n",
           freqmin, freqmax, freqall);

benchmark/json2msgpack/simdjson_dom.h

@@ -15,7 +15,7 @@ struct simdjsondom2msgpack {
    * @param json JSON input
    * @param buf temporary buffer (must be large enough, with simdjson::SIMDJSON_PADDING bytes
    * of padding)
-   * @return std::string_view msgpack output, writting to the temporary buffer
+   * @return std::string_view msgpack output, writing to the temporary buffer
    */
   inline std::string_view to_msgpack(const simdjson::padded_string &json,
                                      uint8_t *buf);

benchmark/json2msgpack/simdjson_ondemand.h

@@ -36,7 +36,7 @@ struct simdjson2msgpack {
    * @param json JSON input
    * @param buf temporary buffer (must be large enough, with simdjson::SIMDJSON_PADDING bytes
    * of padding)
-   * @return std::string_view msgpack output, writting to the temporary buffer
+   * @return std::string_view msgpack output, writing to the temporary buffer
    */
   inline std::string_view to_msgpack(const simdjson::padded_string &json,
                                      uint8_t *buf);

cmake/developer-options.cmake

@@ -110,7 +110,8 @@ endif()
 
 # We compile tools, tests, etc. with C++ 17. Override yourself if you need on a
 # target.
-set(CMAKE_CXX_STANDARD 17)
+set(SIMDJSON_CXX_STANDARD 17 CACHE STRING "the C++ standard to use for simdjson")
+set(CMAKE_CXX_STANDARD ${SIMDJSON_CXX_STANDARD})
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)
 set(CMAKE_MACOSX_RPATH OFF)

doc/basics.md

@@ -25,11 +25,11 @@ An overview of what you need to know to use simdjson, with examples.
     - [Current location in document](#current-location-in-document)
     - [Checking for trailing content](#checking-for-trailing-content)
   - [Rewinding](#rewinding)
-  - [Direct Access to the Raw String](#direct-access-to-the-raw-string)
   - [Newline-Delimited JSON (ndjson) and JSON lines](#newline-delimited-json-ndjson-and-json-lines)
   - [Parsing Numbers Inside Strings](#parsing-numbers-inside-strings)
   - [Dynamic Number Types](#dynamic-number-types)
   - [Raw Strings](#raw-strings)
+  - [General Direct Access to the Raw JSON String](#general-direct-access-to-the-raw-json-string)
   - [Thread Safety](#thread-safety)
   - [Standard Compliance](#standard-compliance)
   - [Backwards Compatibility](#backwards-compatibility)
@@ -202,6 +202,13 @@ Consider reusing the same buffers and limiting memory allocations.
 
 By default, the simdjson library throws exceptions (`simdjson_error`) on errors. We omit `try`-`catch` clauses from our illustrating examples: if you omit `try`-`catch` in your code, an uncaught exception will halt your program. It is also possible to use simdjson without generating exceptions, and you may even build the library without exception support at all. See [Error Handling](#error-handling) for details.
 
+
+Some users may want to browse code along with the compiled assembly. You want to check out the following lists of examples:
+
+* [simdjson examples with errors handled through exceptions](https://godbolt.org/z/98Kx9Kqjn)
+* [simdjson examples with errors without exceptions](https://godbolt.org/z/PKG7GdbPo)
+
+
 Documents are Iterators
 -----------------------
 
@@ -323,13 +330,13 @@ support for users who avoid exceptions. See [the simdjson error handling documen
 * **Validate What You Use:** When calling `iterate`, the document is quickly indexed. If it is
   not a valid Unicode (UTF-8) string or if there is an unclosed string, an error may be reported right away.
   However, it is not fully validated. On Demand only fully validates the values you use and the
-  structure leading to it.
+  structure leading to it. It means that at every step as you traverse the document, you may encounter an error. You can handle errors either with exceptions or with error codes.
 * **Extracting Values:** You can cast a JSON element to a native type:
   `double(element)`. This works for `std::string_view`, double, uint64_t, int64_t, bool,
   ondemand::object and ondemand::array. We also have explicit methods such as `get_string()`, `get_double()`,
   `get_uint64()`, `get_int64()`, `get_bool()`, `get_object()` and `get_array()`. After a cast or an explicit method,
-  the number, string or boolean will be parsed, or the initial `{` or `[` will be verified for ondemand::object and ondemand::array. An exception is thrown if
+  the number, string or boolean will be parsed, or the initial `{` or `[` will be verified for `ondemand::object` and `ondemand::array`. An exception may be thrown if
-  the cast is not possible. Importantly, when getting an ondemand::object or ondemand::array instance, its content is
+  the cast is not possible: there error code is `simdjson::INCORRECT_TYPE` (see [Error Handling](#error-handling)). Importantly, when getting an ondemand::object or ondemand::array instance, its content is
   not validated: you are only guaranteed that the corresponding initial character (`{` or `[`) is present. Thus,
   for example, you could have an ondemand::object instance pointing at the invalid JSON `{ "this is not a valid object" }`: the validation occurs as you access the content.
   The `get_string()` returns a valid UTF-8 string, after
@@ -347,9 +354,9 @@ support for users who avoid exceptions. See [the simdjson error handling documen
   > `get_string()` twice on an object (or to cast an object twice to `std::string_view`).
 * **Field Access:** To get the value of the "foo" field in an object, use `object["foo"]`. This will
   scan through the object looking for the field with the matching string, doing a character-by-character
-  comparison. For efficiency reason, you should avoid looking up the same field repeatedly: e.g., do
+  comparison. It may generate the error `simdjson::NO_SUCH_FIELD` if there is no such key in the object, it may throw an exception (see [Error Handling](#error-handling)). For efficiency reason, you should avoid looking up the same field repeatedly: e.g., do
-  not do `object["foo"]` followed by `object["foo"]` with the same `object` instance. If you consume an
+  not do `object["foo"]` followed by `object["foo"]` with the same `object` instance. Keep in mind that On Demand does not buffer or save the result of the parsing: if you repeatedly access `object["foo"]`, then it must repeatedly seek the key and parse the content. The library does not provide a distinct function to check if a key is present, instead we recommend you attempt to access the key: e.g., by doing `ondemand::value val{}; if(!object["foo"].get(val)) {...}`, you have that `val` contains the requested value inside the if clause.  It is your responsibility as a user to temporarily keep a reference to the value (`auto v = object["foo"]`), or to consume the content and store it in your own data structures. If you consume an
-  object twice: `std::string_view(object["foo"]` followed by `std::string_view(object["foo"]`, your code
+  object twice: `std::string_view(object["foo"]` followed by `std::string_view(object["foo"]` then your code
   is in error. Furthermore, you can only consume one field at a time, on the same object. The
   value instance you get from  `content["bids"]` becomes invalid when you call `content["asks"]`.
   If you have retrieved `content["bids"].get_array()` and you later call
@@ -364,8 +371,9 @@ support for users who avoid exceptions. See [the simdjson error handling documen
   > to support escaped keys, the method `unescaped_key()` provides the desired unescaped keys by
   > parsing and writing out the unescaped keys to a string buffer and returning a `std::string_view`
   > instance. You should expect a performance penalty when using `unescaped_key()`.
+  >
   > ```c++
-  > auto json = R"({"k\u0065y": 1})"_padded;
+  > auto json = R"({"k\u0065y": 1})"_padded;
   > ondemand::parser parser;
   > auto doc = parser.iterate(json);
   > ondemand::object object = doc.get_object();
@@ -879,10 +887,17 @@ auto error = parser.iterate(json).get(doc);
 if (error) { cerr << error << endl; exit(1); }
 ```
 
-When there is no error, the error code simdjson::SUCCESS is returned: it evaluates as false as a Boolean.
+When there is no error, the error code `simdjson::SUCCESS`is returned: it evaluates as false as a Boolean.
 We have several error codes to indicate errors, they all evaluate to true as a Boolean: your software should not generally not depend on exact
 error codes. We may change the error codes in future releases and the exact error codes could vary depending on your system.
 
+Some errors are recoverable:
+
+* You may get the error `simdjson::INCORRECT_TYPE` after trying to convert a value to an incorrect type: e.g., you expected a number and try to convert the value to a number, but it is an array.
+* You may query a key from an object, but the key is missing in which case you get the error `simdjson::NO_SUCH_FIELD`: e.g., you call `obj["myname"]` and the object does not have a key `"myname"`.
+
+Other errors (e.g., `simdjson::INCOMPLETE_ARRAY_OR_OBJECT`) may indicate a fatal error and often follow from the fact that the document is not valid JSON. In which case, it is no longer possible to continue accessing the document: calling the method `is_alive()` on the document instance returns false. All following accesses will keep returning the same fatal error (e.g., `simdjson::INCOMPLETE_ARRAY_OR_OBJECT`).
+
 When you use the code without exceptions, it is your responsibility to check for error before using the
 result: if there is an error, the result value will not be valid and using it will caused undefined behavior. Most compilers should be able to help you if you activate the right
 set of warnings: they can identify variables that are written to but never otherwise accessed.
@@ -1142,6 +1157,28 @@ int main(void) {
 }
 ```
 
+
+You can do handle errors gracefully as well...
+
+```C++
+#include <iostream>
+#include "simdjson.h"
+int main(void) {
+  simdjson::ondemand::parser parser;
+  simdjson::padded_string json_string;
+  simdjson::ondemand::document doc;
+  try {
+    json_string = padded_string::load("twitter.json");
+    doc = parser.iterate(json_string);
+    uint64_t identifier = doc["statuses"].at(0)["id"];
+    std::cout << identifier << std::endl;
+  } catch (simdjson::simdjson_error &error) {
+    std::cerr << "JSON error: " << error.what() << " near "
+              << doc.current_location() << " in " << json_string << std::endl;
+  }
+}
+```
+
 ### Current location in document
 
 Sometimes, it might be helpful to know the current location in the document during iteration. This is especially useful when encountering errors. The `current_location()` method on a
@@ -1304,41 +1341,6 @@ internal string buffer. Thus you should consume values only once
 even if you can iterate through the array or object more than once.
 If you unescape a string within an array more than once, you have unsafe code.
 
-Direct Access to the Raw String
---------------------------------
-
-The simdjson library makes explicit assumptions about types. For examples, numbers
-must be integers (up to 64-bit integers) or binary64 floating-point numbers. Some users
-have different needs. For example, some users might want to support big integers.
-The library makes this possible by providing a `raw_json_token` method which returns
-a `std::string_view` instance containing the value as a string which you may then
-parse as you see fit.
-
-```C++
-simdjson::ondemand::parser parser;
-simdjson::padded_string docdata =  R"({"value":12321323213213213213213213213211223})"_padded;
-simdjson::ondemand::document doc = parser.iterate(docdata);
-simdjson::ondemand::object obj = doc.get_object();
-std::string_view token = obj["value"].raw_json_token();
-// token has value 12321323213213213213213213213211223, it points inside the input string
-```
-
-The `raw_json_token` method even works when the JSON value is a string. In such cases, it
-will return the complete string with the quotes and with eventual escaped sequences as in the
-source document.
-
-```C++
-simdjson::ondemand::parser parser;
-simdjson::padded_string docdata =  R"({"value":"12321323213213213213213213213211223"})"_padded;
-simdjson::ondemand::document doc = parser.iterate(docdata);
-simdjson::ondemand::object obj = doc.get_object();
-string_view token = obj["value"].raw_json_token();
-// token has value "12321323213213213213213213213211223", it points inside the input string
-```
-
-The `raw_json_token()` should be fast and free of allocation.
-
-
 
 Newline-Delimited JSON (ndjson) and JSON lines
 ----------------------------------------------
@@ -1427,10 +1429,9 @@ This parser can't support a document that big
 
 If your documents are large (e.g., larger than a megabyte), then the `iterate_many` function is maybe ill-suited. It is really meant to support reading efficiently streams of relatively small documents (e.g., a few kilobytes each). If you have larger documents, you should use other functions like `iterate`.
 
+We also provide some support for comma-separated documents and other advanced features.
 See [iterate_many.md](iterate_many.md) for detailed information and design.
 
-
-
 Parsing Numbers Inside Strings
 ------------------------------
 
@@ -1643,6 +1644,78 @@ JSON string to a user-provided buffer:
     }
 ```
 
+
+General Direct Access to the Raw JSON String
+--------------------------------
+If your value is a string, the `raw_json_string` gives you direct access to the unprocess
+string. The simdjson library allows you to have access to the raw underlying JSON
+more generally.
+
+The simdjson library makes explicit assumptions about types. For examples, numbers
+must be integers (up to 64-bit integers) or binary64 floating-point numbers. Some users
+have different needs. For example, some users might want to support big integers.
+The library makes this possible by providing a `raw_json_token` method which returns
+a `std::string_view` instance containing the value as a string which you may then
+parse as you see fit.
+
+```C++
+simdjson::ondemand::parser parser;
+simdjson::padded_string docdata =  R"({"value":12321323213213213213213213213211223})"_padded;
+simdjson::ondemand::document doc = parser.iterate(docdata);
+simdjson::ondemand::object obj = doc.get_object();
+std::string_view token = obj["value"].raw_json_token();
+// token has value 12321323213213213213213213213211223, it points inside the input string
+```
+
+The `raw_json_token` method even works when the JSON value is a string. In such cases, it
+will return the complete string with the quotes and with eventual escaped sequences as in the
+source document.
+
+```C++
+simdjson::ondemand::parser parser;
+simdjson::padded_string docdata =  R"({"value":"12321323213213213213213213213211223"})"_padded;
+simdjson::ondemand::document doc = parser.iterate(docdata);
+simdjson::ondemand::object obj = doc.get_object();
+string_view token = obj["value"].raw_json_token();
+// token has value "12321323213213213213213213213211223", it points inside the input string
+```
+
+The `raw_json_token()` should be fast and free of allocation.
+
+If you value is an array or an object, `raw_json_token()` returns effectively a single
+character (`[`) or (`}`) which is not very useful. For arrays and objects, we have another
+method called `raw_json()` which consumes (traverse) the array or the object.
+
+```C++
+simdjson::ondemand::parser parser;
+simdjson::padded_string docdata =  R"({"value":123})"_padded;
+simdjson::ondemand::document doc = parser.iterate(docdata);
+simdjson::ondemand::object obj = doc.get_object();
+string_view token = obj.raw_json(); // gives you `{"value":123}`
+```
+
+
+```C++
+simdjson::ondemand::parser parser;
+simdjson::padded_string docdata =  R"([1,2,3])"_padded;
+simdjson::ondemand::document doc = parser.iterate(docdata);
+simdjson::ondemand::array arr = doc.get_array();
+string_view token = arr.raw_json(); // gives you `[1,2,3]`
+```
+
+Because `raw_json()` consumes to object or the array, if you want to both have
+access to the raw string, and also use the array or object, you should call `reset()`.
+
+```C++
+simdjson::ondemand::parser parser;
+simdjson::padded_string docdata =  R"({"value":123})"_padded;
+simdjson::ondemand::document doc = parser.iterate(docdata);
+simdjson::ondemand::object obj = doc.get_object();
+string_view token = obj.raw_json(); // gives you `{"value":123}`
+obj.reset(); // revise the object
+uint64_t x = obj["value"]; // gives me 123
+```
+
 Thread Safety
 -------------
 
@@ -1881,6 +1954,8 @@ Performance Tips
 - Given a field `field` in an object, calling `field.key()` is often faster than `field.unescaped_key()` so if you do not need an unescaped `std::string_view` instance, prefer `field.key()`.
 - For release builds, we recommend setting `NDEBUG` pre-processor directive when compiling the `simdjson` library. Importantly, using the optimization flags `-O2` or `-O3` under GCC and LLVM clang does not set the `NDEBUG` directive, you must set it manually (e.g., `-DNDEBUG`).
 - For long streams of JSON documents, consider [`iterate_many`](iterate_many.md) and [`parse_many`](parse_many.md) for better performance.
+- Never seek to access a field twice (e.g., o["data"] and later again o["data"]). Instead capture once an ondemand::value and reuse it.
+- If you must access several different keys in an object, it might be preferable to iterate through all the fields in the object instead, and branch on the field keys.
 - If possible, refer to each object and array in your code once. For example, the following code repeatedly refers to the `"data"` key to create an object...
 	```C++
 	std::string_view make = o["data"]["make"];
@@ -1894,4 +1969,4 @@ Performance Tips
 	std::string_view year = data["year"];
 	std::string_view rating = data["rating"];
   ```
-- To better understand the operation of your On Demand parser, and whether it is performing as well as you think it should be, there is a  logger feature built in to simdjson! To use it, define the pre-processor directive `SIMDJSON_VERBOSE_LOGGING` prior to including the `simdjson.h` header, which enables logging in simdjson. Run your code. It may generate a lot of logging output; adding printouts from your application that show each section may be helpful. The log’s output will show step-by-step information on state, buffer pointer position, depth, and key retrieval status.
+- To better understand the operation of your On Demand parser, and whether it is performing as well as you think it should be, there is a  logger feature built in to simdjson! To use it, define the pre-processor directive `SIMDJSON_VERBOSE_LOGGING` prior to including the `simdjson.h` header, which enables logging in simdjson. Run your code. It may generate a lot of logging output; adding printouts from your application that show each section may be helpful. The log's output will show step-by-step information on state, buffer pointer position, depth, and key retrieval status. Importantly, unless `SIMDJSON_VERBOSE_LOGGING` is defined, logging is entirely disabled and thus carries no overhead.

doc/iterate_many.md

@@ -129,6 +129,18 @@ Some official formats **(non-exhaustive list)**:
 API
 ---
 
+Example:
+
+```c++
+auto json = R"({ "foo": 1 } { "foo": 2 } { "foo": 3 } )"_padded;
+ondemand::parser parser;
+ondemand::document_stream docs = parser.iterate_many(json);
+for (auto doc : docs) {
+  std::cout << doc["foo"] << std::endl;
+}
+// Prints 1 2 3
+```
+
 See [basics.md](basics.md#newline-delimited-json-ndjson-and-json-lines) for an overview of the API.
 
 ## Use cases
@@ -237,3 +249,39 @@ This will print:
 ```
 
 Importantly, you should only call `truncated_bytes()` after iterating through all of the documents since the stream cannot tell whether there are truncated documents at the very end when it may not have accessed that part of the data yet.
+
+Comma-separated documents
+-----------
+
+We also support comma-separated documents, but with some performance limitations. The `iterate_many` function  takes in an option to allow parsing of comma separated documents (which defaults on false). In this mode, the entire buffer is processed in one batch. Therefore, the total size of the document should not exceed the maximal capacity of the parser (4 GB). This mode also effectively disallow multithreading. It is therefore mostly suitable for not "very large" inputs. In this mode, the batch_size parameter
+is effectively ignored, as it is set to at least the document size.
+
+Example:
+
+```C++
+    auto json = R"( 1, 2, 3, 4, "a", "b", "c", {"hello": "world"} , [1, 2, 3])"_padded;
+    ondemand::parser parser;
+    ondemand::document_stream doc_stream;
+    // We pass '32' as the batch size, but it is a bogus parameter because, since
+    // we pass 'true' to the allow_comma parameter, the batch size will be set to at least
+    // the document size.
+    auto error = parser.iterate_many(json, 32, true).get(doc_stream);
+    if(error) { std::cerr << error << std::endl; return; }
+    for (auto doc : doc_stream) {
+        std::cout << doc.type() << std::endl;
+    }
+ ```
+
+ This will print:
+
+```
+number
+number
+number
+number
+string
+string
+string
+object
+array
+```

include/simdjson/arm64/numberparsing.h

@@ -3,10 +3,11 @@
 
 namespace simdjson {
 namespace SIMDJSON_IMPLEMENTATION {
-namespace {
+namespace numberparsing {
 
 // we don't have SSE, so let us use a scalar function
 // credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/
+/** @private */
 static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) {
   uint64_t val;
   std::memcpy(&val, chars, sizeof(uint64_t));
@@ -15,7 +16,7 @@ static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars
   return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32);
 }
 
-} // unnamed namespace
+} // namespace numberparsing
 } // namespace SIMDJSON_IMPLEMENTATION
 } // namespace simdjson
 

include/simdjson/fallback/numberparsing.h

@@ -10,8 +10,9 @@ void found_float(double result, const uint8_t *buf);
 
 namespace simdjson {
 namespace SIMDJSON_IMPLEMENTATION {
-namespace {
+namespace numberparsing {
 // credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/
+/** @private */
 static simdjson_inline uint32_t parse_eight_digits_unrolled(const char *chars) {
   uint64_t val;
   memcpy(&val, chars, sizeof(uint64_t));
@@ -19,11 +20,12 @@ static simdjson_inline uint32_t parse_eight_digits_unrolled(const char *chars) {
   val = (val & 0x00FF00FF00FF00FF) * 6553601 >> 16;
   return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32);
 }
+/** @private */
 static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) {
   return parse_eight_digits_unrolled(reinterpret_cast<const char *>(chars));
 }
 
-} // unnamed namespace
+} // namespace numberparsing
 } // namespace SIMDJSON_IMPLEMENTATION
 } // namespace simdjson
 

include/simdjson/generic/numberparsing.h

@@ -3,8 +3,9 @@
 
 namespace simdjson {
 namespace SIMDJSON_IMPLEMENTATION {
+/// @private
+namespace numberparsing {
 
-namespace ondemand {
 /**
  * The type of a JSON number
  */
@@ -13,13 +14,6 @@ enum class number_type {
     signed_integer,          /// a signed integer that fits in a 64-bit word using two's complement
     unsigned_integer         /// a positive integer larger or equal to 1<<63
 };
-}
-
-namespace {
-/// @private
-namespace numberparsing {
-
-
 
 #ifdef JSON_TEST_NUMBERS
 #define INVALID_NUMBER(SRC) (found_invalid_number((SRC)), NUMBER_ERROR)
@@ -34,6 +28,7 @@ namespace numberparsing {
 #endif
 
 namespace {
+
 // Convert a mantissa, an exponent and a sign bit into an ieee64 double.
 // The real_exponent needs to be in [0, 2046] (technically real_exponent = 2047 would be acceptable).
 // The mantissa should be in [0,1<<53). The bit at index (1ULL << 52) while be zeroed.
@@ -45,7 +40,7 @@ simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool
     std::memcpy(&d, &mantissa, sizeof(d));
     return d;
 }
-}
+
 // Attempts to compute i * 10^(power) exactly; and if "negative" is
 // true, negate the result.
 // This function will only work in some cases, when it does not work, success is
@@ -62,10 +57,11 @@ simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative,
 #endif
 #if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0)
   // We cannot be certain that x/y is rounded to nearest.
-  if (0 <= power && power <= 22 && i <= 9007199254740991) {
+  if (0 <= power && power <= 22 && i <= 9007199254740991)
 #else
-  if (-22 <= power && power <= 22 && i <= 9007199254740991) {
+  if (-22 <= power && power <= 22 && i <= 9007199254740991)
 #endif
+  {
     // convert the integer into a double. This is lossless since
     // 0 <= i <= 2^53 - 1.
     d = double(i);
@@ -317,6 +313,7 @@ static bool parse_float_fallback(const uint8_t *ptr, double *outDouble) {
   // to handle that max may be a macro on windows).
   return !(*outDouble > (std::numeric_limits<double>::max)() || *outDouble < std::numeric_limits<double>::lowest());
 }
+
 static bool parse_float_fallback(const uint8_t *ptr, const uint8_t *end_ptr, double *outDouble) {
   *outDouble = simdjson::internal::from_chars(reinterpret_cast<const char *>(ptr), reinterpret_cast<const char *>(end_ptr));
   // We do not accept infinite values.
@@ -350,16 +347,6 @@ simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) {
           0x3333333333333333);
 }
 
-template<typename W>
-error_code slow_float_parsing(simdjson_unused const uint8_t * src, W writer) {
-  double d;
-  if (parse_float_fallback(src, &d)) {
-    writer.append_double(d);
-    return SUCCESS;
-  }
-  return INVALID_NUMBER(src);
-}
-
 template<typename I>
 SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later
 simdjson_inline bool parse_digit(const uint8_t c, I &i) {
@@ -372,7 +359,7 @@ simdjson_inline bool parse_digit(const uint8_t c, I &i) {
   return true;
 }
 
-simdjson_inline error_code parse_decimal(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) {
+simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) {
   // we continue with the fiction that we have an integer. If the
   // floating point number is representable as x * 10^z for some integer
   // z that fits in 53 bits, then we will be able to convert back the
@@ -460,6 +447,20 @@ simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t d
   return digit_count - size_t(start - start_digits);
 }
 
+} // unnamed namespace
+
+/** @private */
+template<typename W>
+error_code slow_float_parsing(simdjson_unused const uint8_t * src, W writer) {
+  double d;
+  if (parse_float_fallback(src, &d)) {
+    writer.append_double(d);
+    return SUCCESS;
+  }
+  return INVALID_NUMBER(src);
+}
+
+/** @private */
 template<typename W>
 simdjson_inline error_code write_float(const uint8_t *const src, bool negative, uint64_t i, const uint8_t * start_digits, size_t digit_count, int64_t exponent, W &writer) {
   // If we frequently had to deal with long strings of digits,
@@ -531,7 +532,7 @@ simdjson_unused simdjson_inline simdjson_result<int64_t> parse_integer_in_string
 simdjson_unused simdjson_inline simdjson_result<double> parse_double_in_string(const uint8_t * const src) noexcept { return 0; }
 simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept  { return false; }
 simdjson_unused simdjson_inline simdjson_result<bool> is_integer(const uint8_t * src) noexcept  { return false; }
-simdjson_unused simdjson_inline simdjson_result<ondemand::number_type> get_number_type(const uint8_t * src) noexcept { return ondemand::number_type::signed_integer; }
+simdjson_unused simdjson_inline simdjson_result<number_type> get_number_type(const uint8_t * src) noexcept { return number_type::signed_integer; }
 #else
 
 // parse the number at src
@@ -573,7 +574,7 @@ simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) {
   if ('.' == *p) {
     is_float = true;
     ++p;
-    SIMDJSON_TRY( parse_decimal(src, p, i, exponent) );
+    SIMDJSON_TRY( parse_decimal_after_separator(src, p, i, exponent) );
     digit_count = int(p - start_digits); // used later to guard against overflows
   }
   if (('e' == *p) || ('E' == *p)) {
@@ -1061,7 +1062,7 @@ simdjson_unused simdjson_inline simdjson_result<bool> is_integer(const uint8_t *
   return false;
 }
 
-simdjson_unused simdjson_inline simdjson_result<ondemand::number_type> get_number_type(const uint8_t * src) noexcept {
+simdjson_unused simdjson_inline simdjson_result<number_type> get_number_type(const uint8_t * src) noexcept {
   bool negative = (*src == '-');
   src += uint8_t(negative);
   const uint8_t *p = src;
@@ -1070,20 +1071,20 @@ simdjson_unused simdjson_inline simdjson_result<ondemand::number_type> get_numbe
   if (jsoncharutils::is_structural_or_whitespace(*p)) {
     // We have an integer.
     // If the number is negative and valid, it must be a signed integer.
-    if(negative) { return ondemand::number_type::signed_integer; }
+    if(negative) { return number_type::signed_integer; }
     // We want values larger or equal to 9223372036854775808 to be unsigned
     // integers, and the other values to be signed integers.
     int digit_count = int(p - src);
     if(digit_count >= 19) {
       const uint8_t * smaller_big_integer = reinterpret_cast<const uint8_t *>("9223372036854775808");
       if((digit_count >= 20) || (memcmp(src, smaller_big_integer, 19) >= 0)) {
-        return ondemand::number_type::unsigned_integer;
+        return number_type::unsigned_integer;
       }
     }
-    return ondemand::number_type::signed_integer;
+    return number_type::signed_integer;
   }
   // Hopefully, we have 'e' or 'E' or '.'.
-  return ondemand::number_type::floating_point_number;
+  return number_type::floating_point_number;
 }
 
 // Never read at src_end or beyond
@@ -1245,10 +1246,20 @@ simdjson_unused simdjson_inline simdjson_result<double> parse_double_in_string(c
   }
   return d;
 }
-} //namespace {}
+
+} // unnamed namespace
 #endif // SIMDJSON_SKIPNUMBERPARSING
 
+inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept {
+    switch (type) {
+        case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break;
+        case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break;
+        case number_type::floating_point_number: out << "floating-point number (binary64)"; break;
+        default: SIMDJSON_UNREACHABLE();
+    }
+    return out;
+}
+
 } // namespace numberparsing
-} // unnamed namespace
 } // namespace SIMDJSON_IMPLEMENTATION
 } // namespace simdjson

include/simdjson/generic/ondemand.h

@@ -10,6 +10,9 @@ namespace ondemand {
 /** Represents the depth of a JSON value (number of nested arrays/objects). */
 using depth_t = int32_t;
 
+/** @copydoc simdjson::SIMDJSON_IMPLEMENTATION::numberparsing::number_type */
+using number_type = simdjson::SIMDJSON_IMPLEMENTATION::numberparsing::number_type;
+
 } // namespace ondemand
 } // namespace SIMDJSON_IMPLEMENTATION
 } // namespace simdjson

include/simdjson/generic/ondemand/array-inl.h

@@ -205,4 +205,8 @@ simdjson_inline  simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdj
   if (error()) { return error(); }
   return first.at_pointer(json_pointer);
 }
+simdjson_inline  simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array>::raw_json() noexcept {
+  if (error()) { return error(); }
+  return first.raw_json();
+}
 } // namespace simdjson

include/simdjson/generic/ondemand/array.h

@@ -186,6 +186,8 @@ public:
   inline simdjson_result<bool> reset() & noexcept;
   simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> at(size_t index) noexcept;
   simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> at_pointer(std::string_view json_pointer) noexcept;
+  simdjson_inline simdjson_result<std::string_view> raw_json() noexcept;
+
 };
 
 } // namespace simdjson

include/simdjson/generic/ondemand/document-inl.h

@@ -456,7 +456,7 @@ simdjson_inline simdjson_result<bool> simdjson_result<SIMDJSON_IMPLEMENTATION::o
   return first.is_integer();
 }
 
-simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number_type> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::get_number_type() noexcept {
+simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::numberparsing::number_type> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::get_number_type() noexcept {
   if (error()) { return error(); }
   return first.get_number_type();
 }
@@ -739,7 +739,7 @@ simdjson_inline simdjson_result<bool> simdjson_result<SIMDJSON_IMPLEMENTATION::o
   if (error()) { return error(); }
   return first.is_integer();
 }
-simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number_type> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::get_number_type() noexcept {
+simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::numberparsing::number_type> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::get_number_type() noexcept {
   if (error()) { return error(); }
   return first.get_number_type();
 }

include/simdjson/generic/ondemand/document.h

@@ -730,7 +730,7 @@ public:
   simdjson_inline bool at_end() const noexcept;
   simdjson_inline bool is_negative() noexcept;
   simdjson_inline simdjson_result<bool> is_integer() noexcept;
-  simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number_type> get_number_type() noexcept;
+  simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::numberparsing::number_type> get_number_type() noexcept;
   simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number> get_number() noexcept;
   /** @copydoc simdjson_inline std::string_view document::raw_json_token() const noexcept */
   simdjson_inline simdjson_result<std::string_view> raw_json_token() noexcept;
@@ -795,7 +795,7 @@ public:
   simdjson_inline simdjson_result<int32_t> current_depth() const noexcept;
   simdjson_inline simdjson_result<bool> is_negative() noexcept;
   simdjson_inline simdjson_result<bool> is_integer() noexcept;
-  simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number_type> get_number_type() noexcept;
+  simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::numberparsing::number_type> get_number_type() noexcept;
   simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number> get_number() noexcept;
   /** @copydoc simdjson_inline std::string_view document_reference::raw_json_token() const noexcept */
   simdjson_inline simdjson_result<std::string_view> raw_json_token() noexcept;

include/simdjson/generic/ondemand/document_stream-inl.h

@@ -84,12 +84,14 @@ simdjson_inline document_stream::document_stream(
   ondemand::parser &_parser,
   const uint8_t *_buf,
   size_t _len,
-  size_t _batch_size
+  size_t _batch_size,
+  bool _allow_comma_separated
 ) noexcept
   : parser{&_parser},
     buf{_buf},
     len{_len},
     batch_size{_batch_size <= MINIMAL_BATCH_SIZE ? MINIMAL_BATCH_SIZE : _batch_size},
+    allow_comma_separated{_allow_comma_separated},
     error{SUCCESS}
     #ifdef SIMDJSON_THREADS_ENABLED
     , use_thread(_parser.threaded) // we need to make a copy because _parser.threaded can change
@@ -107,6 +109,7 @@ simdjson_inline document_stream::document_stream() noexcept
     buf{nullptr},
     len{0},
     batch_size{0},
+    allow_comma_separated{false},
     error{UNINITIALIZED}
     #ifdef SIMDJSON_THREADS_ENABLED
     , use_thread(false)
@@ -290,6 +293,8 @@ inline void document_stream::next_document() noexcept {
   if (error) { return; }
   // Always set depth=1 at the start of document
   doc.iter._depth = 1;
+  // consume comma if comma separated is allowed
+  if (allow_comma_separated) { doc.iter.consume_character(','); }
   // Resets the string buffer at the beginning, thus invalidating the strings.
   doc.iter._string_buf_loc = parser->string_buf.get();
   doc.iter._root = doc.iter.position();

include/simdjson/generic/ondemand/document_stream.h

@@ -222,7 +222,8 @@ private:
     ondemand::parser &parser,
     const uint8_t *buf,
     size_t len,
-    size_t batch_size
+    size_t batch_size,
+    bool allow_comma_separated
   ) noexcept;
 
   /**
@@ -271,6 +272,7 @@ private:
   const uint8_t *buf;
   size_t len;
   size_t batch_size;
+  bool allow_comma_separated;
   /**
    * We are going to use just one document instance. The document owns
    * the json_iterator. It implies that we only ever pass a reference

include/simdjson/generic/ondemand/json_iterator-inl.h

@@ -337,6 +337,14 @@ simdjson_inline void json_iterator::reenter_child(token_position position, depth
   _depth = child_depth;
 }
 
+simdjson_inline error_code json_iterator::consume_character(char c) noexcept {
+  if (*peek() == c) {
+    return_current_and_advance();
+    return SUCCESS;
+  }
+  return TAPE_ERROR;
+}
+
 #if SIMDJSON_DEVELOPMENT_CHECKS
 
 simdjson_inline token_position json_iterator::start_position(depth_t depth) const noexcept {

include/simdjson/generic/ondemand/json_iterator.h

@@ -255,6 +255,7 @@ public:
   simdjson_inline simdjson_result<std::string_view> unescape_wobbly(raw_json_string in) noexcept;
   simdjson_inline void reenter_child(token_position position, depth_t child_depth) noexcept;
 
+  simdjson_inline error_code consume_character(char c) noexcept;
 #if SIMDJSON_DEVELOPMENT_CHECKS
   simdjson_inline token_position start_position(depth_t depth) const noexcept;
   simdjson_inline void set_start_position(depth_t depth, token_position position) noexcept;

include/simdjson/generic/ondemand/json_type-inl.h

@@ -15,15 +15,6 @@ inline std::ostream& operator<<(std::ostream& out, json_type type) noexcept {
     return out;
 }
 
-inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept {
-    switch (type) {
-        case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break;
-        case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break;
-        case number_type::floating_point_number: out << "floating-point number (binary64)"; break;
-        default: SIMDJSON_UNREACHABLE();
-    }
-    return out;
-}
 #if SIMDJSON_EXCEPTIONS
 inline std::ostream& operator<<(std::ostream& out, simdjson_result<json_type> &type) noexcept(false) {
     return out << type.value();

include/simdjson/generic/ondemand/json_type.h

@@ -84,11 +84,11 @@ protected:
    */
   friend class value_iterator;
   template<typename W>
+  friend error_code numberparsing::slow_float_parsing(simdjson_unused const uint8_t * src, W writer);
+  template<typename W>
   friend error_code numberparsing::write_float(const uint8_t *const src, bool negative, uint64_t i, const uint8_t * start_digits, size_t digit_count, int64_t exponent, W &writer);
   template<typename W>
   friend error_code numberparsing::parse_number(const uint8_t *const src, W &writer);
-  template<typename W>
-  friend error_code numberparsing::slow_float_parsing(simdjson_unused const uint8_t * src, W writer);
   /** Store a signed 64-bit value to the number. */
   simdjson_inline void append_s64(int64_t value) noexcept;
   /** Store an unsigned 64-bit value to the number. */
@@ -120,7 +120,6 @@ protected:
  * @param type The json_type.
  */
 inline std::ostream& operator<<(std::ostream& out, json_type type) noexcept;
-inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept;
 
 #if SIMDJSON_EXCEPTIONS
 /**

include/simdjson/generic/ondemand/logger.h

@@ -5,6 +5,9 @@ namespace ondemand {
 class json_iterator;
 class value_iterator;
 
+// Logging should be free unless SIMDJSON_VERBOSE_LOGGING is set. Importantly, it is critical
+// that the call to the log functions be side-effect free. Thus, for example, you should not
+// create temporary std::string instances.
 namespace logger {
 
 #if SIMDJSON_VERBOSE_LOGGING

include/simdjson/generic/ondemand/object-inl.h

@@ -226,4 +226,8 @@ simdjson_inline  simdjson_result<size_t> simdjson_result<SIMDJSON_IMPLEMENTATION
   return first.count_fields();
 }
 
+simdjson_inline  simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::object>::raw_json() noexcept {
+  if (error()) { return error(); }
+  return first.raw_json();
+}
 } // namespace simdjson

include/simdjson/generic/ondemand/object.h

@@ -223,6 +223,7 @@ public:
   inline simdjson_result<bool> reset() noexcept;
   inline simdjson_result<bool> is_empty() noexcept;
   inline simdjson_result<size_t> count_fields() & noexcept;
+  inline simdjson_result<std::string_view> raw_json() noexcept;
 
 };
 

include/simdjson/generic/ondemand/parser-inl.h

@@ -84,18 +84,19 @@ simdjson_warn_unused simdjson_inline simdjson_result<json_iterator> parser::iter
   return json_iterator(reinterpret_cast<const uint8_t *>(json.data()), this);
 }
 
-inline simdjson_result<document_stream> parser::iterate_many(const uint8_t *buf, size_t len, size_t batch_size) noexcept {
+inline simdjson_result<document_stream> parser::iterate_many(const uint8_t *buf, size_t len, size_t batch_size, bool allow_comma_separated) noexcept {
   if(batch_size < MINIMAL_BATCH_SIZE) { batch_size = MINIMAL_BATCH_SIZE; }
-  return document_stream(*this, buf, len, batch_size);
+  if(allow_comma_separated && batch_size < len) { batch_size = len; }
+  return document_stream(*this, buf, len, batch_size, allow_comma_separated);
 }
-inline simdjson_result<document_stream> parser::iterate_many(const char *buf, size_t len, size_t batch_size) noexcept {
+inline simdjson_result<document_stream> parser::iterate_many(const char *buf, size_t len, size_t batch_size, bool allow_comma_separated) noexcept {
-  return iterate_many(reinterpret_cast<const uint8_t *>(buf), len, batch_size);
+  return iterate_many(reinterpret_cast<const uint8_t *>(buf), len, batch_size, allow_comma_separated);
 }
-inline simdjson_result<document_stream> parser::iterate_many(const std::string &s, size_t batch_size) noexcept {
+inline simdjson_result<document_stream> parser::iterate_many(const std::string &s, size_t batch_size, bool allow_comma_separated) noexcept {
-  return iterate_many(s.data(), s.length(), batch_size);
+  return iterate_many(s.data(), s.length(), batch_size, allow_comma_separated);
 }
-inline simdjson_result<document_stream> parser::iterate_many(const padded_string &s, size_t batch_size) noexcept {
+inline simdjson_result<document_stream> parser::iterate_many(const padded_string &s, size_t batch_size, bool allow_comma_separated) noexcept {
-  return iterate_many(s.data(), s.length(), batch_size);
+  return iterate_many(s.data(), s.length(), batch_size, allow_comma_separated);
 }
 
 simdjson_inline size_t parser::capacity() const noexcept {

include/simdjson/generic/ondemand/parser.h

@@ -212,21 +212,26 @@ public:
    *                   spot is cache-related: small enough to fit in cache, yet big enough to
    *                   parse as many documents as possible in one tight loop.
    *                   Defaults to 10MB, which has been a reasonable sweet spot in our tests.
+   * @param allow_comma_separated (defaults on false) This allows a mode where the documents are
+   *                   separated by commas instead of whitespace. It comes with a performance
+   *                   penalty because the entire document is indexed at once (and the document must be
+   *                   less than 4 GB), and there is no multithreading. In this mode, the batch_size parameter
+   *                   is effectively ignored, as it is set to at least the document size.
    * @return The stream, or an error. An empty input will yield 0 documents rather than an EMPTY error. Errors:
    *         - MEMALLOC if the parser does not have enough capacity and memory allocation fails
    *         - CAPACITY if the parser does not have enough capacity and batch_size > max_capacity.
    *         - other json errors if parsing fails. You should not rely on these errors to always the same for the
    *           same document: they may vary under runtime dispatch (so they may vary depending on your system and hardware).
    */
-  inline simdjson_result<document_stream> iterate_many(const uint8_t *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept;
+  inline simdjson_result<document_stream> iterate_many(const uint8_t *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept;
   /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */
-  inline simdjson_result<document_stream> iterate_many(const char *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept;
+  inline simdjson_result<document_stream> iterate_many(const char *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept;
   /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */
-  inline simdjson_result<document_stream> iterate_many(const std::string &s, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept;
+  inline simdjson_result<document_stream> iterate_many(const std::string &s, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept;
-  inline simdjson_result<document_stream> iterate_many(const std::string &&s, size_t batch_size) = delete;// unsafe
+  inline simdjson_result<document_stream> iterate_many(const std::string &&s, size_t batch_size, bool allow_comma_separated = false) = delete;// unsafe
   /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */
-  inline simdjson_result<document_stream> iterate_many(const padded_string &s, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept;
+  inline simdjson_result<document_stream> iterate_many(const padded_string &s, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept;
-  inline simdjson_result<document_stream> iterate_many(const padded_string &&s, size_t batch_size) = delete;// unsafe
+  inline simdjson_result<document_stream> iterate_many(const padded_string &&s, size_t batch_size, bool allow_comma_separated = false) = delete;// unsafe
 
   /** @private We do not want to allow implicit conversion from C string to std::string. */
   simdjson_result<document_stream> iterate_many(const char *buf, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept = delete;

include/simdjson/generic/ondemand/value-inl.h

@@ -357,7 +357,7 @@ simdjson_inline simdjson_result<bool> simdjson_result<SIMDJSON_IMPLEMENTATION::o
   if (error()) { return error(); }
   return first.is_integer();
 }
-simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number_type> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::get_number_type() noexcept {
+simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::numberparsing::number_type> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::get_number_type() noexcept {
   if (error()) { return error(); }
   return first.get_number_type();
 }

include/simdjson/generic/ondemand/value.h

@@ -688,7 +688,7 @@ public:
   simdjson_inline simdjson_result<bool> is_scalar() noexcept;
   simdjson_inline simdjson_result<bool> is_negative() noexcept;
   simdjson_inline simdjson_result<bool> is_integer() noexcept;
-  simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number_type> get_number_type() noexcept;
+  simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::numberparsing::number_type> get_number_type() noexcept;
   simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number> get_number() noexcept;
 
   /** @copydoc simdjson_inline std::string_view value::raw_json_token() const noexcept */

include/simdjson/generic/ondemand/value_iterator-inl.h

@@ -584,7 +584,7 @@ simdjson_inline simdjson_result<bool> value_iterator::is_root_integer(bool check
   return answer;
 }
 
-simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number_type> value_iterator::get_root_number_type(bool check_trailing) noexcept {
+simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::numberparsing::number_type> value_iterator::get_root_number_type(bool check_trailing) noexcept {
   auto max_len = peek_start_length();
   auto json = peek_root_scalar("number");
   // Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/,
@@ -946,11 +946,13 @@ simdjson_inline void value_iterator::move_at_container_start() noexcept {
 }
 
 simdjson_inline simdjson_result<bool> value_iterator::reset_array() noexcept {
+  if(error()) { return error(); }
   move_at_container_start();
   return started_array();
 }
 
 simdjson_inline simdjson_result<bool> value_iterator::reset_object() noexcept {
+  if(error()) { return error(); }
   move_at_container_start();
   return started_object();
 }

include/simdjson/haswell.h

@@ -9,7 +9,7 @@
 #define SIMDJSON_TARGET_HASWELL
 #define SIMDJSON_UNTARGET_HASWELL
 #else
-#define SIMDJSON_TARGET_HASWELL SIMDJSON_TARGET_REGION("avx2,bmi,pclmul,lzcnt")
+#define SIMDJSON_TARGET_HASWELL SIMDJSON_TARGET_REGION("avx2,bmi,pclmul,lzcnt,popcnt")
 #define SIMDJSON_UNTARGET_HASWELL SIMDJSON_UNTARGET_REGION
 #endif
 

include/simdjson/haswell/numberparsing.h

@@ -3,8 +3,9 @@
 
 namespace simdjson {
 namespace SIMDJSON_IMPLEMENTATION {
-namespace {
+namespace numberparsing {
 
+/** @private */
 static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) {
   // this actually computes *16* values so we are being wasteful.
   const __m128i ascii0 = _mm_set1_epi8('0');
@@ -23,7 +24,7 @@ static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars
       t4); // only captures the sum of the first 8 digits, drop the rest
 }
 
-} // unnamed namespace
+} // namespace numberparsing
 } // namespace SIMDJSON_IMPLEMENTATION
 } // namespace simdjson
 

include/simdjson/icelake.h

@@ -9,7 +9,7 @@
 #define SIMDJSON_TARGET_ICELAKE
 #define SIMDJSON_UNTARGET_ICELAKE
 #else
-#define SIMDJSON_TARGET_ICELAKE SIMDJSON_TARGET_REGION("avx512f,avx512dq,avx512cd,avx512bw,avx512vbmi,avx512vbmi2,avx512vl,avx2,bmi,pclmul,lzcnt")
+#define SIMDJSON_TARGET_ICELAKE SIMDJSON_TARGET_REGION("avx512f,avx512dq,avx512cd,avx512bw,avx512vbmi,avx512vbmi2,avx512vl,avx2,bmi,pclmul,lzcnt,popcnt")
 #define SIMDJSON_UNTARGET_ICELAKE SIMDJSON_UNTARGET_REGION
 #endif
 

include/simdjson/icelake/numberparsing.h

@@ -3,7 +3,7 @@
 
 namespace simdjson {
 namespace SIMDJSON_IMPLEMENTATION {
-namespace {
+namespace numberparsing {
 
 static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) {
   // this actually computes *16* values so we are being wasteful.
@@ -23,7 +23,7 @@ static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars
       t4); // only captures the sum of the first 8 digits, drop the rest
 }
 
-} // unnamed namespace
+} // namespace numberparsing
 } // namespace SIMDJSON_IMPLEMENTATION
 } // namespace simdjson
 

include/simdjson/implementations.h

@@ -78,9 +78,9 @@
 #define SIMDJSON_CAN_ALWAYS_RUN_WESTMERE (SIMDJSON_IMPLEMENTATION_WESTMERE && SIMDJSON_IS_X86_64 && __SSE4_2__ && __PCLMUL__)
 
 #ifndef SIMDJSON_IMPLEMENTATION_PPC64
-#define SIMDJSON_IMPLEMENTATION_PPC64 (SIMDJSON_IS_PPC64)
+#define SIMDJSON_IMPLEMENTATION_PPC64 (SIMDJSON_IS_PPC64 && SIMDJSON_IS_PPC64_VMX)
 #endif
-#define SIMDJSON_CAN_ALWAYS_RUN_PPC64 SIMDJSON_IMPLEMENTATION_PPC64 && SIMDJSON_IS_PPC64
+#define SIMDJSON_CAN_ALWAYS_RUN_PPC64 SIMDJSON_IMPLEMENTATION_PPC64 && SIMDJSON_IS_PPC64 && SIMDJSON_IS_PPC64_VMX
 
 // Default Fallback to on unless a builtin implementation has already been selected.
 #ifndef SIMDJSON_IMPLEMENTATION_FALLBACK

include/simdjson/internal/isadetection.h

@@ -57,7 +57,6 @@ POSSIBILITY OF SUCH DAMAGE.
 namespace simdjson {
 namespace internal {
 
-
 enum instruction_set {
   DEFAULT = 0x0,
   NEON = 0x1,
@@ -107,7 +106,10 @@ constexpr uint32_t cpuid_avx512cd_bit = 1 << 28;    ///< @private bit 28 of EBX
 constexpr uint32_t cpuid_avx512bw_bit = 1 << 30;    ///< @private bit 30 of EBX for EAX=0x7
 constexpr uint32_t cpuid_avx512vl_bit = 1U << 31;    ///< @private bit 31 of EBX for EAX=0x7
 constexpr uint32_t cpuid_avx512vbmi2_bit = 1 << 6;  ///< @private bit 6 of ECX for EAX=0x7
+constexpr uint64_t cpuid_avx256_saved = uint64_t(1) << 2; ///< @private bit 2 = AVX
+constexpr uint64_t cpuid_avx512_saved = uint64_t(7) << 5; ///< @private bits 5,6,7 = opmask, ZMM_hi256, hi16_ZMM
 constexpr uint32_t cpuid_sse42_bit = 1 << 20;       ///< @private bit 20 of ECX for EAX=0x1
+constexpr uint32_t cpuid_osxsave = (uint32_t(1) << 26) | (uint32_t(1) << 27); ///< @private bits 26+27 of ECX for EAX=0x1
 constexpr uint32_t cpuid_pclmulqdq_bit = 1 << 1;    ///< @private bit  1 of ECX for EAX=0x1
 }
 
@@ -117,7 +119,7 @@ static inline void cpuid(uint32_t *eax, uint32_t *ebx, uint32_t *ecx,
                          uint32_t *edx) {
 #if defined(_MSC_VER)
   int cpu_info[4];
-  __cpuid(cpu_info, *eax);
+  __cpuidex(cpu_info, *eax, *ecx);
   *eax = cpu_info[0];
   *ebx = cpu_info[1];
   *ecx = cpu_info[2];
@@ -135,10 +137,48 @@ static inline void cpuid(uint32_t *eax, uint32_t *ebx, uint32_t *ecx,
 #endif
 }
 
+
+static inline uint64_t xgetbv() {
+#if defined(_MSC_VER)
+  return _xgetbv(0);
+#else
+  uint32_t xcr0_lo, xcr0_hi;
+  asm volatile("xgetbv\n\t" : "=a" (xcr0_lo), "=d" (xcr0_hi) : "c" (0));
+  return xcr0_lo | (uint64_t(xcr0_hi) << 32);
+#endif
+}
+
 static inline uint32_t detect_supported_architectures() {
   uint32_t eax, ebx, ecx, edx;
   uint32_t host_isa = 0x0;
 
+  // EBX for EAX=0x1
+  eax = 0x1;
+  ecx = 0x0;
+  cpuid(&eax, &ebx, &ecx, &edx);
+
+  if (ecx & cpuid_sse42_bit) {
+    host_isa |= instruction_set::SSE42;
+  } else {
+    return host_isa; // everything after is redundant
+  }
+
+  if (ecx & cpuid_pclmulqdq_bit) {
+    host_isa |= instruction_set::PCLMULQDQ;
+  }
+
+
+  if ((ecx & cpuid_osxsave) != cpuid_osxsave) {
+    return host_isa;
+  }
+
+  // xgetbv for checking if the OS saves registers
+  uint64_t xcr0 = xgetbv();
+
+  if ((xcr0 & cpuid_avx256_saved) == 0) {
+    return host_isa;
+  }
+
   // ECX for EAX=0x7
   eax = 0x7;
   ecx = 0x0;
@@ -154,6 +194,10 @@ static inline uint32_t detect_supported_architectures() {
     host_isa |= instruction_set::BMI2;
   }
 
+  if (!((xcr0 & cpuid_avx512_saved) == cpuid_avx512_saved)) {
+     return host_isa;
+  }
+
   if (ebx & cpuid_avx512f_bit) {
     host_isa |= instruction_set::AVX512F;
   }
@@ -190,18 +234,6 @@ static inline uint32_t detect_supported_architectures() {
     host_isa |= instruction_set::AVX512VBMI2;
   }
 
-  // EBX for EAX=0x1
-  eax = 0x1;
-  cpuid(&eax, &ebx, &ecx, &edx);
-
-  if (ecx & cpuid_sse42_bit) {
-    host_isa |= instruction_set::SSE42;
-  }
-
-  if (ecx & cpuid_pclmulqdq_bit) {
-    host_isa |= instruction_set::PCLMULQDQ;
-  }
-
   return host_isa;
 }
 #else // fallback

include/simdjson/portability.h

@@ -32,19 +32,14 @@
 #endif // __clang__
 #endif // _MSC_VER
 
-#if SIMDJSON_REGULAR_VISUAL_STUDIO
-// https://en.wikipedia.org/wiki/C_alternative_tokens
-// This header should have no effect, except maybe
-// under Visual Studio.
-#include <iso646.h>
-#endif
-
 #if defined(__x86_64__) || defined(_M_AMD64)
 #define SIMDJSON_IS_X86_64 1
 #elif defined(__aarch64__) || defined(_M_ARM64)
 #define SIMDJSON_IS_ARM64 1
 #elif defined(__PPC64__) || defined(_M_PPC64)
-#define SIMDJSON_IS_PPC64 1
+#if defined(__ALTIVEC__)
+#define SIMDJSON_IS_PPC64_VMX 1
+#endif // defined(__ALTIVEC__)
 #else
 #define SIMDJSON_IS_32BITS 1
 
@@ -175,8 +170,9 @@ use a 64-bit target such as x64, 64-bit ARM or 64-bit PPC.")
 #define simdjson_strncasecmp strncasecmp
 #endif
 
-#ifdef NDEBUG
+#if defined(NDEBUG) || defined(__OPTIMIZE__) || (defined(_MSC_VER) && !defined(_DEBUG))
-
+// If NDEBUG is set, or __OPTIMIZE__ is set, or we are under MSVC in release mode,
+// then do away with asserts and use __assume.
 #if SIMDJSON_VISUAL_STUDIO
 #define SIMDJSON_UNREACHABLE() __assume(0)
 #define SIMDJSON_ASSUME(COND) __assume(COND)
@@ -185,8 +181,8 @@ use a 64-bit target such as x64, 64-bit ARM or 64-bit PPC.")
 #define SIMDJSON_ASSUME(COND) do { if (!(COND)) __builtin_unreachable(); } while (0)
 #endif
 
-#else // NDEBUG
+#else // defined(NDEBUG) || defined(__OPTIMIZE__) || (defined(_MSC_VER) && !defined(_DEBUG))
-
+// This should only ever be enabled in debug mode.
 #define SIMDJSON_UNREACHABLE() assert(0);
 #define SIMDJSON_ASSUME(COND) assert(COND)
 

include/simdjson/ppc64/numberparsing.h

@@ -9,12 +9,12 @@
 
 namespace simdjson {
 namespace SIMDJSON_IMPLEMENTATION {
-namespace {
+namespace numberparsing {
 
 // we don't have appropriate instructions, so let us use a scalar function
 // credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/
-static simdjson_inline uint32_t
+/** @private */
-parse_eight_digits_unrolled(const uint8_t *chars) {
+static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) {
   uint64_t val;
   std::memcpy(&val, chars, sizeof(uint64_t));
 #ifdef __BIG_ENDIAN__
@@ -29,7 +29,7 @@ parse_eight_digits_unrolled(const uint8_t *chars) {
   return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32);
 }
 
-} // unnamed namespace
+} // namespace numberparsing
 } // namespace SIMDJSON_IMPLEMENTATION
 } // namespace simdjson
 

include/simdjson/simdjson_version.h

@@ -4,7 +4,7 @@
 #define SIMDJSON_SIMDJSON_VERSION_H
 
 /** The version of simdjson being used (major.minor.revision) */
-#define SIMDJSON_VERSION "3.1.7"
+#define SIMDJSON_VERSION "3.2.1"
 
 namespace simdjson {
 enum {
@@ -15,11 +15,11 @@ enum {
   /**
    * The minor version (major.MINOR.revision) of simdjson being used.
    */
-  SIMDJSON_VERSION_MINOR = 1,
+  SIMDJSON_VERSION_MINOR = 2,
   /**
    * The revision (major.minor.REVISION) of simdjson being used.
    */
-  SIMDJSON_VERSION_REVISION = 7
+  SIMDJSON_VERSION_REVISION = 1
 };
 } // namespace simdjson
 

include/simdjson/westmere.h

@@ -9,7 +9,7 @@
 #define SIMDJSON_TARGET_WESTMERE
 #define SIMDJSON_UNTARGET_WESTMERE
 #else
-#define SIMDJSON_TARGET_WESTMERE SIMDJSON_TARGET_REGION("sse4.2,pclmul")
+#define SIMDJSON_TARGET_WESTMERE SIMDJSON_TARGET_REGION("sse4.2,pclmul,popcnt")
 #define SIMDJSON_UNTARGET_WESTMERE SIMDJSON_UNTARGET_REGION
 #endif
 

include/simdjson/westmere/numberparsing.h

@@ -3,8 +3,9 @@
 
 namespace simdjson {
 namespace SIMDJSON_IMPLEMENTATION {
-namespace {
+namespace numberparsing {
 
+/** @private */
 static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) {
   // this actually computes *16* values so we are being wasteful.
   const __m128i ascii0 = _mm_set1_epi8('0');
@@ -23,7 +24,7 @@ static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars
       t4); // only captures the sum of the first 8 digits, drop the rest
 }
 
-} // unnamed namespace
+} // namespace numberparsing
 } // namespace SIMDJSON_IMPLEMENTATION
 } // namespace simdjson
 

singleheader/amalgamate.py

@@ -112,7 +112,8 @@ def dofile(fid, prepath, filename):
                     print(f"// redefining SIMDJSON_IMPLEMENTATION to \"{current_implementation}\"\n// {line}", file=fid)
                 elif undefines_simdjson_implementation.search(line):
                     # Don't include #undef SIMDJSON_IMPLEMENTATION since we're handling it ourselves
-                    print(f"// {line}")
+                    # print(f"// {line}")
+                    pass
                 else:
                     # copy the line, with SIMDJSON_IMPLEMENTATION replace to what it is currently defined to
                     print(uses_simdjson_implementation.sub(current_implementation+"\\1",line), file=fid)
@@ -125,8 +126,10 @@ def dofile(fid, prepath, filename):
 # Forcing it to be UTC is difficult, because it needs to be portable
 # between gnu date and busybox date.
 try:
-    timestamp = subprocess.run(['git', 'show', '-s', '--format=%ci', 'HEAD'],
+    proc = subprocess.run(['git', 'show', '-s', '--format=%ci', 'HEAD'],
-                           stdout=subprocess.PIPE).stdout.decode('utf-8').strip()
+                           stdout=subprocess.PIPE)
+    print("the commandline is {}".format(proc.args))
+    timestamp = proc.stdout.decode('utf-8').strip()
 except:
     print("git not found, timestamp based on current time")
     timestamp = str(datetime.datetime.now())

singleheader/simdjson.cpp

@@ -1,4 +1,4 @@
-/* auto-generated on 2023-04-08 16:11:40 -0400. Do not edit! */
+/* auto-generated on 2023-07-06 21:34:14 -0400. Do not edit! */
 /* begin file src/simdjson.cpp */
 #include "simdjson.h"
 

tests/dom/basictests.cpp

@@ -9,7 +9,6 @@
 #include <set>
 #include <sstream>
 #include <utility>
-#include <ciso646>
 #include <unistd.h>
 
 #include "simdjson.h"
@@ -67,6 +66,15 @@ namespace number_tests {
     return true;
   }
 
+  bool issue2017() {
+    TEST_START();
+    simdjson::dom::parser parser;
+    simdjson::padded_string docdata = R"({"score":0.8825149536132812})"_padded;
+    double score;
+    ASSERT_SUCCESS(parser.parse(docdata)["score"].get_double().get(score));
+    ASSERT_EQUAL(score, 0.8825149536132812);
+    TEST_SUCCEED();
+  }
 
   bool small_integers() {
     std::cout << __func__ << std::endl;
@@ -378,7 +386,8 @@ namespace number_tests {
   }
 
   bool run() {
-    return truncated_borderline() &&
+    return issue2017() &&
+           truncated_borderline() &&
            specific_tests() &&
            ground_truth() &&
            small_integers() &&

tests/dom/readme_examples_noexceptions.cpp

@@ -81,13 +81,13 @@ void basics_error_3() {
           dom::object innerobj;
           if ((error = key_value.value.get(innerobj))) { cerr << error << endl; exit(1); }
 
-          double va, vb;
+          double va{}, vb{};
           if ((error = innerobj["a"].get(va))) { cerr << error << endl; exit(1); }
           cout << "a: " << va << ", ";
           if ((error = innerobj["b"].get(vb))) { cerr << error << endl; exit(1); }
           cout << "b: " << vb << ", ";
 
-          int64_t vc;
+          int64_t vc{};
           if ((error = innerobj["c"].get(vc))) { cerr << error << endl; exit(1); }
           cout << "c: " << vc << endl;
       }

tests/dom/stringparsingcheck.cpp

@@ -60,7 +60,6 @@ static inline void write_utf8(unsigned codepoint, char *&end) {
     *end++ = static_cast<char>(0x80 | ((codepoint >> 6) & 0x3F));
     *end++ = static_cast<char>(0x80 | (codepoint & 0x3F));
   } else {
-    assert(codepoint < 0x200000);
     *end++ = static_cast<char>(0xF0 | (codepoint >> 18));
     *end++ = static_cast<char>(0x80 | ((codepoint >> 12) & 0x3F));
     *end++ = static_cast<char>(0x80 | ((codepoint >> 6) & 0x3F));

tests/minify_tests.cpp

@@ -1,5 +1,4 @@
 #include <cinttypes>
-#include <ciso646>
 #include <cmath>
 #include <cstdio>
 #include <cstdlib>

tests/ondemand/CMakeLists.txt

@@ -3,6 +3,7 @@ link_libraries(simdjson)
 include_directories(..)
 add_subdirectory(compilation_failure_tests)
 add_cpp_test(ondemand_log_tests              LABELS ondemand acceptance per_implementation)
+add_cpp_test(ondemand_log_error_tests        LABELS ondemand acceptance per_implementation)
 add_cpp_test(ondemand_tostring_tests         LABELS ondemand acceptance per_implementation)
 add_cpp_test(ondemand_active_tests           LABELS ondemand acceptance per_implementation)
 add_cpp_test(ondemand_array_tests            LABELS ondemand acceptance per_implementation)
@@ -24,6 +25,7 @@ add_cpp_test(ondemand_readme_examples        LABELS ondemand acceptance per_impl
 add_cpp_test(ondemand_scalar_tests           LABELS ondemand acceptance per_implementation)
 add_cpp_test(ondemand_twitter_tests          LABELS ondemand acceptance per_implementation)
 add_cpp_test(ondemand_wrong_type_error_tests LABELS ondemand acceptance per_implementation)
+add_cpp_test(ondemand_iterate_many_csv       LABELS ondemand acceptance per_implementation)
 
 if(HAVE_POSIX_FORK AND HAVE_POSIX_WAIT) # assert tests use fork and wait, which aren't on MSVC
   add_cpp_test(ondemand_assert_out_of_order_values LABELS assert per_implementation explicitonly ondemand)

tests/ondemand/ondemand_error_tests.cpp

@@ -6,6 +6,59 @@ using namespace simdjson;
 namespace error_tests {
   using namespace std;
 
+  bool badbadjson() {
+    TEST_START();
+    ondemand::parser parser;
+    auto json_string = R"({
+      "main": "therain"_"in_spain"_
+})"_padded;
+    ondemand::document document;
+    auto error = parser.iterate(json_string).get(document);
+    if(error != simdjson::SUCCESS) {
+      return false;
+    }
+    ondemand::object obj;
+    error =  document.get_object().get(obj);
+    if(error != simdjson::SUCCESS) {
+      return false;
+    }
+    std::string_view name_value{};
+    error = obj["name"].get_string().get(name_value);
+    ASSERT_ERROR(error,TAPE_ERROR);
+    // Check for "main" field
+    std::string_view main_value{};
+    error = obj["main"].get_string().get(main_value);
+    ASSERT_ERROR(error,TAPE_ERROR);
+    TEST_SUCCEED();
+  }
+
+
+  bool badbadjson2() {
+    TEST_START();
+    ondemand::parser parser;
+    auto json_string = R"({
+      "main": "therain"_"in_spain"_
+})"_padded;
+    ondemand::document document;
+    auto error = parser.iterate(json_string).get(document);
+    if(error != simdjson::SUCCESS) {
+      return false;
+    }
+    ondemand::object obj;
+    error =  document.get_object().get(obj);
+    if(error != simdjson::SUCCESS) {
+      return false;
+    }
+
+    std::string_view main_value{};
+    error = obj["main"].get_string().get(main_value);
+    ASSERT_ERROR(error, simdjson::SUCCESS);
+    std::string_view name_value{};
+    error = obj["name"].get_string().get(name_value);
+    ASSERT_ERROR(error,TAPE_ERROR);
+    TEST_SUCCEED();
+  }
+
   bool issue1834() {
     TEST_START();
     ondemand::parser parser;
@@ -313,6 +366,8 @@ namespace error_tests {
 
   bool run() {
     return
+           badbadjson() &&
+           badbadjson2() &&
            issue1834() &&
            issue1834_2() &&
 #if SIMDJSON_EXCEPTIONS

tests/ondemand/ondemand_iterate_many_csv.cpp

@@ -0,0 +1,160 @@
+#include "simdjson.h"
+#include "test_ondemand.h"
+
+#include <cstdint>
+
+using namespace simdjson;
+
+namespace iterate_many_csv_tests {
+using namespace std;
+
+bool normal() {
+  TEST_START();
+  auto json = R"( 1, 2, 3, 4, "a", "b", "c", {"hello": "world"} , [1, 2, 3])"_padded;
+  ondemand::parser parser;
+  ondemand::document_stream doc_stream;
+  ASSERT_SUCCESS(parser.iterate_many(json, json.size(), true).get(doc_stream));
+
+  for (auto doc : doc_stream)
+  {
+    ASSERT_SUCCESS(doc);
+  }
+
+  TEST_SUCCEED();
+}
+
+bool small_batch_size() {
+  TEST_START();
+  auto json = R"( 1, 2, 3, 4, "a", "b", "c", {"hello": "world"} , [1, 2, 3])"_padded;
+  ondemand::parser parser;
+  ondemand::document_stream doc_stream;
+  ASSERT_SUCCESS(parser.iterate_many(json, 32, true).get(doc_stream));
+
+  for (auto doc : doc_stream)
+  {
+    ASSERT_SUCCESS(doc);
+  }
+
+  TEST_SUCCEED();
+}
+
+bool trailing_comma() {
+  TEST_START();
+  auto json = R"(1,)"_padded;
+  ondemand::parser parser;
+  ondemand::document_stream doc_stream;
+  ASSERT_SUCCESS(parser.iterate_many(json, json.size(), true).get(doc_stream));
+
+  for (auto doc : doc_stream)
+  {
+    ASSERT_SUCCESS(doc);
+  }
+
+  TEST_SUCCEED();
+}
+
+bool check_parsed_values() {
+  TEST_START();
+
+  auto json = R"(  1  , "a" , [100, 1]  , {"hello"  :    "world"}  , )"_padded;
+  ondemand::parser parser;
+  ondemand::document_stream doc_stream;
+  ASSERT_SUCCESS(parser.iterate_many(json, json.size(), true).get(doc_stream));
+
+  auto begin = doc_stream.begin();
+  auto end = doc_stream.end();
+  int cnt = 0;
+  auto it = begin;
+  for (; it != end && cnt < 4; ++it, ++cnt) {
+    auto doc = *it;
+    switch (cnt)
+    {
+    case 0:
+    {
+      int64_t actual;
+      ASSERT_SUCCESS(doc.get_int64().get(actual));
+      ASSERT_EQUAL(actual, 1);
+      break;
+    }
+    case 1:
+    {
+      std::string_view sv;
+      ASSERT_SUCCESS(doc.get_string().get(sv));
+      ASSERT_EQUAL(sv, "a");
+      break;
+    }
+    case 2:
+    {
+      std::vector<int64_t> expected{100, 1};
+      ondemand::array arr;
+      ASSERT_SUCCESS(doc.get_array().get(arr));
+      size_t element_count;
+      ASSERT_SUCCESS(arr.count_elements().get(element_count));
+      ASSERT_EQUAL(element_count, 2);
+      int i = 0;
+      for (auto a : arr)
+      {
+        int64_t actual;
+        ASSERT_SUCCESS(a.get(actual));
+        ASSERT_EQUAL(actual, expected[i++]);
+      }
+      break;
+    }
+    case 3:
+    {
+      ondemand::object obj;
+      ASSERT_SUCCESS(doc.get_object().get(obj));
+      std::string_view sv;
+      obj.find_field("hello").get(sv);
+      ASSERT_EQUAL(sv, "world");
+      break;
+    }
+    default:
+      TEST_FAIL("Too many cases")
+    }
+  }
+
+  ASSERT_EQUAL(cnt, 4);
+  ASSERT_TRUE(!(it != end));
+
+  TEST_SUCCEED();
+}
+
+#if SIMDJSON_EXCEPTIONS
+
+bool leading_comma() {
+  TEST_START();
+  auto json = R"(,1)"_padded;
+  ondemand::parser parser;
+  ondemand::document_stream doc_stream;
+  ASSERT_SUCCESS(parser.iterate_many(json, json.size(), true).get(doc_stream));
+
+  try {
+    auto begin = doc_stream.begin();
+    auto end = doc_stream.end();
+    for (auto it = begin; it != end; ++it) {}
+  } catch (simdjson_error& e) {
+    ASSERT_ERROR(e.error(), TAPE_ERROR);
+  }
+
+  TEST_SUCCEED();
+}
+
+#endif
+
+bool run() {
+  return normal() &&
+         small_batch_size() &&
+         trailing_comma() &&
+         check_parsed_values() &&
+#if SIMDJSON_EXCEPTIONS
+         leading_comma() &&
+#endif
+         true;
+}
+
+}
+
+int main(int argc, char *argv[]) {
+  return test_main(argc, argv, iterate_many_csv_tests::run);
+}

tests/ondemand/ondemand_json_pointer_tests.cpp

@@ -195,7 +195,7 @@ namespace json_pointer_tests {
         ASSERT_SUCCESS(parser.iterate(cars_json).get(cars));
         for (int i = 0; i < 3; i++) {
             double x;
-            std::string json_pointer = "/" + std::to_string(i) + "/tire_pressure/1";
+            std::string json_pointer = std::string("/") + std::to_string(i) + std::string("/tire_pressure/1");
             ASSERT_SUCCESS(cars.at_pointer(json_pointer).get(x));
             measured.push_back(x);
         }
@@ -315,7 +315,7 @@ namespace json_pointer_tests {
         std::vector<car_type> content;
         for (int i = 0; i < 3; i++) {
             ondemand::object obj;
-            std::string json_pointer = "/" + std::to_string(i);
+            std::string json_pointer = std::string("/") + std::to_string(i);
             // Each successive at_pointer call invalidates
             // previously parsed values, strings, objects and array.
             ASSERT_SUCCESS(cars.at_pointer(json_pointer).get(obj));
@@ -360,7 +360,7 @@ namespace json_pointer_tests {
         ondemand::document cars = parser.iterate(cars_json);
         std::vector<car_type> content;
         for (int i = 0; i < 3; i++) {
-            std::string json_pointer = "/" + std::to_string(i);
+            std::string json_pointer = std::string("/") + std::to_string(i);
             // Each successive at_pointer call invalidates
             // previously parsed values, strings, objects and array.
             ondemand::object obj(cars.at_pointer(json_pointer).get_object());

tests/ondemand/ondemand_log_error_tests.cpp

@@ -0,0 +1,67 @@
+#define SIMDJSON_VERBOSE_LOGGING 1
+#include "simdjson.h"
+#include "test_ondemand.h"
+
+#include <iostream>
+#include <string>
+#include <stdlib.h>
+
+using namespace simdjson;
+
+namespace log_error_tests {
+#if SIMDJSON_EXCEPTIONS
+
+using namespace std;
+
+bool tape_error()
+{
+  TEST_START();
+  auto json = R"( {"a", "hello"} )"_padded;
+  ondemand::parser parser;
+  try {
+    ondemand::document doc = parser.iterate(json);
+    std::cout << doc["a"] << std::endl;
+    TEST_FAIL("Should have thrown an exception!")
+  } catch (simdjson_error& e) {
+    ASSERT_ERROR(e.error(), TAPE_ERROR);
+  }
+  TEST_SUCCEED();
+}
+
+bool no_such_field()
+{
+  TEST_START();
+  auto json = R"( {"a": "hello"} )"_padded;
+  ondemand::parser parser;
+  try {
+    ondemand::document doc = parser.iterate(json);
+    std::cout << doc["missing_key"] << std::endl;
+    TEST_FAIL("Should have thrown an exception!")
+  } catch (simdjson_error& e) {
+    ASSERT_ERROR(e.error(), NO_SUCH_FIELD);
+  }
+  TEST_SUCCEED();
+}
+#endif // SIMDJSON_EXCEPTIONS
+
+bool run()
+{
+  SIMDJSON_PUSH_DISABLE_WARNINGS
+  SIMDJSON_DISABLE_DEPRECATED_WARNING // Disable CRT_SECURE warning on MSVC: manually verified this is safe
+  std::string str = "SIMDJSON_LOG_LEVEL=ERROR";
+  putenv(str.data());
+  bool rc =
+#if SIMDJSON_EXCEPTIONS
+            tape_error() &&
+            no_such_field() &&
+#endif // #if SIMDJSON_EXCEPTIONS
+            true;
+  SIMDJSON_POP_DISABLE_WARNINGS
+  return rc;
+}
+
+}
+
+int main(int argc, char *argv[]) {
+  return test_main(argc, argv, log_error_tests::run);
+}

tests/ondemand/ondemand_misc_tests.cpp

@@ -133,7 +133,7 @@ namespace misc_tests {
     ASSERT_SUCCESS(parser.iterate(json).get(doc));
     std::string_view view;
     ASSERT_SUCCESS( doc.get_string().get(view));
-    ASSERT_EQUAL(view, u8"wow:\uFFFF");
+    ASSERT_EQUAL(view, "wow:\xef\xbf\xbf");
     TEST_SUCCEED();
   }
 
@@ -536,9 +536,9 @@ namespace misc_tests {
     return true;
   }
   simdjson_warn_unused bool test_raw_json_token(string_view json, string_view expected_token, int expected_start_index = 0) {
-    string title = "'";
+    string title("'");
     title.append(json.data(), json.length());
-    title += "'";
+    title += std::string("'");
     padded_string json_padded = json;
     SUBTEST(title, test_ondemand_doc(json_padded, [&](auto doc) {
       string_view token;
@@ -552,11 +552,11 @@ namespace misc_tests {
     // Test values
     auto json_in_hash = string(R"({"a":)");
     json_in_hash.append(json.data(), json.length());
-    json_in_hash += "}";
+    json_in_hash += std::string("}");
     json_padded = json_in_hash;
-    title = "'";
+    title = std::string("'");
     title.append(json_in_hash.data(), json_in_hash.length());
-    title += "'";
+    title += std::string("'");
     SUBTEST(title, test_ondemand_doc(json_padded, [&](auto doc) {
       string_view token;
       ASSERT_SUCCESS( doc["a"].raw_json_token().get(token) );

tests/ondemand/ondemand_number_tests.cpp

@@ -333,6 +333,8 @@ namespace number_tests {
     TEST_SUCCEED();
   }
 
+
+
   bool get_root_number_tests() {
     TEST_START();
     ondemand::parser parser;
@@ -389,8 +391,21 @@ namespace number_tests {
 
     TEST_SUCCEED();
   }
+  bool issue2017() {
+    TEST_START();
+    ondemand::parser parser;
+    ondemand::document doc;
+    padded_string docdata = R"({"score":0.8825149536132812})"_padded;
+    ASSERT_SUCCESS(parser.iterate(docdata).get(doc));
+    double score;
+    ASSERT_SUCCESS(doc["score"].get_double().get(score));
+    ASSERT_EQUAL(score, 0.8825149536132812);
+    TEST_SUCCEED();
+  }
+
   bool run() {
-    return issue_1898() &&
+    return issue2017() &&
+           issue_1898() &&
            issue1878() &&
            get_root_number_tests() &&
            get_number_tests()&&

tests/ondemand/ondemand_readme_examples.cpp

@@ -21,108 +21,150 @@ bool string2() {
 
 
 #if SIMDJSON_EXCEPTIONS
+bool gen_raw1() {
+  TEST_START();
+  simdjson::ondemand::parser parser;
+  simdjson::padded_string docdata =  R"({"value":123})"_padded;
+  simdjson::ondemand::document doc = parser.iterate(docdata);
+  simdjson::ondemand::object obj = doc.get_object();
+  string_view token = obj.raw_json(); // gives you `{"value":123}`
+  ASSERT_EQUAL(token, R"({"value":123})");
+  TEST_SUCCEED();
+}
 
-  bool at_end() {
+bool gen_raw2() {
-    auto json = R"([1, 2] foo ])"_padded;
+  TEST_START();
-    ondemand::parser parser;
+  simdjson::ondemand::parser parser;
-    ondemand::document doc = parser.iterate(json);
+  simdjson::padded_string docdata =  R"([1,2,3])"_padded;
-    ondemand::array array = doc.get_array();
+  simdjson::ondemand::document doc = parser.iterate(docdata);
-    for (uint64_t values : array) {
+  simdjson::ondemand::array arr = doc.get_array();
-      std::cout << values << std::endl;
+  string_view token = arr.raw_json(); // gives you `[1,2,3]`
-    }
+  ASSERT_EQUAL(token, R"([1,2,3])");
-    if(!doc.at_end()) {
+  TEST_SUCCEED();
-      std::cerr << "trailing content at byte index " << doc.current_location() - json.data() << std::endl;
+}
-    }
-    TEST_SUCCEED();
-  }
-  bool number_tests() {
-    ondemand::parser parser;
-    padded_string docdata = R"([1.0, 3, 1, 3.1415,-13231232,9999999999999999999])"_padded;
-    ondemand::document doc = parser.iterate(docdata);
-    ondemand::array arr = doc.get_array();
-    for(ondemand::value val : arr) {
-      std::cout << val << " ";
-      std::cout << "negative: " << val.is_negative() << " ";
-      std::cout << "is_integer: " << val.is_integer() << " ";
-      ondemand::number num = val.get_number();
-      ondemand::number_type t = num.get_number_type();
-      // direct computation without materializing the number:
-      ondemand::number_type dt = val.get_number_type();
-      if(t != dt) { throw std::runtime_error("bug"); }
-      switch(t) {
-        case ondemand::number_type::signed_integer:
-          std::cout  << "integer: " << int64_t(num) << " ";
-          std::cout  << "integer: " << num.get_int64() << std::endl;
-          break;
-        case ondemand::number_type::unsigned_integer:
-          std::cout  << "large 64-bit integer: " << uint64_t(num) << " ";
-          std::cout << "large 64-bit integer: " << num.get_uint64() << std::endl;
-          break;
-        case ondemand::number_type::floating_point_number:
-          std::cout  << "float: " << double(num) << " ";
-          std::cout << "float: " << num.get_double() << std::endl;
-          break;
-      }
-    }
-    return true;
-  }
 
-void recursive_print_json(ondemand::value element) {
+bool gen_raw3() {
-    bool add_comma;
+  TEST_START();
-    switch (element.type()) {
+  simdjson::ondemand::parser parser;
-    case ondemand::json_type::array:
+  simdjson::padded_string docdata =  R"({"value":123})"_padded;
-      cout << "[";
+  simdjson::ondemand::document doc = parser.iterate(docdata);
-      add_comma = false;
+  simdjson::ondemand::object obj = doc.get_object();
-      for (auto child : element.get_array()) {
+  string_view token = obj.raw_json(); // gives you `{"value":123}`
-        if (add_comma) {
+  ASSERT_EQUAL(token, R"({"value":123})");
-          cout << ",";
+  obj.reset(); // revise the object
-        }
+  uint64_t x = obj["value"]; // gives me 123
-        // We need the call to value() to get
+  ASSERT_EQUAL(x, 123);
-        // an ondemand::value type.
+
-        recursive_print_json(child.value());
+  TEST_SUCCEED();
-        add_comma = true;
+}
-      }
+
-      cout << "]";
+bool at_end() {
-      break;
+  TEST_START();
-    case ondemand::json_type::object:
+  auto json = R"([1, 2] foo ])"_padded;
-      cout << "{";
+  ondemand::parser parser;
-      add_comma = false;
+  ondemand::document doc = parser.iterate(json);
-      for (auto field : element.get_object()) {
+  ondemand::array array = doc.get_array();
-        if (add_comma) {
+  for (uint64_t values : array) {
-          cout << ",";
+    std::cout << values << std::endl;
-        }
+  }
-        // key() returns the key as it appears in the raw
+  if(!doc.at_end()) {
-        // JSON document, if we want the unescaped key,
+    std::cerr << "trailing content at byte index " << doc.current_location() - json.data() << std::endl;
-        // we should do field.unescaped_key().
+  }
-        cout << "\"" << field.key() << "\": ";
+  TEST_SUCCEED();
-        recursive_print_json(field.value());
+}
-        add_comma = true;
+
-      }
+bool number_tests() {
-      cout << "}\n";
+  TEST_START();
-      break;
+  ondemand::parser parser;
-    case ondemand::json_type::number:
+  padded_string docdata = R"([1.0, 3, 1, 3.1415,-13231232,9999999999999999999])"_padded;
-      // assume it fits in a double
+  ondemand::document doc = parser.iterate(docdata);
-      cout << element.get_double();
+  ondemand::array arr = doc.get_array();
-      break;
+  for(ondemand::value val : arr) {
-    case ondemand::json_type::string:
+    std::cout << val << " ";
-      // get_string() would return escaped string, but
+    std::cout << "negative: " << val.is_negative() << " ";
-      // we are happy with unescaped string.
+    std::cout << "is_integer: " << val.is_integer() << " ";
-      cout << "\"" << element.get_raw_json_string() << "\"";
+    ondemand::number num = val.get_number();
-      break;
+    ondemand::number_type t = num.get_number_type();
-    case ondemand::json_type::boolean:
+    // direct computation without materializing the number:
-      cout << element.get_bool();
+    ondemand::number_type dt = val.get_number_type();
-      break;
+    if(t != dt) { throw std::runtime_error("bug"); }
-    case ondemand::json_type::null:
+    switch(t) {
-      // we check that the value is indeed null
+      case ondemand::number_type::signed_integer:
-      // otherwise: an error is thrown.
+        std::cout  << "integer: " << int64_t(num) << " ";
-      if(element.is_null()) {
+        std::cout  << "integer: " << num.get_int64() << std::endl;
-        cout << "null";
+        break;
-      }
+      case ondemand::number_type::unsigned_integer:
-      break;
+        std::cout  << "large 64-bit integer: " << uint64_t(num) << " ";
+        std::cout << "large 64-bit integer: " << num.get_uint64() << std::endl;
+        break;
+      case ondemand::number_type::floating_point_number:
+        std::cout  << "float: " << double(num) << " ";
+        std::cout << "float: " << num.get_double() << std::endl;
+        break;
     }
+  }
+  TEST_SUCCEED();
+}
+
+bool recursive_print_json(ondemand::value element) {
+  TEST_START();
+  bool add_comma;
+  switch (element.type()) {
+  case ondemand::json_type::array:
+    cout << "[";
+    add_comma = false;
+    for (auto child : element.get_array()) {
+      if (add_comma) {
+        cout << ",";
+      }
+      // We need the call to value() to get
+      // an ondemand::value type.
+      recursive_print_json(child.value());
+      add_comma = true;
+    }
+    cout << "]";
+    break;
+  case ondemand::json_type::object:
+    cout << "{";
+    add_comma = false;
+    for (auto field : element.get_object()) {
+      if (add_comma) {
+        cout << ",";
+      }
+      // key() returns the key as it appears in the raw
+      // JSON document, if we want the unescaped key,
+      // we should do field.unescaped_key().
+      cout << "\"" << field.key() << "\": ";
+      recursive_print_json(field.value());
+      add_comma = true;
+    }
+    cout << "}\n";
+    break;
+  case ondemand::json_type::number:
+    // assume it fits in a double
+    cout << element.get_double();
+    break;
+  case ondemand::json_type::string:
+    // get_string() would return escaped string, but
+    // we are happy with unescaped string.
+    cout << "\"" << element.get_raw_json_string() << "\"";
+    break;
+  case ondemand::json_type::boolean:
+    cout << element.get_bool();
+    break;
+  case ondemand::json_type::null:
+    // we check that the value is indeed null
+    // otherwise: an error is thrown.
+    if(element.is_null()) {
+      cout << "null";
+    }
+    break;
+  }
+  TEST_SUCCEED();
 }
 
 bool basics_treewalk() {
+  TEST_START();
   padded_string json[3] = {R"( [
     { "make": "Toyota", "model": "Camry",  "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
     { "make": "Kia",    "model": "Soul",   "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
@@ -135,77 +177,82 @@ bool basics_treewalk() {
     recursive_print_json(val);
     std::cout << std::endl;
   }
-  return true;
+  TEST_SUCCEED();
 }
 
-void print_depth_space(ondemand::value element) {
+bool print_depth_space(ondemand::value element) {
+  TEST_START();
   for(auto i = 0; i < element.current_depth(); i++) {
     cout << " ";
   }
+  TEST_SUCCEED();
 }
 
-void recursive_print_json_breakline(ondemand::value element) {
+bool recursive_print_json_breakline(ondemand::value element) {
-    bool add_comma;
+  TEST_START();
-    switch (element.type()) {
+  bool add_comma;
-    case ondemand::json_type::array:
+  switch (element.type()) {
-      cout << endl;
+  case ondemand::json_type::array:
-      print_depth_space(element);
+    cout << endl;
-      cout << "[";
+    print_depth_space(element);
-      add_comma = false;
+    cout << "[";
-      for (auto child : element.get_array()) {
+    add_comma = false;
-        if (add_comma) {
+    for (auto child : element.get_array()) {
-          print_depth_space(element);
+      if (add_comma) {
-          cout << ",";
+        print_depth_space(element);
-        }
+        cout << ",";
-        // We need the call to value() to get
-        // an ondemand::value type.
-        recursive_print_json_breakline(child.value());
-        add_comma = true;
       }
-      cout << "]";
+      // We need the call to value() to get
-      break;
+      // an ondemand::value type.
-    case ondemand::json_type::object:
+      recursive_print_json_breakline(child.value());
-      cout << endl;
+      add_comma = true;
-      print_depth_space(element);
-      cout << "{";
-      add_comma = false;
-      for (auto field : element.get_object()) {
-        if (add_comma) {
-          print_depth_space(element);
-          cout << ",";
-        }
-        // key() returns the key as it appears in the raw
-        // JSON document, if we want the unescaped key,
-        // we should do field.unescaped_key().
-        cout << "\"" << field.key() << "\": ";
-        recursive_print_json_breakline(field.value());
-        add_comma = true;
-      }
-      cout << "}\n";
-      break;
-    case ondemand::json_type::number:
-      // assume it fits in a double
-      cout << element.get_double();
-      break;
-    case ondemand::json_type::string:
-      // get_string() would return escaped string, but
-      // we are happy with unescaped string.
-      cout << "\"" << element.get_raw_json_string() << "\"";
-      break;
-    case ondemand::json_type::boolean:
-      cout << element.get_bool();
-      break;
-    case ondemand::json_type::null:
-      // We check that the value is indeed null
-      // otherwise: an error is thrown.
-      if(element.is_null()) {
-        cout << "null";
-      }
-      break;
     }
+    cout << "]";
+    break;
+  case ondemand::json_type::object:
+    cout << endl;
+    print_depth_space(element);
+    cout << "{";
+    add_comma = false;
+    for (auto field : element.get_object()) {
+      if (add_comma) {
+        print_depth_space(element);
+        cout << ",";
+      }
+      // key() returns the key as it appears in the raw
+      // JSON document, if we want the unescaped key,
+      // we should do field.unescaped_key().
+      cout << "\"" << field.key() << "\": ";
+      recursive_print_json_breakline(field.value());
+      add_comma = true;
+    }
+    cout << "}\n";
+    break;
+  case ondemand::json_type::number:
+    // assume it fits in a double
+    cout << element.get_double();
+    break;
+  case ondemand::json_type::string:
+    // get_string() would return escaped string, but
+    // we are happy with unescaped string.
+    cout << "\"" << element.get_raw_json_string() << "\"";
+    break;
+  case ondemand::json_type::boolean:
+    cout << element.get_bool();
+    break;
+  case ondemand::json_type::null:
+    // We check that the value is indeed null
+    // otherwise: an error is thrown.
+    if(element.is_null()) {
+      cout << "null";
+    }
+    break;
+  }
+  TEST_SUCCEED();
 }
 
 bool basics_treewalk_breakline() {
+  TEST_START();
   padded_string json[3] = {R"( [
     { "make": "Toyota", "model": "Camry",  "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
     { "make": "Kia",    "model": "Soul",   "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
@@ -218,7 +265,7 @@ bool basics_treewalk_breakline() {
     recursive_print_json_breakline(val);
     std::cout << std::endl;
   }
-  return true;
+  TEST_SUCCEED();
 }
 
 bool basics_1() {
@@ -721,7 +768,7 @@ bool json_pointer_multiple() {
 	ASSERT_SUCCESS(cars.count_elements().get(size));
 	double expected[] = {39.9, 31, 30};
 	for (size_t i = 0; i < size; i++) {
-		std::string json_pointer = "/" + std::to_string(i) + "/tire_pressure/1";
+		std::string json_pointer = std::string("/") + std::to_string(i) + std::string("/tire_pressure/1");
 		double x;
 		ASSERT_SUCCESS(cars.at_pointer(json_pointer).get(x));
 		ASSERT_EQUAL(x,expected[i]);
@@ -974,6 +1021,38 @@ int load_example_except() {
   std::cout << identifier << std::endl;
   return EXIT_SUCCESS;
 }
+int load_example_except_morecomplete(void) {
+  TEST_START();
+  simdjson::ondemand::parser parser;
+  simdjson::padded_string json_string;
+  simdjson::ondemand::document doc;
+  try {
+    json_string = padded_string::load("twitter.json");
+    doc = parser.iterate(json_string);
+    uint64_t identifier = doc["statuses"].at(0)["id"];
+    std::cout << identifier << std::endl;
+  } catch (simdjson::simdjson_error &error) {
+    std::cerr << "JSON error: " << error.what() << " near "
+              << doc.current_location() << " in " << json_string << std::endl;
+  }
+  return EXIT_SUCCESS;
+}
+
+bool allow_comma_separated_example() {
+  TEST_START();
+  auto json = R"( 1, 2, 3, 4, "a", "b", "c", {"hello": "world"} , [1, 2, 3])"_padded;
+  ondemand::parser parser;
+  ondemand::document_stream doc_stream;
+  // We pass '32' as the batch size, but it is a bogus parameter because, since
+  // we pass 'true' to the allow_comma parameter, the batch size will be set to at least
+  // the document size.
+  auto error = parser.iterate_many(json, 32, true).get(doc_stream);
+  if(error) { std::cerr << error << std::endl; return false; }
+  for (auto doc : doc_stream) {
+    std::cout << doc.type() << std::endl;
+  }
+  TEST_SUCCEED();
+}
 #endif
 bool test_load_example() {
   TEST_START();
@@ -1004,7 +1083,6 @@ bool current_location_tape_error() {
   TEST_SUCCEED();
 }
 
-
 bool current_location_user_error() {
   TEST_START();
   auto json = R"( [1,2,3] )"_padded;
@@ -1235,8 +1313,10 @@ bool example1958() {
 bool run() {
   return true
 #if SIMDJSON_EXCEPTIONS
+    && gen_raw1() && gen_raw2() && gen_raw3()
     && at_end()
     && example1956() && example1958()
+    && allow_comma_separated_example()
 //    && basics_1() // Fails because twitter.json isn't in current directory. Compile test only.
     &&  basics_treewalk()
     &&  basics_treewalk_breakline()

tests/ondemand/ondemand_tostring_tests.cpp

@@ -1,5 +1,4 @@
 #include <cinttypes>
-#include <ciso646>
 #include <cmath>
 #include <cstdio>
 #include <cstdlib>

tools/release.py

@@ -23,8 +23,9 @@ def toversionstring(major, minor, rev):
 
 def topaddedversionstring(major, minor, rev):
     return str(major)+str(minor).zfill(3)+str(rev).zfill(3)
-
+print("Calling git rev-parse --abbrev-ref HEAD")
 pipe = subprocess.Popen(["git", "rev-parse", "--abbrev-ref", "HEAD"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+print("the commandline is {}".format(pipe.args))
 branchresult = pipe.communicate()[0].decode().strip()
 
 if(branchresult != "master"):
@@ -34,8 +35,9 @@ ret = subprocess.call(["git", "remote", "update"])
 
 if(ret != 0):
     sys.exit(ret)
-
+print("Calling git log HEAD.. --oneline")
 pipe = subprocess.Popen(["git", "log", "HEAD..", "--oneline"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+print("the commandline is {}".format(pipe.args))
 uptodateresult = pipe.communicate()[0].decode().strip()
 
 if(len(uptodateresult) != 0):
@@ -80,15 +82,6 @@ else :
 
 atleastminor= (currentv[0] != newversion[0]) or (currentv[1] != newversion[1])
 
-if(atleastminor):
-    print(colored(0, 255, 0, "This is more than a revision."))
-    releasefile = maindir + os.sep + "RELEASES.md"
-    releasedata = open(releasefile).read()
-    pattern = re.compile("#\s+\d+\.\d+")
-    m = pattern.search(releasedata)
-    if(m == None):
-        print(colored(255, 0, 0, "You are preparing a new minor release and you have not yet updated RELEASES.md."))
-        sys.exit(-1)
 
 versionfilerel = os.sep + "include" + os.sep + "simdjson" + os.sep + "simdjson_version.h"
 versionfile = maindir + versionfilerel
@@ -160,12 +153,17 @@ for line in fileinput.input(doxyfile, inplace=1, backup='.bak'):
 print("modified "+doxyfile+", a backup was made")
 
 
+print("running amalgamate.py")
+cp = subprocess.run(["python3", maindir+ os.sep + "singleheader/amalgamate.py"], stdout=subprocess.DEVNULL)  # doesn't capture output
+print("the commandline is {}".format(cp.args))
 
-cp = subprocess.run(["python3", "amalgamate.py"], stdout=subprocess.DEVNULL, cwd=maindir+ os.sep + "singleheader")  # doesn't capture output
 if(cp.returncode != 0):
     print("Failed to run amalgamate")
 
+print("running doxygen")
 cp = subprocess.run(["doxygen"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, cwd=maindir)  # doesn't capture output
+print("the commandline is {}".format(cp.args))
+
 if(cp.returncode != 0):
     print("Failed to run doxygen")