New release

Add comma separated value parsing as an option in iterate_many (#2016 )
* Add comma separated value parsing * Fix failing tests * Make tests work for exceptions * Fix test * Fix try catch making test fail --------- Co-authored-by: Yong Xiang Ng <yxng@drwholdings.com>
2026-06-08 17:27:07 +00:00 · 2023-06-15 09:21:07 -04:00 · 2023-06-15 09:11:14 -04:00 · 2023-06-15 09:10:13 -04:00 · 2023-06-07 09:57:28 -04:00 · 2023-06-05 09:04:36 -04:00
45 changed files with 1058 additions and 369 deletions
@@ -0,0 +1,33 @@
+name: Ubuntu 22.04 CI (GCC 12, CXX 20)
+
+on: [push, pull_request]
+
+jobs:
+  ubuntu-build:
+    if: >-
+      ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
+      ! contains(toJSON(github.event.commits.*.message), '[skip github]')
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/cache@v2
+        with:
+          path: dependencies/.cache
+          key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
+      - name: Use cmake
+        run: |
+          mkdir builddebug &&
+          cd builddebug &&
+          CXX=g++-12 cmake -DSIMDJSON_CXX_STANDARD=20 -DCMAKE_BUILD_TYPE=Debug -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF ..  &&
+          cmake --build .   &&
+          ctest -j --output-on-failure -LE explicitonly   &&
+          cd .. &&
+          mkdir build &&
+          cd build &&
+          CXX=g++-12 cmake -DSIMDJSON_CXX_STANDARD=20  -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination ..  &&
+          cmake --build .   &&
+          ctest -j --output-on-failure -LE explicitonly   &&
+          cmake --install . &&
+          echo -e '#include <simdjson.h>\nint main(int argc,char**argv) {simdjson::dom::parser parser;simdjson::dom::element tweets = parser.load(argv[1]); }' > tmp.cpp && c++ -Idestination/include -Ldestination/lib -std=c++17 -Wl,-rpath,destination/lib -o linkandrun tmp.cpp -lsimdjson && ./linkandrun jsonexamples/twitter.json &&
+          cd ../tests/installation_tests/find &&
+          mkdir build && cd build && cmake -DCMAKE_INSTALL_PREFIX:PATH=../../../build/destination .. &&  cmake --build .
@@ -0,0 +1,44 @@
+name: VS17-CI CXX20
+
+on: [push, pull_request]
+
+jobs:
+  ci:
+    if: >-
+      ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
+      ! contains(toJSON(github.event.commits.*.message), '[skip github]')
+    name: windows-vs17
+    runs-on: windows-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - {gen: Visual Studio 17 2022, arch: Win32, shared: ON}
+          - {gen: Visual Studio 17 2022, arch: Win32, shared: OFF}
+          - {gen: Visual Studio 17 2022, arch: x64, shared: ON}
+          - {gen: Visual Studio 17 2022, arch: x64, shared: OFF}
+    steps:
+    - name: checkout
+      uses: actions/checkout@v2
+    - name: Configure
+      run: |
+        cmake -DSIMDJSON_CXX_STANDARD=20 -G "${{matrix.gen}}" -A ${{matrix.arch}} -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -DBUILD_SHARED_LIBS=${{matrix.shared}} -B build
+    - name: Build Debug
+      run: cmake --build build --config Debug --verbose
+    - name: Build Release
+      run: cmake --build build --config Release --verbose
+    - name: Run Release tests
+      run: |
+        cd build
+        ctest -C Release  -LE explicitonly  --output-on-failure
+    - name: Run Debug tests
+      run: |
+        cd build
+        ctest -C Debug  -LE explicitonly  --output-on-failure
+    - name: Install
+      run: |
+        cmake --install build --config Release
+    - name: Test Installation
+      run: |
+        cmake -G "${{matrix.gen}}" -A ${{matrix.arch}}  -B build_install_test tests/installation_tests/find
+        cmake --build build_install_test --config Release
@@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.14)
 project(
    simdjson
    # The version number is modified by tools/release.py
-    VERSION 3.1.8
+    VERSION 3.2.0
    DESCRIPTION "Parsing gigabytes of JSON per second"
    HOMEPAGE_URL "https://simdjson.org/"
    LANGUAGES CXX C
@@ -20,8 +20,8 @@ string(
 # ---- Options, variables ----

 # These version numbers are modified by tools/release.py
-set(SIMDJSON_LIB_VERSION "15.0.0" CACHE STRING "simdjson library version")
-set(SIMDJSON_LIB_SOVERSION "15" CACHE STRING "simdjson library soversion")
+set(SIMDJSON_LIB_VERSION "16.0.0" CACHE STRING "simdjson library version")
+set(SIMDJSON_LIB_SOVERSION "16" CACHE STRING "simdjson library soversion")

 option(SIMDJSON_ENABLE_THREADS "Link with thread support" ON)

@@ -38,7 +38,7 @@ PROJECT_NAME           = simdjson
 # could be handy for archiving the generated documentation or if some version
 # control system is used.

-PROJECT_NUMBER         = "3.1.8"
+PROJECT_NUMBER         = "3.2.0"

 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a
@@ -28,16 +28,38 @@ This library is part of the [Awesome Modern C++](https://awesomecpp.com) list.
 Table of Contents
 -----------------

+* [Real-world usage](#real-world-usage)
 * [Quick Start](#quick-start)
 * [Documentation](#documentation)
 * [Performance results](#performance-results)
-* [Real-world usage](#real-world-usage)
 * [Bindings and Ports of simdjson](#bindings-and-ports-of-simdjson)
 * [About simdjson](#about-simdjson)
 * [Funding](#funding)
 * [Contributing to simdjson](#contributing-to-simdjson)
 * [License](#license)

+
+Real-world usage
+----------------
+
+- [ClickHouse](https://github.com/ClickHouse/ClickHouse)
+- [Facebook/Meta Velox](https://velox-lib.io)
+- [milvus](https://github.com/milvus-io/milvus)
+- [Clang Build Analyzer](https://github.com/aras-p/ClangBuildAnalyzer)
+- [Shopify HeapProfiler](https://github.com/Shopify/heap-profiler)
+- [StarRocks](https://github.com/StarRocks/starrocks)
+- [Microsoft FishStore](https://github.com/microsoft/FishStore)
+- [Intel PCM](https://github.com/intel/pcm)
+- [WatermelonDB](https://github.com/Nozbe/WatermelonDB)
+- [Apache Doris](https://github.com/apache/doris)
+- [Dgraph](https://github.com/dgraph-io/dgraph)
+- [UJRPC](https://github.com/unum-cloud/ujrpc)
+- [fastgltf](https://github.com/spnda/fastgltf)
+- [vast](https://github.com/tenzir/vast)
+- [ada-url](https://github.com/ada-url/ada)
+
+If you are planning to use simdjson in a product, please work from one of our releases.
+
 Quick Start
 -----------

@@ -64,13 +86,13 @@ int main(void) {
    ondemand::document tweets = parser.iterate(json);
    std::cout << uint64_t(tweets["search_metadata"]["count"]) << " results." << std::endl;
 }
-
-   ```
+```
 3. `c++ -o quickstart quickstart.cpp simdjson.cpp`
 4. `./quickstart`
-   ```
+
+  ```
   100 results.
-   ```
+  ```

 Documentation
 -------------
@@ -81,7 +103,13 @@ Usage documentation is available:
 * [Performance](doc/performance.md) shows some more advanced scenarios and how to tune for them.
 * [Implementation Selection](doc/implementation-selection.md) describes runtime CPU detection and
  how you can work with it.
-* [API](https://simdjson.org/api/1.0.0/annotated.html) contains the automatically generated API documentation.
+* [API](https://simdjson.github.io/simdjson/) contains the automatically generated API documentation.
+
+
+Some users may want to browse code along with the compiled assembly. You want to check out the following lists of examples:
+
+* [simdjson examples with errors handled through exceptions](https://godbolt.org/z/98Kx9Kqjn)
+* [simdjson examples with errors without exceptions](https://godbolt.org/z/PKG7GdbPo)

 Performance results
 -------------------
@@ -110,24 +138,6 @@ For NDJSON files, we can exceed 3 GB/s with [our  multithreaded parsing function



-Real-world usage
----------------
-
- [ClickHouse](https://github.com/ClickHouse/ClickHouse)
- [Clang Build Analyzer](https://github.com/aras-p/ClangBuildAnalyzer)
- [Shopify HeapProfiler](https://github.com/Shopify/heap-profiler)
- [StarRocks](https://github.com/StarRocks/starrocks)
- [Microsoft FishStore](https://github.com/microsoft/FishStore)
- [Intel PCM](https://github.com/intel/pcm)
- [WatermelonDB](https://github.com/Nozbe/WatermelonDB)
- [Apache Doris](https://github.com/apache/doris)
- [Dgraph](https://github.com/dgraph-io/dgraph)
- [UJRPC](https://github.com/unum-cloud/ujrpc)
- [fastgltf](https://github.com/spnda/fastgltf)
- [Clang Build Analyzer](https://github.com/aras-p/ClangBuildAnalyzer)
- [vast](https://github.com/tenzir/vast)
-
-If you are planning to use simdjson in a product, please work from one of our releases.

 Bindings and Ports of simdjson
 ------------------------------
@@ -521,7 +521,7 @@ static void twitter_image_sizes(State& state) {
    set<tuple<uint64_t, uint64_t>> image_sizes;
    for (dom::object tweet : doc["statuses"]) {
      dom::array media;
-      if (not (error = tweet["entities"]["media"].get(media))) {
+      if (! (error = tweet["entities"]["media"].get(media))) {
        for (dom::object image : media) {
          for (auto size : image["sizes"].get_object()) {
             image_sizes.emplace(size.value["w"], size.value["h"]);
@@ -638,7 +638,7 @@ static void error_code_twitter_image_sizes(State& state) noexcept {
    if ((error = doc["statuses"].get(statuses))) { return; }
    for (dom::element tweet : statuses) {
      dom::array images;
-      if (not (error = tweet["entities"]["media"].get(images))) {
+      if (! (error = tweet["entities"]["media"].get(images))) {
        for (dom::element image : images) {
          dom::object sizes;
          if ((error = image["sizes"].get(sizes))) { return; }
@@ -520,7 +520,7 @@ struct benchmarker {
        double freqall = (all_stages_without_allocation.best.cycles() / all_stages_without_allocation.best.elapsed_sec()) / 1000000000.0;
        double freqmin = min(freq1, freq2);
        double freqmax = max(freq1, freq2);
-        if((freqall < 0.95 * freqmin) or (freqall > 1.05 * freqmax)) {
+        if((freqall < 0.95 * freqmin) || (freqall > 1.05 * freqmax)) {
          printf("\nWarning: The processor frequency fluctuates in an expected way!!!\n"
          "Range for stage 1 and stage 2 : [%.3f GHz, %.3f GHz], overall: %.3f GHz.\n",
          freqmin, freqmax, freqall);
@@ -110,7 +110,8 @@ endif()

 # We compile tools, tests, etc. with C++ 17. Override yourself if you need on a
 # target.
-set(CMAKE_CXX_STANDARD 17)
+set(SIMDJSON_CXX_STANDARD 17 CACHE STRING "the C++ standard to use for simdjson")
+set(CMAKE_CXX_STANDARD ${SIMDJSON_CXX_STANDARD})
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)
 set(CMAKE_MACOSX_RPATH OFF)
@@ -25,11 +25,11 @@ An overview of what you need to know to use simdjson, with examples.
    - [Current location in document](#current-location-in-document)
    - [Checking for trailing content](#checking-for-trailing-content)
  - [Rewinding](#rewinding)
-  - [Direct Access to the Raw String](#direct-access-to-the-raw-string)
  - [Newline-Delimited JSON (ndjson) and JSON lines](#newline-delimited-json-ndjson-and-json-lines)
  - [Parsing Numbers Inside Strings](#parsing-numbers-inside-strings)
  - [Dynamic Number Types](#dynamic-number-types)
  - [Raw Strings](#raw-strings)
+  - [General Direct Access to the Raw JSON String](#general-direct-access-to-the-raw-json-string)
  - [Thread Safety](#thread-safety)
  - [Standard Compliance](#standard-compliance)
  - [Backwards Compatibility](#backwards-compatibility)
@@ -202,6 +202,13 @@ Consider reusing the same buffers and limiting memory allocations.

 By default, the simdjson library throws exceptions (`simdjson_error`) on errors. We omit `try`-`catch` clauses from our illustrating examples: if you omit `try`-`catch` in your code, an uncaught exception will halt your program. It is also possible to use simdjson without generating exceptions, and you may even build the library without exception support at all. See [Error Handling](#error-handling) for details.

+
+Some users may want to browse code along with the compiled assembly. You want to check out the following lists of examples:
+
+* [simdjson examples with errors handled through exceptions](https://godbolt.org/z/98Kx9Kqjn)
+* [simdjson examples with errors without exceptions](https://godbolt.org/z/PKG7GdbPo)
+
+
 Documents are Iterators
 -----------------------

@@ -323,7 +330,7 @@ support for users who avoid exceptions. See [the simdjson error handling documen
 * **Validate What You Use:** When calling `iterate`, the document is quickly indexed. If it is
  not a valid Unicode (UTF-8) string or if there is an unclosed string, an error may be reported right away.
  However, it is not fully validated. On Demand only fully validates the values you use and the
-  structure leading to it.
+  structure leading to it. It means that at every step as you traverse the document, you may encounter an error. You can handle errors either with exceptions or with error codes.
 * **Extracting Values:** You can cast a JSON element to a native type:
  `double(element)`. This works for `std::string_view`, double, uint64_t, int64_t, bool,
  ondemand::object and ondemand::array. We also have explicit methods such as `get_string()`, `get_double()`,
@@ -348,8 +355,8 @@ support for users who avoid exceptions. See [the simdjson error handling documen
 * **Field Access:** To get the value of the "foo" field in an object, use `object["foo"]`. This will
  scan through the object looking for the field with the matching string, doing a character-by-character
  comparison. For efficiency reason, you should avoid looking up the same field repeatedly: e.g., do
-  not do `object["foo"]` followed by `object["foo"]` with the same `object` instance. If you consume an
-  object twice: `std::string_view(object["foo"]` followed by `std::string_view(object["foo"]`, your code
+  not do `object["foo"]` followed by `object["foo"]` with the same `object` instance. Keep in mind that On Demand does not buffer or save the result of the parsing: if you repeatedly access `object["foo"]`, then it must repeatedly seek the key and parse the content. The library does not provide a distinct function to check if a key is present, instead we recommend you attempt to access the key: e.g., by doing `ondemand::value val{}; if(!object["foo"].get(val)) {...}`, you have that `val` contains the requested value inside the if clause.  It is your responsability as a user to temporarily keep a reference to the value (`auto v = object["foo"]`), or to consume the content and store it in your own data structures. If you consume an
+  object twice: `std::string_view(object["foo"]` followed by `std::string_view(object["foo"]` then your code
  is in error. Furthermore, you can only consume one field at a time, on the same object. The
  value instance you get from  `content["bids"]` becomes invalid when you call `content["asks"]`.
  If you have retrieved `content["bids"].get_array()` and you later call
@@ -364,8 +371,9 @@ support for users who avoid exceptions. See [the simdjson error handling documen
  > to support escaped keys, the method `unescaped_key()` provides the desired unescaped keys by
  > parsing and writing out the unescaped keys to a string buffer and returning a `std::string_view`
  > instance. You should expect a performance penalty when using `unescaped_key()`.
+  >
  > ```c++
-  > auto json = R"({"k\u0065y": 1})"_padded;
+  > auto json = R"({"k\u0065y": 1})"_padded;
  > ondemand::parser parser;
  > auto doc = parser.iterate(json);
  > ondemand::object object = doc.get_object();
@@ -1142,6 +1150,28 @@ int main(void) {
 }
 ```

+
+You can do handle errors gracefully as well...
+
+```C++
+#include <iostream>
+#include "simdjson.h"
+int main(void) {
+  simdjson::ondemand::parser parser;
+  simdjson::padded_string json_string;
+  simdjson::ondemand::document doc;
+  try {
+    json_string = padded_string::load("twitter.json");
+    doc = parser.iterate(json_string);
+    uint64_t identifier = doc["statuses"].at(0)["id"];
+    std::cout << identifier << std::endl;
+  } catch (simdjson::simdjson_error &error) {
+    std::cerr << "JSON error: " << error.what() << " near "
+              << doc.current_location() << " in " << json_string << std::endl;
+  }
+}
+```
+
 ### Current location in document

 Sometimes, it might be helpful to know the current location in the document during iteration. This is especially useful when encountering errors. The `current_location()` method on a
@@ -1304,41 +1334,6 @@ internal string buffer. Thus you should consume values only once
 even if you can iterate through the array or object more than once.
 If you unescape a string within an array more than once, you have unsafe code.

-Direct Access to the Raw String
--------------------------------
-
-The simdjson library makes explicit assumptions about types. For examples, numbers
-must be integers (up to 64-bit integers) or binary64 floating-point numbers. Some users
-have different needs. For example, some users might want to support big integers.
-The library makes this possible by providing a `raw_json_token` method which returns
-a `std::string_view` instance containing the value as a string which you may then
-parse as you see fit.
-
-```C++
-simdjson::ondemand::parser parser;
-simdjson::padded_string docdata =  R"({"value":12321323213213213213213213213211223})"_padded;
-simdjson::ondemand::document doc = parser.iterate(docdata);
-simdjson::ondemand::object obj = doc.get_object();
-std::string_view token = obj["value"].raw_json_token();
-// token has value 12321323213213213213213213213211223, it points inside the input string
-```
-
-The `raw_json_token` method even works when the JSON value is a string. In such cases, it
-will return the complete string with the quotes and with eventual escaped sequences as in the
-source document.
-
-```C++
-simdjson::ondemand::parser parser;
-simdjson::padded_string docdata =  R"({"value":"12321323213213213213213213213211223"})"_padded;
-simdjson::ondemand::document doc = parser.iterate(docdata);
-simdjson::ondemand::object obj = doc.get_object();
-string_view token = obj["value"].raw_json_token();
-// token has value "12321323213213213213213213213211223", it points inside the input string
-```
-
-The `raw_json_token()` should be fast and free of allocation.
-
-

 Newline-Delimited JSON (ndjson) and JSON lines
 ----------------------------------------------
@@ -1643,6 +1638,78 @@ JSON string to a user-provided buffer:
    }
 ```

+
+General Direct Access to the Raw JSON String
+--------------------------------
+If your value is a string, the `raw_json_string` gives you direct access to the unprocess
+string. The simdjson library allows you to have access to the raw underlying JSON
+more generally.
+
+The simdjson library makes explicit assumptions about types. For examples, numbers
+must be integers (up to 64-bit integers) or binary64 floating-point numbers. Some users
+have different needs. For example, some users might want to support big integers.
+The library makes this possible by providing a `raw_json_token` method which returns
+a `std::string_view` instance containing the value as a string which you may then
+parse as you see fit.
+
+```C++
+simdjson::ondemand::parser parser;
+simdjson::padded_string docdata =  R"({"value":12321323213213213213213213213211223})"_padded;
+simdjson::ondemand::document doc = parser.iterate(docdata);
+simdjson::ondemand::object obj = doc.get_object();
+std::string_view token = obj["value"].raw_json_token();
+// token has value 12321323213213213213213213213211223, it points inside the input string
+```
+
+The `raw_json_token` method even works when the JSON value is a string. In such cases, it
+will return the complete string with the quotes and with eventual escaped sequences as in the
+source document.
+
+```C++
+simdjson::ondemand::parser parser;
+simdjson::padded_string docdata =  R"({"value":"12321323213213213213213213213211223"})"_padded;
+simdjson::ondemand::document doc = parser.iterate(docdata);
+simdjson::ondemand::object obj = doc.get_object();
+string_view token = obj["value"].raw_json_token();
+// token has value "12321323213213213213213213213211223", it points inside the input string
+```
+
+The `raw_json_token()` should be fast and free of allocation.
+
+If you value is an array or an object, `raw_json_token()` returns effectively a single
+character (`[`) or (`}`) which is not very useful. For arrays and objects, we have another
+method called `raw_json()` which consumes (traverse) the array or the object.
+
+```C++
+simdjson::ondemand::parser parser;
+simdjson::padded_string docdata =  R"({"value":123})"_padded;
+simdjson::ondemand::document doc = parser.iterate(docdata);
+simdjson::ondemand::object obj = doc.get_object();
+string_view token = obj.raw_json(); // gives you `{"value":123}`
+```
+
+
+```C++
+simdjson::ondemand::parser parser;
+simdjson::padded_string docdata =  R"([1,2,3])"_padded;
+simdjson::ondemand::document doc = parser.iterate(docdata);
+simdjson::ondemand::array arr = doc.get_array();
+string_view token = arr.raw_json(); // gives you `[1,2,3]`
+```
+
+Because `raw_json()` consumes to object or the array, if you want to both have
+access to the raw string, and also use the array or object, you should call `reset()`.
+
+```C++
+simdjson::ondemand::parser parser;
+simdjson::padded_string docdata =  R"({"value":123})"_padded;
+simdjson::ondemand::document doc = parser.iterate(docdata);
+simdjson::ondemand::object obj = doc.get_object();
+string_view token = obj.raw_json(); // gives you `{"value":123}`
+obj.reset(); // revise the object
+uint64_t x = obj["value"]; // gives me 123
+```
+
 Thread Safety
 -------------

@@ -1894,4 +1961,4 @@ Performance Tips
 	std::string_view year = data["year"];
 	std::string_view rating = data["rating"];
  ```
- To better understand the operation of your On Demand parser, and whether it is performing as well as you think it should be, there is a  logger feature built in to simdjson! To use it, define the pre-processor directive `SIMDJSON_VERBOSE_LOGGING` prior to including the `simdjson.h` header, which enables logging in simdjson. Run your code. It may generate a lot of logging output; adding printouts from your application that show each section may be helpful. The log’s output will show step-by-step information on state, buffer pointer position, depth, and key retrieval status.
+- To better understand the operation of your On Demand parser, and whether it is performing as well as you think it should be, there is a  logger feature built in to simdjson! To use it, define the pre-processor directive `SIMDJSON_VERBOSE_LOGGING` prior to including the `simdjson.h` header, which enables logging in simdjson. Run your code. It may generate a lot of logging output; adding printouts from your application that show each section may be helpful. The log’s output will show step-by-step information on state, buffer pointer position, depth, and key retrieval status. The log level is set to `INFO` by default, but can be modified with the environment variable `SIMDJSON_LOG_LEVEL`. Setting the environment variable `SIMDJSON_LOG_LEVEL` to `ERROR` will log only errors.
@@ -237,3 +237,8 @@ This will print:
 ```

 Importantly, you should only call `truncated_bytes()` after iterating through all of the documents since the stream cannot tell whether there are truncated documents at the very end when it may not have accessed that part of the data yet.
+
+Comma separated documents
+-----------
+
+`iterate_many` also takes in an option to allow parsing of comma separated documents. In this mode, the entire buffer is processed in 1 batch and batch size will be increased to be as large as the JSON passed. Therefore, the capacity of the parser has to be sufficient to support the batch size set.
@@ -205,4 +205,8 @@ simdjson_inline  simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdj
  if (error()) { return error(); }
  return first.at_pointer(json_pointer);
 }
+simdjson_inline  simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array>::raw_json() noexcept {
+  if (error()) { return error(); }
+  return first.raw_json();
+}
 } // namespace simdjson
@@ -186,6 +186,8 @@ public:
  inline simdjson_result<bool> reset() & noexcept;
  simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> at(size_t index) noexcept;
  simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> at_pointer(std::string_view json_pointer) noexcept;
+  simdjson_inline simdjson_result<std::string_view> raw_json() noexcept;
+
 };

 } // namespace simdjson
@@ -84,12 +84,14 @@ simdjson_inline document_stream::document_stream(
  ondemand::parser &_parser,
  const uint8_t *_buf,
  size_t _len,
-  size_t _batch_size
+  size_t _batch_size,
+  bool _allow_comma_separated
 ) noexcept
  : parser{&_parser},
    buf{_buf},
    len{_len},
    batch_size{_batch_size <= MINIMAL_BATCH_SIZE ? MINIMAL_BATCH_SIZE : _batch_size},
+    allow_comma_separated{_allow_comma_separated},
    error{SUCCESS}
    #ifdef SIMDJSON_THREADS_ENABLED
    , use_thread(_parser.threaded) // we need to make a copy because _parser.threaded can change
@@ -107,6 +109,7 @@ simdjson_inline document_stream::document_stream() noexcept
    buf{nullptr},
    len{0},
    batch_size{0},
+    allow_comma_separated{false},
    error{UNINITIALIZED}
    #ifdef SIMDJSON_THREADS_ENABLED
    , use_thread(false)
@@ -290,6 +293,8 @@ inline void document_stream::next_document() noexcept {
  if (error) { return; }
  // Always set depth=1 at the start of document
  doc.iter._depth = 1;
+  // consume comma if comma separated is allowed
+  if (allow_comma_separated) { doc.iter.consume_character(','); }
  // Resets the string buffer at the beginning, thus invalidating the strings.
  doc.iter._string_buf_loc = parser->string_buf.get();
  doc.iter._root = doc.iter.position();
@@ -222,7 +222,8 @@ private:
    ondemand::parser &parser,
    const uint8_t *buf,
    size_t len,
-    size_t batch_size
+    size_t batch_size,
+    bool allow_comma_separated
  ) noexcept;

  /**
@@ -271,6 +272,7 @@ private:
  const uint8_t *buf;
  size_t len;
  size_t batch_size;
+  bool allow_comma_separated;
  /**
   * We are going to use just one document instance. The document owns
   * the json_iterator. It implies that we only ever pass a reference
@@ -337,6 +337,14 @@ simdjson_inline void json_iterator::reenter_child(token_position position, depth
  _depth = child_depth;
 }

+simdjson_inline error_code json_iterator::consume_character(char c) noexcept {
+  if (*peek() == c) {
+    return_current_and_advance();
+    return SUCCESS;
+  }
+  return TAPE_ERROR;
+}
+
 #if SIMDJSON_DEVELOPMENT_CHECKS

 simdjson_inline token_position json_iterator::start_position(depth_t depth) const noexcept {
@@ -255,6 +255,7 @@ public:
  simdjson_inline simdjson_result<std::string_view> unescape_wobbly(raw_json_string in) noexcept;
  simdjson_inline void reenter_child(token_position position, depth_t child_depth) noexcept;

+  simdjson_inline error_code consume_character(char c) noexcept;
 #if SIMDJSON_DEVELOPMENT_CHECKS
  simdjson_inline token_position start_position(depth_t depth) const noexcept;
  simdjson_inline void set_start_position(depth_t depth, token_position position) noexcept;
@@ -297,8 +298,8 @@ protected:
  friend class raw_json_string;
  friend class parser;
  friend class value_iterator;
-  friend simdjson_inline void logger::log_line(const json_iterator &iter, const char *title_prefix, const char *title, std::string_view detail, int delta, int depth_delta) noexcept;
-  friend simdjson_inline void logger::log_line(const json_iterator &iter, token_position index, depth_t depth, const char *title_prefix, const char *title, std::string_view detail) noexcept;
+  friend simdjson_inline void logger::log_line(const json_iterator &iter, const char *title_prefix, const char *title, std::string_view detail, int delta, int depth_delta, logger::log_level level) noexcept;
+  friend simdjson_inline void logger::log_line(const json_iterator &iter, token_position index, depth_t depth, const char *title_prefix, const char *title, std::string_view detail, logger::log_level level) noexcept;
 }; // json_iterator

 } // namespace ondemand
@@ -1,3 +1,4 @@
+#include <memory>
 namespace simdjson {
 namespace SIMDJSON_IMPLEMENTATION {
 namespace ondemand {
@@ -18,36 +19,68 @@ static inline char printable_char(char c) {
  }
 }

+static inline log_level get_log_level_from_env()
+{
+  SIMDJSON_PUSH_DISABLE_WARNINGS
+  SIMDJSON_DISABLE_DEPRECATED_WARNING // Disable CRT_SECURE warning on MSVC: manually verified this is safe
+  char *lvl = getenv("SIMDJSON_LOG_LEVEL");
+  SIMDJSON_POP_DISABLE_WARNINGS
+  if (lvl && simdjson_strcasecmp(lvl, "ERROR") == 0) { return log_level::LOG_ERROR; }
+  return log_level::LOG_INFO;
+}
+
+static inline log_level log_threshold()
+{
+  static log_level threshold = get_log_level_from_env();
+  return threshold;
+}
+
+static inline bool should_log(log_level level)
+{
+  return level >= log_threshold();
+}
+
+template<typename... Args>
+inline std::string string_format(const std::string& format, const Args&... args)
+{
+  int size_s = std::snprintf(nullptr, 0, format.c_str(), args...) + 1;
+  auto size = static_cast<size_t>(size_s);
+  if (size <= 0) return std::string();
+  std::unique_ptr<char[]> buf(new char[size]);
+  std::snprintf(buf.get(), size, format.c_str(), args...);
+  return std::string(buf.get(), buf.get() + size - 1);
+}
+
 inline void log_event(const json_iterator &iter, const char *type, std::string_view detail, int delta, int depth_delta) noexcept {
-  log_line(iter, "", type, detail, delta, depth_delta);
+  log_line(iter, "", type, detail, delta, depth_delta, log_level::LOG_INFO);
 }

 inline void log_value(const json_iterator &iter, token_position index, depth_t depth, const char *type, std::string_view detail) noexcept {
-  log_line(iter, index, depth, "", type, detail);
+  log_line(iter, index, depth, "", type, detail, log_level::LOG_INFO);
 }
 inline void log_value(const json_iterator &iter, const char *type, std::string_view detail, int delta, int depth_delta) noexcept {
-  log_line(iter, "", type, detail, delta, depth_delta);
+  log_line(iter, "", type, detail, delta, depth_delta, log_level::LOG_INFO);
 }

 inline void log_start_value(const json_iterator &iter, token_position index, depth_t depth, const char *type, std::string_view detail) noexcept {
-  log_line(iter, index, depth, "+", type, detail);
+  log_line(iter, index, depth, "+", type, detail, log_level::LOG_INFO);
  if (LOG_ENABLED) { log_depth++; }
 }
 inline void log_start_value(const json_iterator &iter, const char *type, int delta, int depth_delta) noexcept {
-  log_line(iter, "+", type, "", delta, depth_delta);
+  log_line(iter, "+", type, "", delta, depth_delta, log_level::LOG_INFO);
  if (LOG_ENABLED) { log_depth++; }
 }

 inline void log_end_value(const json_iterator &iter, const char *type, int delta, int depth_delta) noexcept {
  if (LOG_ENABLED) { log_depth--; }
-  log_line(iter, "-", type, "", delta, depth_delta);
+  log_line(iter, "-", type, "", delta, depth_delta, log_level::LOG_INFO);
 }

 inline void log_error(const json_iterator &iter, const char *error, const char *detail, int delta, int depth_delta) noexcept {
-  log_line(iter, "ERROR: ", error, detail, delta, depth_delta);
+  log_line(iter, "ERROR: ", error, detail, delta, depth_delta, log_level::LOG_ERROR);
 }
 inline void log_error(const json_iterator &iter, token_position index, depth_t depth, const char *error, const char *detail) noexcept {
-  log_line(iter, index, depth, "ERROR: ", error, detail);
+  log_line(iter, index, depth, "ERROR: ", error, detail, log_level::LOG_ERROR);
 }

 inline void log_event(const value_iterator &iter, const char *type, std::string_view detail, int delta, int depth_delta) noexcept {
@@ -71,7 +104,7 @@ inline void log_error(const value_iterator &iter, const char *error, const char
 }

 inline void log_headers() noexcept {
-  if (LOG_ENABLED) {
+  if (LOG_ENABLED && simdjson_unlikely(should_log(log_level::LOG_INFO))) {
    // Technically a static variable is not thread-safe, but if you are using threads
    // and logging... well...
    static bool displayed_hint{false};
@@ -121,11 +154,11 @@ inline void log_headers() noexcept {
  }
 }

-inline void log_line(const json_iterator &iter, const char *title_prefix, const char *title, std::string_view detail, int delta, int depth_delta) noexcept {
-  log_line(iter, iter.position()+delta, depth_t(iter.depth()+depth_delta), title_prefix, title, detail);
+inline void log_line(const json_iterator &iter, const char *title_prefix, const char *title, std::string_view detail, int delta, int depth_delta, log_level level) noexcept {
+  log_line(iter, iter.position()+delta, depth_t(iter.depth()+depth_delta), title_prefix, title, detail, level);
 }
-inline void log_line(const json_iterator &iter, token_position index, depth_t depth, const char *title_prefix, const char *title, std::string_view detail) noexcept {
-  if (LOG_ENABLED) {
+inline void log_line(const json_iterator &iter, token_position index, depth_t depth, const char *title_prefix, const char *title, std::string_view detail, log_level level) noexcept {
+  if (LOG_ENABLED && simdjson_unlikely(should_log(level))) {
    const int indent = depth*2;
    const auto buf = iter.token.buf;
    printf("| %*s%s%-*s ",
@@ -7,6 +7,11 @@ class value_iterator;

 namespace logger {

+enum class log_level : int32_t {
+  LOG_INFO = 0,
+  LOG_ERROR = 1
+};
+
 #if SIMDJSON_VERBOSE_LOGGING
  static constexpr const bool LOG_ENABLED = true;
 #else
@@ -16,9 +21,11 @@ namespace logger {
 // We do not want these functions to be 'really inlined' since real inlining is
 // for performance purposes and if you are using the loggers, you do not care about
 // performance (or should not).
+template<typename... Args>
+static inline std::string string_format(const std::string& format, const Args&... args);
 static inline void log_headers() noexcept;
-static inline void log_line(const json_iterator &iter, token_position index, depth_t depth, const char *title_prefix, const char *title, std::string_view detail) noexcept;
-static inline void log_line(const json_iterator &iter, const char *title_prefix, const char *title, std::string_view detail, int delta, int depth_delta) noexcept;
+static inline void log_line(const json_iterator &iter, token_position index, depth_t depth, const char *title_prefix, const char *title, std::string_view detail, log_level level) noexcept;
+static inline void log_line(const json_iterator &iter, const char *title_prefix, const char *title, std::string_view detail, int delta, int depth_delta, log_level level) noexcept;
 static inline void log_event(const json_iterator &iter, const char *type, std::string_view detail="", int delta=0, int depth_delta=0) noexcept;
 static inline void log_value(const json_iterator &iter, token_position index, depth_t depth, const char *type, std::string_view detail="") noexcept;
 static inline void log_value(const json_iterator &iter, const char *type, std::string_view detail="", int delta=-1, int depth_delta=0) noexcept;
@@ -5,13 +5,23 @@ namespace ondemand {
 simdjson_inline simdjson_result<value> object::find_field_unordered(const std::string_view key) & noexcept {
  bool has_value;
  SIMDJSON_TRY( iter.find_field_unordered_raw(key).get(has_value) );
-  if (!has_value) { return NO_SUCH_FIELD; }
+  if (!has_value) {
+    auto key_str = std::string(key.data(), key.size());
+    auto msg = logger::string_format("Cannot find key: %s", key_str.c_str());
+    logger::log_error(iter, msg.c_str());
+    return NO_SUCH_FIELD;
+  }
  return value(iter.child());
 }
 simdjson_inline simdjson_result<value> object::find_field_unordered(const std::string_view key) && noexcept {
  bool has_value;
  SIMDJSON_TRY( iter.find_field_unordered_raw(key).get(has_value) );
-  if (!has_value) { return NO_SUCH_FIELD; }
+  if (!has_value) {
+    auto key_str = std::string(key.data(), key.size());
+    auto msg = logger::string_format("Cannot find key: %s", key_str.c_str());
+    logger::log_error(iter, msg.c_str());
+    return NO_SUCH_FIELD;
+  }
  return value(iter.child());
 }
 simdjson_inline simdjson_result<value> object::operator[](const std::string_view key) & noexcept {
@@ -23,13 +33,23 @@ simdjson_inline simdjson_result<value> object::operator[](const std::string_view
 simdjson_inline simdjson_result<value> object::find_field(const std::string_view key) & noexcept {
  bool has_value;
  SIMDJSON_TRY( iter.find_field_raw(key).get(has_value) );
-  if (!has_value) { return NO_SUCH_FIELD; }
+  if (!has_value) {
+    auto key_str = std::string(key.data(), key.size());
+    auto msg = logger::string_format("Cannot find key: %s", key_str.c_str());
+    logger::log_error(iter, msg.c_str());
+    return NO_SUCH_FIELD;
+  }
  return value(iter.child());
 }
 simdjson_inline simdjson_result<value> object::find_field(const std::string_view key) && noexcept {
  bool has_value;
  SIMDJSON_TRY( iter.find_field_raw(key).get(has_value) );
-  if (!has_value) { return NO_SUCH_FIELD; }
+  if (!has_value) {
+    auto key_str = std::string(key.data(), key.size());
+    auto msg = logger::string_format("Cannot find key: %s", key_str.c_str());
+    logger::log_error(iter, msg.c_str());
+    return NO_SUCH_FIELD;
+  }
  return value(iter.child());
 }

@@ -226,4 +246,8 @@ simdjson_inline  simdjson_result<size_t> simdjson_result<SIMDJSON_IMPLEMENTATION
  return first.count_fields();
 }

+simdjson_inline  simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::object>::raw_json() noexcept {
+  if (error()) { return error(); }
+  return first.raw_json();
+}
 } // namespace simdjson
@@ -223,6 +223,7 @@ public:
  inline simdjson_result<bool> reset() noexcept;
  inline simdjson_result<bool> is_empty() noexcept;
  inline simdjson_result<size_t> count_fields() & noexcept;
+  inline simdjson_result<std::string_view> raw_json() noexcept;

 };

@@ -84,18 +84,19 @@ simdjson_warn_unused simdjson_inline simdjson_result<json_iterator> parser::iter
  return json_iterator(reinterpret_cast<const uint8_t *>(json.data()), this);
 }

-inline simdjson_result<document_stream> parser::iterate_many(const uint8_t *buf, size_t len, size_t batch_size) noexcept {
+inline simdjson_result<document_stream> parser::iterate_many(const uint8_t *buf, size_t len, size_t batch_size, bool allow_comma_separated) noexcept {
  if(batch_size < MINIMAL_BATCH_SIZE) { batch_size = MINIMAL_BATCH_SIZE; }
-  return document_stream(*this, buf, len, batch_size);
+  if(allow_comma_separated && batch_size < len) { batch_size = len; }
+  return document_stream(*this, buf, len, batch_size, allow_comma_separated);
 }
-inline simdjson_result<document_stream> parser::iterate_many(const char *buf, size_t len, size_t batch_size) noexcept {
-  return iterate_many(reinterpret_cast<const uint8_t *>(buf), len, batch_size);
+inline simdjson_result<document_stream> parser::iterate_many(const char *buf, size_t len, size_t batch_size, bool allow_comma_separated) noexcept {
+  return iterate_many(reinterpret_cast<const uint8_t *>(buf), len, batch_size, allow_comma_separated);
 }
-inline simdjson_result<document_stream> parser::iterate_many(const std::string &s, size_t batch_size) noexcept {
-  return iterate_many(s.data(), s.length(), batch_size);
+inline simdjson_result<document_stream> parser::iterate_many(const std::string &s, size_t batch_size, bool allow_comma_separated) noexcept {
+  return iterate_many(s.data(), s.length(), batch_size, allow_comma_separated);
 }
-inline simdjson_result<document_stream> parser::iterate_many(const padded_string &s, size_t batch_size) noexcept {
-  return iterate_many(s.data(), s.length(), batch_size);
+inline simdjson_result<document_stream> parser::iterate_many(const padded_string &s, size_t batch_size, bool allow_comma_separated) noexcept {
+  return iterate_many(s.data(), s.length(), batch_size, allow_comma_separated);
 }

 simdjson_inline size_t parser::capacity() const noexcept {
@@ -218,15 +218,15 @@ public:
   *         - other json errors if parsing fails. You should not rely on these errors to always the same for the
   *           same document: they may vary under runtime dispatch (so they may vary depending on your system and hardware).
   */
-  inline simdjson_result<document_stream> iterate_many(const uint8_t *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept;
+  inline simdjson_result<document_stream> iterate_many(const uint8_t *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept;
  /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */
-  inline simdjson_result<document_stream> iterate_many(const char *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept;
+  inline simdjson_result<document_stream> iterate_many(const char *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept;
  /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */
-  inline simdjson_result<document_stream> iterate_many(const std::string &s, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept;
-  inline simdjson_result<document_stream> iterate_many(const std::string &&s, size_t batch_size) = delete;// unsafe
+  inline simdjson_result<document_stream> iterate_many(const std::string &s, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept;
+  inline simdjson_result<document_stream> iterate_many(const std::string &&s, size_t batch_size, bool allow_comma_separated = false) = delete;// unsafe
  /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */
-  inline simdjson_result<document_stream> iterate_many(const padded_string &s, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept;
-  inline simdjson_result<document_stream> iterate_many(const padded_string &&s, size_t batch_size) = delete;// unsafe
+  inline simdjson_result<document_stream> iterate_many(const padded_string &s, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept;
+  inline simdjson_result<document_stream> iterate_many(const padded_string &&s, size_t batch_size, bool allow_comma_separated = false) = delete;// unsafe

  /** @private We do not want to allow implicit conversion from C string to std::string. */
  simdjson_result<document_stream> iterate_many(const char *buf, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept = delete;
@@ -116,8 +116,8 @@ protected:
  friend class json_iterator;
  friend class value_iterator;
  friend class object;
-  friend simdjson_inline void logger::log_line(const json_iterator &iter, const char *title_prefix, const char *title, std::string_view detail, int delta, int depth_delta) noexcept;
-  friend simdjson_inline void logger::log_line(const json_iterator &iter, token_position index, depth_t depth, const char *title_prefix, const char *title, std::string_view detail) noexcept;
+  friend simdjson_inline void logger::log_line(const json_iterator &iter, const char *title_prefix, const char *title, std::string_view detail, int delta, int depth_delta, logger::log_level lvl) noexcept;
+  friend simdjson_inline void logger::log_line(const json_iterator &iter, token_position index, depth_t depth, const char *title_prefix, const char *title, std::string_view detail, logger::log_level lvl) noexcept;
 };

 } // namespace ondemand
@@ -946,11 +946,13 @@ simdjson_inline void value_iterator::move_at_container_start() noexcept {
 }

 simdjson_inline simdjson_result<bool> value_iterator::reset_array() noexcept {
+  if(error()) { return error(); }
  move_at_container_start();
  return started_array();
 }

 simdjson_inline simdjson_result<bool> value_iterator::reset_object() noexcept {
+  if(error()) { return error(); }
  move_at_container_start();
  return started_object();
 }
@@ -9,7 +9,7 @@
 #define SIMDJSON_TARGET_HASWELL
 #define SIMDJSON_UNTARGET_HASWELL
 #else
-#define SIMDJSON_TARGET_HASWELL SIMDJSON_TARGET_REGION("avx2,bmi,pclmul,lzcnt")
+#define SIMDJSON_TARGET_HASWELL SIMDJSON_TARGET_REGION("avx2,bmi,pclmul,lzcnt,popcnt")
 #define SIMDJSON_UNTARGET_HASWELL SIMDJSON_UNTARGET_REGION
 #endif

@@ -9,7 +9,7 @@
 #define SIMDJSON_TARGET_ICELAKE
 #define SIMDJSON_UNTARGET_ICELAKE
 #else
-#define SIMDJSON_TARGET_ICELAKE SIMDJSON_TARGET_REGION("avx512f,avx512dq,avx512cd,avx512bw,avx512vbmi,avx512vbmi2,avx512vl,avx2,bmi,pclmul,lzcnt")
+#define SIMDJSON_TARGET_ICELAKE SIMDJSON_TARGET_REGION("avx512f,avx512dq,avx512cd,avx512bw,avx512vbmi,avx512vbmi2,avx512vl,avx2,bmi,pclmul,lzcnt,popcnt")
 #define SIMDJSON_UNTARGET_ICELAKE SIMDJSON_UNTARGET_REGION
 #endif

@@ -32,13 +32,6 @@
 #endif // __clang__
 #endif // _MSC_VER

-#if SIMDJSON_REGULAR_VISUAL_STUDIO
-// https://en.wikipedia.org/wiki/C_alternative_tokens
-// This header should have no effect, except maybe
-// under Visual Studio.
-#include <iso646.h>
-#endif
-
 #if defined(__x86_64__) || defined(_M_AMD64)
 #define SIMDJSON_IS_X86_64 1
 #elif defined(__aarch64__) || defined(_M_ARM64)
@@ -177,8 +170,9 @@ use a 64-bit target such as x64, 64-bit ARM or 64-bit PPC.")
 #define simdjson_strncasecmp strncasecmp
 #endif

-#ifdef NDEBUG
-
+#if defined(NDEBUG) || defined(__OPTIMIZE__) || (defined(_MSC_VER) && !defined(_DEBUG))
+// If NDEBUG is set, or __OPTIMIZE__ is set, or we are under MSVC in release mode,
+// then do away with asserts and use __assume.
 #if SIMDJSON_VISUAL_STUDIO
 #define SIMDJSON_UNREACHABLE() __assume(0)
 #define SIMDJSON_ASSUME(COND) __assume(COND)
@@ -187,8 +181,8 @@ use a 64-bit target such as x64, 64-bit ARM or 64-bit PPC.")
 #define SIMDJSON_ASSUME(COND) do { if (!(COND)) __builtin_unreachable(); } while (0)
 #endif

-#else // NDEBUG
-
+#else // defined(NDEBUG) || defined(__OPTIMIZE__) || (defined(_MSC_VER) && !defined(_DEBUG))
+// This should only ever be enabled in debug mode.
 #define SIMDJSON_UNREACHABLE() assert(0);
 #define SIMDJSON_ASSUME(COND) assert(COND)

@@ -4,7 +4,7 @@
 #define SIMDJSON_SIMDJSON_VERSION_H

 /** The version of simdjson being used (major.minor.revision) */
-#define SIMDJSON_VERSION "3.1.8"
+#define SIMDJSON_VERSION "3.2.0"

 namespace simdjson {
 enum {
@@ -15,11 +15,11 @@ enum {
  /**
   * The minor version (major.MINOR.revision) of simdjson being used.
   */
-  SIMDJSON_VERSION_MINOR = 1,
+  SIMDJSON_VERSION_MINOR = 2,
  /**
   * The revision (major.minor.REVISION) of simdjson being used.
   */
-  SIMDJSON_VERSION_REVISION = 8
+  SIMDJSON_VERSION_REVISION = 0
 };
 } // namespace simdjson

@@ -9,7 +9,7 @@
 #define SIMDJSON_TARGET_WESTMERE
 #define SIMDJSON_UNTARGET_WESTMERE
 #else
-#define SIMDJSON_TARGET_WESTMERE SIMDJSON_TARGET_REGION("sse4.2,pclmul")
+#define SIMDJSON_TARGET_WESTMERE SIMDJSON_TARGET_REGION("sse4.2,pclmul,popcnt")
 #define SIMDJSON_UNTARGET_WESTMERE SIMDJSON_UNTARGET_REGION
 #endif

@@ -112,7 +112,8 @@ def dofile(fid, prepath, filename):
                    print(f"// redefining SIMDJSON_IMPLEMENTATION to \"{current_implementation}\"\n// {line}", file=fid)
                elif undefines_simdjson_implementation.search(line):
                    # Don't include #undef SIMDJSON_IMPLEMENTATION since we're handling it ourselves
-                    print(f"// {line}")
+                    # print(f"// {line}")
+                    pass
                else:
                    # copy the line, with SIMDJSON_IMPLEMENTATION replace to what it is currently defined to
                    print(uses_simdjson_implementation.sub(current_implementation+"\\1",line), file=fid)
@@ -1,4 +1,4 @@
-/* auto-generated on 2023-05-14 17:17:10 -0400. Do not edit! */
+/* auto-generated on 2023-06-15 09:11:14 -0400. Do not edit! */
 /* begin file src/simdjson.cpp */
 #include "simdjson.h"

@@ -1,4 +1,4 @@
-/* auto-generated on 2023-05-14 17:17:10 -0400. Do not edit! */
+/* auto-generated on 2023-06-15 09:11:14 -0400. Do not edit! */
 /* begin file include/simdjson.h */
 #ifndef SIMDJSON_H
 #define SIMDJSON_H
@@ -43,7 +43,7 @@
 #define SIMDJSON_SIMDJSON_VERSION_H

 /** The version of simdjson being used (major.minor.revision) */
-#define SIMDJSON_VERSION "3.1.8"
+#define SIMDJSON_VERSION "3.2.0"

 namespace simdjson {
 enum {
@@ -54,11 +54,11 @@ enum {
  /**
   * The minor version (major.MINOR.revision) of simdjson being used.
   */
-  SIMDJSON_VERSION_MINOR = 1,
+  SIMDJSON_VERSION_MINOR = 2,
  /**
   * The revision (major.minor.REVISION) of simdjson being used.
   */
-  SIMDJSON_VERSION_REVISION = 8
+  SIMDJSON_VERSION_REVISION = 0
 };
 } // namespace simdjson

@@ -149,13 +149,6 @@ enum {
 #endif // __clang__
 #endif // _MSC_VER

-#if SIMDJSON_REGULAR_VISUAL_STUDIO
-// https://en.wikipedia.org/wiki/C_alternative_tokens
-// This header should have no effect, except maybe
-// under Visual Studio.
-#include <iso646.h>
-#endif
-
 #if defined(__x86_64__) || defined(_M_AMD64)
 #define SIMDJSON_IS_X86_64 1
 #elif defined(__aarch64__) || defined(_M_ARM64)
@@ -294,8 +287,9 @@ use a 64-bit target such as x64, 64-bit ARM or 64-bit PPC.")
 #define simdjson_strncasecmp strncasecmp
 #endif

-#ifdef NDEBUG
-
+#if defined(NDEBUG) || defined(__OPTIMIZE__) || (defined(_MSC_VER) && !defined(_DEBUG))
+// If NDEBUG is set, or __OPTIMIZE__ is set, or we are under MSVC in release mode,
+// then do away with asserts and use __assume.
 #if SIMDJSON_VISUAL_STUDIO
 #define SIMDJSON_UNREACHABLE() __assume(0)
 #define SIMDJSON_ASSUME(COND) __assume(COND)
@@ -304,8 +298,8 @@ use a 64-bit target such as x64, 64-bit ARM or 64-bit PPC.")
 #define SIMDJSON_ASSUME(COND) do { if (!(COND)) __builtin_unreachable(); } while (0)
 #endif

-#else // NDEBUG
-
+#else // defined(NDEBUG) || defined(__OPTIMIZE__) || (defined(_MSC_VER) && !defined(_DEBUG))
+// This should only ever be enabled in debug mode.
 #define SIMDJSON_UNREACHABLE() assert(0);
 #define SIMDJSON_ASSUME(COND) assert(COND)

@@ -13838,7 +13832,7 @@ simdjson_unused simdjson_inline simdjson_result<double> parse_double_in_string(c
 #define SIMDJSON_TARGET_ICELAKE
 #define SIMDJSON_UNTARGET_ICELAKE
 #else
-#define SIMDJSON_TARGET_ICELAKE SIMDJSON_TARGET_REGION("avx512f,avx512dq,avx512cd,avx512bw,avx512vbmi,avx512vbmi2,avx512vl,avx2,bmi,pclmul,lzcnt")
+#define SIMDJSON_TARGET_ICELAKE SIMDJSON_TARGET_REGION("avx512f,avx512dq,avx512cd,avx512bw,avx512vbmi,avx512vbmi2,avx512vl,avx2,bmi,pclmul,lzcnt,popcnt")
 #define SIMDJSON_UNTARGET_ICELAKE SIMDJSON_UNTARGET_REGION
 #endif

@@ -16040,7 +16034,7 @@ SIMDJSON_UNTARGET_ICELAKE
 #define SIMDJSON_TARGET_HASWELL
 #define SIMDJSON_UNTARGET_HASWELL
 #else
-#define SIMDJSON_TARGET_HASWELL SIMDJSON_TARGET_REGION("avx2,bmi,pclmul,lzcnt")
+#define SIMDJSON_TARGET_HASWELL SIMDJSON_TARGET_REGION("avx2,bmi,pclmul,lzcnt,popcnt")
 #define SIMDJSON_UNTARGET_HASWELL SIMDJSON_UNTARGET_REGION
 #endif

@@ -20534,7 +20528,7 @@ simdjson_unused simdjson_inline simdjson_result<double> parse_double_in_string(c
 #define SIMDJSON_TARGET_WESTMERE
 #define SIMDJSON_UNTARGET_WESTMERE
 #else
-#define SIMDJSON_TARGET_WESTMERE SIMDJSON_TARGET_REGION("sse4.2,pclmul")
+#define SIMDJSON_TARGET_WESTMERE SIMDJSON_TARGET_REGION("sse4.2,pclmul,popcnt")
 #define SIMDJSON_UNTARGET_WESTMERE SIMDJSON_UNTARGET_REGION
 #endif

@@ -23027,6 +23021,11 @@ class value_iterator;

 namespace logger {

+enum class log_level : int32_t {
+  LOG_INFO = 0,
+  LOG_ERROR = 1
+};
+
 #if SIMDJSON_VERBOSE_LOGGING
  static constexpr const bool LOG_ENABLED = true;
 #else
@@ -23036,9 +23035,11 @@ namespace logger {
 // We do not want these functions to be 'really inlined' since real inlining is
 // for performance purposes and if you are using the loggers, you do not care about
 // performance (or should not).
+template<typename... Args>
+static inline std::string string_format(const std::string& format, const Args&... args);
 static inline void log_headers() noexcept;
-static inline void log_line(const json_iterator &iter, token_position index, depth_t depth, const char *title_prefix, const char *title, std::string_view detail) noexcept;
-static inline void log_line(const json_iterator &iter, const char *title_prefix, const char *title, std::string_view detail, int delta, int depth_delta) noexcept;
+static inline void log_line(const json_iterator &iter, token_position index, depth_t depth, const char *title_prefix, const char *title, std::string_view detail, log_level level) noexcept;
+static inline void log_line(const json_iterator &iter, const char *title_prefix, const char *title, std::string_view detail, int delta, int depth_delta, log_level level) noexcept;
 static inline void log_event(const json_iterator &iter, const char *type, std::string_view detail="", int delta=0, int depth_delta=0) noexcept;
 static inline void log_value(const json_iterator &iter, token_position index, depth_t depth, const char *type, std::string_view detail="") noexcept;
 static inline void log_value(const json_iterator &iter, const char *type, std::string_view detail="", int delta=-1, int depth_delta=0) noexcept;
@@ -23381,8 +23382,8 @@ protected:
  friend class json_iterator;
  friend class value_iterator;
  friend class object;
-  friend simdjson_inline void logger::log_line(const json_iterator &iter, const char *title_prefix, const char *title, std::string_view detail, int delta, int depth_delta) noexcept;
-  friend simdjson_inline void logger::log_line(const json_iterator &iter, token_position index, depth_t depth, const char *title_prefix, const char *title, std::string_view detail) noexcept;
+  friend simdjson_inline void logger::log_line(const json_iterator &iter, const char *title_prefix, const char *title, std::string_view detail, int delta, int depth_delta, logger::log_level lvl) noexcept;
+  friend simdjson_inline void logger::log_line(const json_iterator &iter, token_position index, depth_t depth, const char *title_prefix, const char *title, std::string_view detail, logger::log_level lvl) noexcept;
 };

 } // namespace ondemand
@@ -23660,6 +23661,7 @@ public:
  simdjson_inline simdjson_result<std::string_view> unescape_wobbly(raw_json_string in) noexcept;
  simdjson_inline void reenter_child(token_position position, depth_t child_depth) noexcept;

+  simdjson_inline error_code consume_character(char c) noexcept;
 #if SIMDJSON_DEVELOPMENT_CHECKS
  simdjson_inline token_position start_position(depth_t depth) const noexcept;
  simdjson_inline void set_start_position(depth_t depth, token_position position) noexcept;
@@ -23702,8 +23704,8 @@ protected:
  friend class raw_json_string;
  friend class parser;
  friend class value_iterator;
-  friend simdjson_inline void logger::log_line(const json_iterator &iter, const char *title_prefix, const char *title, std::string_view detail, int delta, int depth_delta) noexcept;
-  friend simdjson_inline void logger::log_line(const json_iterator &iter, token_position index, depth_t depth, const char *title_prefix, const char *title, std::string_view detail) noexcept;
+  friend simdjson_inline void logger::log_line(const json_iterator &iter, const char *title_prefix, const char *title, std::string_view detail, int delta, int depth_delta, logger::log_level level) noexcept;
+  friend simdjson_inline void logger::log_line(const json_iterator &iter, token_position index, depth_t depth, const char *title_prefix, const char *title, std::string_view detail, logger::log_level level) noexcept;
 }; // json_iterator

 } // namespace ondemand
@@ -24561,6 +24563,8 @@ public:
  inline simdjson_result<bool> reset() & noexcept;
  simdjson_inline simdjson_result<SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::value> at(size_t index) noexcept;
  simdjson_inline simdjson_result<SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::value> at_pointer(std::string_view json_pointer) noexcept;
+  simdjson_inline simdjson_result<std::string_view> raw_json() noexcept;
+
 };

 } // namespace simdjson
@@ -26376,6 +26380,7 @@ public:
  inline simdjson_result<bool> reset() noexcept;
  inline simdjson_result<bool> is_empty() noexcept;
  inline simdjson_result<size_t> count_fields() & noexcept;
+  inline simdjson_result<std::string_view> raw_json() noexcept;

 };

@@ -26601,15 +26606,15 @@ public:
   *         - other json errors if parsing fails. You should not rely on these errors to always the same for the
   *           same document: they may vary under runtime dispatch (so they may vary depending on your system and hardware).
   */
-  inline simdjson_result<document_stream> iterate_many(const uint8_t *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept;
+  inline simdjson_result<document_stream> iterate_many(const uint8_t *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept;
  /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */
-  inline simdjson_result<document_stream> iterate_many(const char *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept;
+  inline simdjson_result<document_stream> iterate_many(const char *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept;
  /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */
-  inline simdjson_result<document_stream> iterate_many(const std::string &s, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept;
-  inline simdjson_result<document_stream> iterate_many(const std::string &&s, size_t batch_size) = delete;// unsafe
+  inline simdjson_result<document_stream> iterate_many(const std::string &s, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept;
+  inline simdjson_result<document_stream> iterate_many(const std::string &&s, size_t batch_size, bool allow_comma_separated = false) = delete;// unsafe
  /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */
-  inline simdjson_result<document_stream> iterate_many(const padded_string &s, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept;
-  inline simdjson_result<document_stream> iterate_many(const padded_string &&s, size_t batch_size) = delete;// unsafe
+  inline simdjson_result<document_stream> iterate_many(const padded_string &s, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept;
+  inline simdjson_result<document_stream> iterate_many(const padded_string &&s, size_t batch_size, bool allow_comma_separated = false) = delete;// unsafe

  /** @private We do not want to allow implicit conversion from C string to std::string. */
  simdjson_result<document_stream> iterate_many(const char *buf, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept = delete;
@@ -26952,7 +26957,8 @@ private:
    ondemand::parser &parser,
    const uint8_t *buf,
    size_t len,
-    size_t batch_size
+    size_t batch_size,
+    bool allow_comma_separated
  ) noexcept;

  /**
@@ -27001,6 +27007,7 @@ private:
  const uint8_t *buf;
  size_t len;
  size_t batch_size;
+  bool allow_comma_separated;
  /**
   * We are going to use just one document instance. The document owns
   * the json_iterator. It implies that we only ever pass a reference
@@ -27361,6 +27368,7 @@ simdjson_inline simdjson_result<SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::json_
 } // namespace simdjson
 /* end file include/simdjson/generic/ondemand/json_type-inl.h */
 /* begin file include/simdjson/generic/ondemand/logger-inl.h */
+#include <memory>
 namespace simdjson {
 namespace SIMDJSON_BUILTIN_IMPLEMENTATION {
 namespace ondemand {
@@ -27381,36 +27389,68 @@ static inline char printable_char(char c) {
  }
 }

+static inline log_level get_log_level_from_env()
+{
+  SIMDJSON_PUSH_DISABLE_WARNINGS
+  SIMDJSON_DISABLE_DEPRECATED_WARNING // Disable CRT_SECURE warning on MSVC: manually verified this is safe
+  char *lvl = getenv("SIMDJSON_LOG_LEVEL");
+  SIMDJSON_POP_DISABLE_WARNINGS
+  if (lvl && simdjson_strcasecmp(lvl, "ERROR") == 0) { return log_level::LOG_ERROR; }
+  return log_level::LOG_INFO;
+}
+
+static inline log_level log_threshold()
+{
+  static log_level threshold = get_log_level_from_env();
+  return threshold;
+}
+
+static inline bool should_log(log_level level)
+{
+  return level >= log_threshold();
+}
+
+template<typename... Args>
+inline std::string string_format(const std::string& format, const Args&... args)
+{
+  int size_s = std::snprintf(nullptr, 0, format.c_str(), args...) + 1;
+  auto size = static_cast<size_t>(size_s);
+  if (size <= 0) return std::string();
+  std::unique_ptr<char[]> buf(new char[size]);
+  std::snprintf(buf.get(), size, format.c_str(), args...);
+  return std::string(buf.get(), buf.get() + size - 1);
+}
+
 inline void log_event(const json_iterator &iter, const char *type, std::string_view detail, int delta, int depth_delta) noexcept {
-  log_line(iter, "", type, detail, delta, depth_delta);
+  log_line(iter, "", type, detail, delta, depth_delta, log_level::LOG_INFO);
 }

 inline void log_value(const json_iterator &iter, token_position index, depth_t depth, const char *type, std::string_view detail) noexcept {
-  log_line(iter, index, depth, "", type, detail);
+  log_line(iter, index, depth, "", type, detail, log_level::LOG_INFO);
 }
 inline void log_value(const json_iterator &iter, const char *type, std::string_view detail, int delta, int depth_delta) noexcept {
-  log_line(iter, "", type, detail, delta, depth_delta);
+  log_line(iter, "", type, detail, delta, depth_delta, log_level::LOG_INFO);
 }

 inline void log_start_value(const json_iterator &iter, token_position index, depth_t depth, const char *type, std::string_view detail) noexcept {
-  log_line(iter, index, depth, "+", type, detail);
+  log_line(iter, index, depth, "+", type, detail, log_level::LOG_INFO);
  if (LOG_ENABLED) { log_depth++; }
 }
 inline void log_start_value(const json_iterator &iter, const char *type, int delta, int depth_delta) noexcept {
-  log_line(iter, "+", type, "", delta, depth_delta);
+  log_line(iter, "+", type, "", delta, depth_delta, log_level::LOG_INFO);
  if (LOG_ENABLED) { log_depth++; }
 }

 inline void log_end_value(const json_iterator &iter, const char *type, int delta, int depth_delta) noexcept {
  if (LOG_ENABLED) { log_depth--; }
-  log_line(iter, "-", type, "", delta, depth_delta);
+  log_line(iter, "-", type, "", delta, depth_delta, log_level::LOG_INFO);
 }

 inline void log_error(const json_iterator &iter, const char *error, const char *detail, int delta, int depth_delta) noexcept {
-  log_line(iter, "ERROR: ", error, detail, delta, depth_delta);
+  log_line(iter, "ERROR: ", error, detail, delta, depth_delta, log_level::LOG_ERROR);
 }
 inline void log_error(const json_iterator &iter, token_position index, depth_t depth, const char *error, const char *detail) noexcept {
-  log_line(iter, index, depth, "ERROR: ", error, detail);
+  log_line(iter, index, depth, "ERROR: ", error, detail, log_level::LOG_ERROR);
 }

 inline void log_event(const value_iterator &iter, const char *type, std::string_view detail, int delta, int depth_delta) noexcept {
@@ -27434,7 +27474,7 @@ inline void log_error(const value_iterator &iter, const char *error, const char
 }

 inline void log_headers() noexcept {
-  if (LOG_ENABLED) {
+  if (LOG_ENABLED && simdjson_unlikely(should_log(log_level::LOG_INFO))) {
    // Technically a static variable is not thread-safe, but if you are using threads
    // and logging... well...
    static bool displayed_hint{false};
@@ -27484,11 +27524,11 @@ inline void log_headers() noexcept {
  }
 }

-inline void log_line(const json_iterator &iter, const char *title_prefix, const char *title, std::string_view detail, int delta, int depth_delta) noexcept {
-  log_line(iter, iter.position()+delta, depth_t(iter.depth()+depth_delta), title_prefix, title, detail);
+inline void log_line(const json_iterator &iter, const char *title_prefix, const char *title, std::string_view detail, int delta, int depth_delta, log_level level) noexcept {
+  log_line(iter, iter.position()+delta, depth_t(iter.depth()+depth_delta), title_prefix, title, detail, level);
 }
-inline void log_line(const json_iterator &iter, token_position index, depth_t depth, const char *title_prefix, const char *title, std::string_view detail) noexcept {
-  if (LOG_ENABLED) {
+inline void log_line(const json_iterator &iter, token_position index, depth_t depth, const char *title_prefix, const char *title, std::string_view detail, log_level level) noexcept {
+  if (LOG_ENABLED && simdjson_unlikely(should_log(level))) {
    const int indent = depth*2;
    const auto buf = iter.token.buf;
    printf("| %*s%s%-*s ",
@@ -28146,6 +28186,14 @@ simdjson_inline void json_iterator::reenter_child(token_position position, depth
  _depth = child_depth;
 }

+simdjson_inline error_code json_iterator::consume_character(char c) noexcept {
+  if (*peek() == c) {
+    return_current_and_advance();
+    return SUCCESS;
+  }
+  return TAPE_ERROR;
+}
+
 #if SIMDJSON_DEVELOPMENT_CHECKS

 simdjson_inline token_position json_iterator::start_position(depth_t depth) const noexcept {
@@ -29142,11 +29190,13 @@ simdjson_inline void value_iterator::move_at_container_start() noexcept {
 }

 simdjson_inline simdjson_result<bool> value_iterator::reset_array() noexcept {
+  if(error()) { return error(); }
  move_at_container_start();
  return started_array();
 }

 simdjson_inline simdjson_result<bool> value_iterator::reset_object() noexcept {
+  if(error()) { return error(); }
  move_at_container_start();
  return started_object();
 }
@@ -29633,6 +29683,10 @@ simdjson_inline  simdjson_result<SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::valu
  if (error()) { return error(); }
  return first.at_pointer(json_pointer);
 }
+simdjson_inline  simdjson_result<std::string_view> simdjson_result<SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::array>::raw_json() noexcept {
+  if (error()) { return error(); }
+  return first.raw_json();
+}
 } // namespace simdjson
 /* end file include/simdjson/generic/ondemand/array-inl.h */
 /* begin file include/simdjson/generic/ondemand/document-inl.h */
@@ -30955,13 +31009,23 @@ namespace ondemand {
 simdjson_inline simdjson_result<value> object::find_field_unordered(const std::string_view key) & noexcept {
  bool has_value;
  SIMDJSON_TRY( iter.find_field_unordered_raw(key).get(has_value) );
-  if (!has_value) { return NO_SUCH_FIELD; }
+  if (!has_value) {
+    auto key_str = std::string(key.data(), key.size());
+    auto msg = logger::string_format("Cannot find key: %s", key_str.c_str());
+    logger::log_error(iter, msg.c_str());
+    return NO_SUCH_FIELD;
+  }
  return value(iter.child());
 }
 simdjson_inline simdjson_result<value> object::find_field_unordered(const std::string_view key) && noexcept {
  bool has_value;
  SIMDJSON_TRY( iter.find_field_unordered_raw(key).get(has_value) );
-  if (!has_value) { return NO_SUCH_FIELD; }
+  if (!has_value) {
+    auto key_str = std::string(key.data(), key.size());
+    auto msg = logger::string_format("Cannot find key: %s", key_str.c_str());
+    logger::log_error(iter, msg.c_str());
+    return NO_SUCH_FIELD;
+  }
  return value(iter.child());
 }
 simdjson_inline simdjson_result<value> object::operator[](const std::string_view key) & noexcept {
@@ -30973,13 +31037,23 @@ simdjson_inline simdjson_result<value> object::operator[](const std::string_view
 simdjson_inline simdjson_result<value> object::find_field(const std::string_view key) & noexcept {
  bool has_value;
  SIMDJSON_TRY( iter.find_field_raw(key).get(has_value) );
-  if (!has_value) { return NO_SUCH_FIELD; }
+  if (!has_value) {
+    auto key_str = std::string(key.data(), key.size());
+    auto msg = logger::string_format("Cannot find key: %s", key_str.c_str());
+    logger::log_error(iter, msg.c_str());
+    return NO_SUCH_FIELD;
+  }
  return value(iter.child());
 }
 simdjson_inline simdjson_result<value> object::find_field(const std::string_view key) && noexcept {
  bool has_value;
  SIMDJSON_TRY( iter.find_field_raw(key).get(has_value) );
-  if (!has_value) { return NO_SUCH_FIELD; }
+  if (!has_value) {
+    auto key_str = std::string(key.data(), key.size());
+    auto msg = logger::string_format("Cannot find key: %s", key_str.c_str());
+    logger::log_error(iter, msg.c_str());
+    return NO_SUCH_FIELD;
+  }
  return value(iter.child());
 }

@@ -31176,6 +31250,10 @@ simdjson_inline  simdjson_result<size_t> simdjson_result<SIMDJSON_BUILTIN_IMPLEM
  return first.count_fields();
 }

+simdjson_inline  simdjson_result<std::string_view> simdjson_result<SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::object>::raw_json() noexcept {
+  if (error()) { return error(); }
+  return first.raw_json();
+}
 } // namespace simdjson
 /* end file include/simdjson/generic/ondemand/object-inl.h */
 /* begin file include/simdjson/generic/ondemand/parser-inl.h */
@@ -31265,18 +31343,19 @@ simdjson_warn_unused simdjson_inline simdjson_result<json_iterator> parser::iter
  return json_iterator(reinterpret_cast<const uint8_t *>(json.data()), this);
 }

-inline simdjson_result<document_stream> parser::iterate_many(const uint8_t *buf, size_t len, size_t batch_size) noexcept {
+inline simdjson_result<document_stream> parser::iterate_many(const uint8_t *buf, size_t len, size_t batch_size, bool allow_comma_separated) noexcept {
  if(batch_size < MINIMAL_BATCH_SIZE) { batch_size = MINIMAL_BATCH_SIZE; }
-  return document_stream(*this, buf, len, batch_size);
+  if(allow_comma_separated && batch_size < len) { batch_size = len; }
+  return document_stream(*this, buf, len, batch_size, allow_comma_separated);
 }
-inline simdjson_result<document_stream> parser::iterate_many(const char *buf, size_t len, size_t batch_size) noexcept {
-  return iterate_many(reinterpret_cast<const uint8_t *>(buf), len, batch_size);
+inline simdjson_result<document_stream> parser::iterate_many(const char *buf, size_t len, size_t batch_size, bool allow_comma_separated) noexcept {
+  return iterate_many(reinterpret_cast<const uint8_t *>(buf), len, batch_size, allow_comma_separated);
 }
-inline simdjson_result<document_stream> parser::iterate_many(const std::string &s, size_t batch_size) noexcept {
-  return iterate_many(s.data(), s.length(), batch_size);
+inline simdjson_result<document_stream> parser::iterate_many(const std::string &s, size_t batch_size, bool allow_comma_separated) noexcept {
+  return iterate_many(s.data(), s.length(), batch_size, allow_comma_separated);
 }
-inline simdjson_result<document_stream> parser::iterate_many(const padded_string &s, size_t batch_size) noexcept {
-  return iterate_many(s.data(), s.length(), batch_size);
+inline simdjson_result<document_stream> parser::iterate_many(const padded_string &s, size_t batch_size, bool allow_comma_separated) noexcept {
+  return iterate_many(s.data(), s.length(), batch_size, allow_comma_separated);
 }

 simdjson_inline size_t parser::capacity() const noexcept {
@@ -31413,12 +31492,14 @@ simdjson_inline document_stream::document_stream(
  ondemand::parser &_parser,
  const uint8_t *_buf,
  size_t _len,
-  size_t _batch_size
+  size_t _batch_size,
+  bool _allow_comma_separated
 ) noexcept
  : parser{&_parser},
    buf{_buf},
    len{_len},
    batch_size{_batch_size <= MINIMAL_BATCH_SIZE ? MINIMAL_BATCH_SIZE : _batch_size},
+    allow_comma_separated{_allow_comma_separated},
    error{SUCCESS}
    #ifdef SIMDJSON_THREADS_ENABLED
    , use_thread(_parser.threaded) // we need to make a copy because _parser.threaded can change
@@ -31436,6 +31517,7 @@ simdjson_inline document_stream::document_stream() noexcept
    buf{nullptr},
    len{0},
    batch_size{0},
+    allow_comma_separated{false},
    error{UNINITIALIZED}
    #ifdef SIMDJSON_THREADS_ENABLED
    , use_thread(false)
@@ -31619,6 +31701,8 @@ inline void document_stream::next_document() noexcept {
  if (error) { return; }
  // Always set depth=1 at the start of document
  doc.iter._depth = 1;
+  // consume comma if comma separated is allowed
+  if (allow_comma_separated) { doc.iter.consume_character(','); }
  // Resets the string buffer at the beginning, thus invalidating the strings.
  doc.iter._string_buf_loc = parser->string_buf.get();
  doc.iter._root = doc.iter.position();
@@ -9,7 +9,6 @@
 #include <set>
 #include <sstream>
 #include <utility>
-#include <ciso646>
 #include <unistd.h>

 #include "simdjson.h"
@@ -67,6 +66,15 @@ namespace number_tests {
    return true;
  }

+  bool issue2017() {
+    TEST_START();
+    simdjson::dom::parser parser;
+    simdjson::padded_string docdata = R"({"score":0.8825149536132812})"_padded;
+    double score;
+    ASSERT_SUCCESS(parser.parse(docdata)["score"].get_double().get(score));
+    ASSERT_EQUAL(score, 0.8825149536132812);
+    TEST_SUCCEED();
+  }

  bool small_integers() {
    std::cout << __func__ << std::endl;
@@ -378,7 +386,8 @@ namespace number_tests {
  }

  bool run() {
-    return truncated_borderline() &&
+    return issue2017() &&
+           truncated_borderline() &&
           specific_tests() &&
           ground_truth() &&
           small_integers() &&
@@ -60,7 +60,6 @@ static inline void write_utf8(unsigned codepoint, char *&end) {
    *end++ = static_cast<char>(0x80 | ((codepoint >> 6) & 0x3F));
    *end++ = static_cast<char>(0x80 | (codepoint & 0x3F));
  } else {
-    assert(codepoint < 0x200000);
    *end++ = static_cast<char>(0xF0 | (codepoint >> 18));
    *end++ = static_cast<char>(0x80 | ((codepoint >> 12) & 0x3F));
    *end++ = static_cast<char>(0x80 | ((codepoint >> 6) & 0x3F));
@@ -1,5 +1,4 @@
 #include <cinttypes>
-#include <ciso646>
 #include <cmath>
 #include <cstdio>
 #include <cstdlib>
@@ -3,6 +3,7 @@ link_libraries(simdjson)
 include_directories(..)
 add_subdirectory(compilation_failure_tests)
 add_cpp_test(ondemand_log_tests              LABELS ondemand acceptance per_implementation)
+add_cpp_test(ondemand_log_error_tests        LABELS ondemand acceptance per_implementation)
 add_cpp_test(ondemand_tostring_tests         LABELS ondemand acceptance per_implementation)
 add_cpp_test(ondemand_active_tests           LABELS ondemand acceptance per_implementation)
 add_cpp_test(ondemand_array_tests            LABELS ondemand acceptance per_implementation)
@@ -24,6 +25,7 @@ add_cpp_test(ondemand_readme_examples        LABELS ondemand acceptance per_impl
 add_cpp_test(ondemand_scalar_tests           LABELS ondemand acceptance per_implementation)
 add_cpp_test(ondemand_twitter_tests          LABELS ondemand acceptance per_implementation)
 add_cpp_test(ondemand_wrong_type_error_tests LABELS ondemand acceptance per_implementation)
+add_cpp_test(ondemand_iterate_many_csv       LABELS ondemand acceptance per_implementation)

 if(HAVE_POSIX_FORK AND HAVE_POSIX_WAIT) # assert tests use fork and wait, which aren't on MSVC
  add_cpp_test(ondemand_assert_out_of_order_values LABELS assert per_implementation explicitonly ondemand)
@@ -6,6 +6,59 @@ using namespace simdjson;
 namespace error_tests {
  using namespace std;

+  bool badbadjson() {
+    TEST_START();
+    ondemand::parser parser;
+    auto json_string = R"({
+      "main": "therain"_"in_spain"_
+})"_padded;
+    ondemand::document document;
+    auto error = parser.iterate(json_string).get(document);
+    if(error != simdjson::SUCCESS) {
+      return false;
+    }
+    ondemand::object obj;
+    error =  document.get_object().get(obj);
+    if(error != simdjson::SUCCESS) {
+      return false;
+    }
+    std::string_view name_value{};
+    error = obj["name"].get_string().get(name_value);
+    ASSERT_ERROR(error,TAPE_ERROR);
+    // Check for "main" field
+    std::string_view main_value{};
+    error = obj["main"].get_string().get(main_value);
+    ASSERT_ERROR(error,TAPE_ERROR);
+    TEST_SUCCEED();
+  }
+
+
+  bool badbadjson2() {
+    TEST_START();
+    ondemand::parser parser;
+    auto json_string = R"({
+      "main": "therain"_"in_spain"_
+})"_padded;
+    ondemand::document document;
+    auto error = parser.iterate(json_string).get(document);
+    if(error != simdjson::SUCCESS) {
+      return false;
+    }
+    ondemand::object obj;
+    error =  document.get_object().get(obj);
+    if(error != simdjson::SUCCESS) {
+      return false;
+    }
+
+    std::string_view main_value{};
+    error = obj["main"].get_string().get(main_value);
+    ASSERT_ERROR(error, simdjson::SUCCESS);
+    std::string_view name_value{};
+    error = obj["name"].get_string().get(name_value);
+    ASSERT_ERROR(error,TAPE_ERROR);
+    TEST_SUCCEED();
+  }
+
  bool issue1834() {
    TEST_START();
    ondemand::parser parser;
@@ -313,6 +366,8 @@ namespace error_tests {

  bool run() {
    return
+           badbadjson() &&
+           badbadjson2() &&
           issue1834() &&
           issue1834_2() &&
 #if SIMDJSON_EXCEPTIONS
@@ -0,0 +1,160 @@
+#include "simdjson.h"
+#include "test_ondemand.h"
+
+#include <cstdint>
+
+using namespace simdjson;
+
+namespace iterate_many_csv_tests {
+using namespace std;
+
+bool normal() {
+  TEST_START();
+  auto json = R"( 1, 2, 3, 4, "a", "b", "c", {"hello": "world"} , [1, 2, 3])"_padded;
+  ondemand::parser parser;
+  ondemand::document_stream doc_stream;
+  ASSERT_SUCCESS(parser.iterate_many(json, json.size(), true).get(doc_stream));
+
+  for (auto doc : doc_stream)
+  {
+    ASSERT_SUCCESS(doc);
+  }
+
+  TEST_SUCCEED();
+}
+
+bool small_batch_size() {
+  TEST_START();
+  auto json = R"( 1, 2, 3, 4, "a", "b", "c", {"hello": "world"} , [1, 2, 3])"_padded;
+  ondemand::parser parser;
+  ondemand::document_stream doc_stream;
+  ASSERT_SUCCESS(parser.iterate_many(json, 32, true).get(doc_stream));
+
+  for (auto doc : doc_stream)
+  {
+    ASSERT_SUCCESS(doc);
+  }
+
+  TEST_SUCCEED();
+}
+
+bool trailing_comma() {
+  TEST_START();
+  auto json = R"(1,)"_padded;
+  ondemand::parser parser;
+  ondemand::document_stream doc_stream;
+  ASSERT_SUCCESS(parser.iterate_many(json, json.size(), true).get(doc_stream));
+
+  for (auto doc : doc_stream)
+  {
+    ASSERT_SUCCESS(doc);
+  }
+
+  TEST_SUCCEED();
+}
+
+bool check_parsed_values() {
+  TEST_START();
+
+  auto json = R"(  1  , "a" , [100, 1]  , {"hello"  :    "world"}  , )"_padded;
+  ondemand::parser parser;
+  ondemand::document_stream doc_stream;
+  ASSERT_SUCCESS(parser.iterate_many(json, json.size(), true).get(doc_stream));
+
+  auto begin = doc_stream.begin();
+  auto end = doc_stream.end();
+  int cnt = 0;
+  auto it = begin;
+  for (; it != end && cnt < 4; ++it, ++cnt) {
+    auto doc = *it;
+    switch (cnt)
+    {
+    case 0:
+    {
+      int64_t actual;
+      ASSERT_SUCCESS(doc.get_int64().get(actual));
+      ASSERT_EQUAL(actual, 1);
+      break;
+    }
+    case 1:
+    {
+      std::string_view sv;
+      ASSERT_SUCCESS(doc.get_string().get(sv));
+      ASSERT_EQUAL(sv, "a");
+      break;
+    }
+    case 2:
+    {
+      std::vector<int64_t> expected{100, 1};
+      ondemand::array arr;
+      ASSERT_SUCCESS(doc.get_array().get(arr));
+      size_t element_count;
+      ASSERT_SUCCESS(arr.count_elements().get(element_count));
+      ASSERT_EQUAL(element_count, 2);
+      int i = 0;
+      for (auto a : arr)
+      {
+        int64_t actual;
+        ASSERT_SUCCESS(a.get(actual));
+        ASSERT_EQUAL(actual, expected[i++]);
+      }
+      break;
+    }
+    case 3:
+    {
+      ondemand::object obj;
+      ASSERT_SUCCESS(doc.get_object().get(obj));
+      std::string_view sv;
+      obj.find_field("hello").get(sv);
+      ASSERT_EQUAL(sv, "world");
+      break;
+    }
+    default:
+      TEST_FAIL("Too many cases")
+    }
+  }
+
+  ASSERT_EQUAL(cnt, 4);
+  ASSERT_TRUE(!(it != end));
+
+  TEST_SUCCEED();
+}
+
+#if SIMDJSON_EXCEPTIONS
+
+bool leading_comma() {
+  TEST_START();
+  auto json = R"(,1)"_padded;
+  ondemand::parser parser;
+  ondemand::document_stream doc_stream;
+  ASSERT_SUCCESS(parser.iterate_many(json, json.size(), true).get(doc_stream));
+
+  try {
+    auto begin = doc_stream.begin();
+    auto end = doc_stream.end();
+    for (auto it = begin; it != end; ++it) {}
+  } catch (simdjson_error& e) {
+    ASSERT_ERROR(e.error(), TAPE_ERROR);
+  }
+
+  TEST_SUCCEED();
+}
+
+#endif
+
+bool run() {
+  return normal() &&
+         small_batch_size() &&
+         trailing_comma() &&
+         check_parsed_values() &&
+#if SIMDJSON_EXCEPTIONS
+         leading_comma() &&
+#endif
+         true;
+}
+
+}
+
+int main(int argc, char *argv[]) {
+  return test_main(argc, argv, iterate_many_csv_tests::run);
+}
@@ -195,7 +195,7 @@ namespace json_pointer_tests {
        ASSERT_SUCCESS(parser.iterate(cars_json).get(cars));
        for (int i = 0; i < 3; i++) {
            double x;
-            std::string json_pointer = "/" + std::to_string(i) + "/tire_pressure/1";
+            std::string json_pointer = std::string("/") + std::to_string(i) + std::string("/tire_pressure/1");
            ASSERT_SUCCESS(cars.at_pointer(json_pointer).get(x));
            measured.push_back(x);
        }
@@ -315,7 +315,7 @@ namespace json_pointer_tests {
        std::vector<car_type> content;
        for (int i = 0; i < 3; i++) {
            ondemand::object obj;
-            std::string json_pointer = "/" + std::to_string(i);
+            std::string json_pointer = std::string("/") + std::to_string(i);
            // Each successive at_pointer call invalidates
            // previously parsed values, strings, objects and array.
            ASSERT_SUCCESS(cars.at_pointer(json_pointer).get(obj));
@@ -360,7 +360,7 @@ namespace json_pointer_tests {
        ondemand::document cars = parser.iterate(cars_json);
        std::vector<car_type> content;
        for (int i = 0; i < 3; i++) {
-            std::string json_pointer = "/" + std::to_string(i);
+            std::string json_pointer = std::string("/") + std::to_string(i);
            // Each successive at_pointer call invalidates
            // previously parsed values, strings, objects and array.
            ondemand::object obj(cars.at_pointer(json_pointer).get_object());
@@ -0,0 +1,67 @@
+#define SIMDJSON_VERBOSE_LOGGING 1
+#include "simdjson.h"
+#include "test_ondemand.h"
+
+#include <iostream>
+#include <string>
+#include <stdlib.h>
+
+using namespace simdjson;
+
+namespace log_error_tests {
+#if SIMDJSON_EXCEPTIONS
+
+using namespace std;
+
+bool tape_error()
+{
+  TEST_START();
+  auto json = R"( {"a", "hello"} )"_padded;
+  ondemand::parser parser;
+  try {
+    ondemand::document doc = parser.iterate(json);
+    std::cout << doc["a"] << std::endl;
+    TEST_FAIL("Should have thrown an exception!")
+  } catch (simdjson_error& e) {
+    ASSERT_ERROR(e.error(), TAPE_ERROR);
+  }
+  TEST_SUCCEED();
+}
+
+bool no_such_field()
+{
+  TEST_START();
+  auto json = R"( {"a": "hello"} )"_padded;
+  ondemand::parser parser;
+  try {
+    ondemand::document doc = parser.iterate(json);
+    std::cout << doc["missing_key"] << std::endl;
+    TEST_FAIL("Should have thrown an exception!")
+  } catch (simdjson_error& e) {
+    ASSERT_ERROR(e.error(), NO_SUCH_FIELD);
+  }
+  TEST_SUCCEED();
+}
+#endif // SIMDJSON_EXCEPTIONS
+
+bool run()
+{
+  SIMDJSON_PUSH_DISABLE_WARNINGS
+  SIMDJSON_DISABLE_DEPRECATED_WARNING // Disable CRT_SECURE warning on MSVC: manually verified this is safe
+  std::string str = "SIMDJSON_LOG_LEVEL=ERROR";
+  putenv(str.data());
+  bool rc =
+#if SIMDJSON_EXCEPTIONS
+            tape_error() &&
+            no_such_field() &&
+#endif // #if SIMDJSON_EXCEPTIONS
+            true;
+  SIMDJSON_POP_DISABLE_WARNINGS
+  return rc;
+}
+
+}
+
+int main(int argc, char *argv[]) {
+  return test_main(argc, argv, log_error_tests::run);
+}
@@ -133,7 +133,7 @@ namespace misc_tests {
    ASSERT_SUCCESS(parser.iterate(json).get(doc));
    std::string_view view;
    ASSERT_SUCCESS( doc.get_string().get(view));
-    ASSERT_EQUAL(view, u8"wow:\uFFFF");
+    ASSERT_EQUAL(view, "wow:\xef\xbf\xbf");
    TEST_SUCCEED();
  }

@@ -536,9 +536,9 @@ namespace misc_tests {
    return true;
  }
  simdjson_warn_unused bool test_raw_json_token(string_view json, string_view expected_token, int expected_start_index = 0) {
-    string title = "'";
+    string title("'");
    title.append(json.data(), json.length());
-    title += "'";
+    title += std::string("'");
    padded_string json_padded = json;
    SUBTEST(title, test_ondemand_doc(json_padded, [&](auto doc) {
      string_view token;
@@ -552,11 +552,11 @@ namespace misc_tests {
    // Test values
    auto json_in_hash = string(R"({"a":)");
    json_in_hash.append(json.data(), json.length());
-    json_in_hash += "}";
+    json_in_hash += std::string("}");
    json_padded = json_in_hash;
-    title = "'";
+    title = std::string("'");
    title.append(json_in_hash.data(), json_in_hash.length());
-    title += "'";
+    title += std::string("'");
    SUBTEST(title, test_ondemand_doc(json_padded, [&](auto doc) {
      string_view token;
      ASSERT_SUCCESS( doc["a"].raw_json_token().get(token) );
@@ -333,6 +333,8 @@ namespace number_tests {
    TEST_SUCCEED();
  }

+
+
  bool get_root_number_tests() {
    TEST_START();
    ondemand::parser parser;
@@ -389,8 +391,21 @@ namespace number_tests {

    TEST_SUCCEED();
  }
+  bool issue2017() {
+    TEST_START();
+    ondemand::parser parser;
+    ondemand::document doc;
+    padded_string docdata = R"({"score":0.8825149536132812})"_padded;
+    ASSERT_SUCCESS(parser.iterate(docdata).get(doc));
+    double score;
+    ASSERT_SUCCESS(doc["score"].get_double().get(score));
+    ASSERT_EQUAL(score, 0.8825149536132812);
+    TEST_SUCCEED();
+  }
+
  bool run() {
-    return issue_1898() &&
+    return issue2017() &&
+           issue_1898() &&
           issue1878() &&
           get_root_number_tests() &&
           get_number_tests()&&
@@ -21,108 +21,150 @@ bool string2() {


 #if SIMDJSON_EXCEPTIONS
+bool gen_raw1() {
+  TEST_START();
+  simdjson::ondemand::parser parser;
+  simdjson::padded_string docdata =  R"({"value":123})"_padded;
+  simdjson::ondemand::document doc = parser.iterate(docdata);
+  simdjson::ondemand::object obj = doc.get_object();
+  string_view token = obj.raw_json(); // gives you `{"value":123}`
+  ASSERT_EQUAL(token, R"({"value":123})");
+  TEST_SUCCEED();
+}

-  bool at_end() {
-    auto json = R"([1, 2] foo ])"_padded;
-    ondemand::parser parser;
-    ondemand::document doc = parser.iterate(json);
-    ondemand::array array = doc.get_array();
-    for (uint64_t values : array) {
-      std::cout << values << std::endl;
-    }
-    if(!doc.at_end()) {
-      std::cerr << "trailing content at byte index " << doc.current_location() - json.data() << std::endl;
-    }
-    TEST_SUCCEED();
-  }
-  bool number_tests() {
-    ondemand::parser parser;
-    padded_string docdata = R"([1.0, 3, 1, 3.1415,-13231232,9999999999999999999])"_padded;
-    ondemand::document doc = parser.iterate(docdata);
-    ondemand::array arr = doc.get_array();
-    for(ondemand::value val : arr) {
-      std::cout << val << " ";
-      std::cout << "negative: " << val.is_negative() << " ";
-      std::cout << "is_integer: " << val.is_integer() << " ";
-      ondemand::number num = val.get_number();
-      ondemand::number_type t = num.get_number_type();
-      // direct computation without materializing the number:
-      ondemand::number_type dt = val.get_number_type();
-      if(t != dt) { throw std::runtime_error("bug"); }
-      switch(t) {
-        case ondemand::number_type::signed_integer:
-          std::cout  << "integer: " << int64_t(num) << " ";
-          std::cout  << "integer: " << num.get_int64() << std::endl;
-          break;
-        case ondemand::number_type::unsigned_integer:
-          std::cout  << "large 64-bit integer: " << uint64_t(num) << " ";
-          std::cout << "large 64-bit integer: " << num.get_uint64() << std::endl;
-          break;
-        case ondemand::number_type::floating_point_number:
-          std::cout  << "float: " << double(num) << " ";
-          std::cout << "float: " << num.get_double() << std::endl;
-          break;
-      }
-    }
-    return true;
-  }
+bool gen_raw2() {
+  TEST_START();
+  simdjson::ondemand::parser parser;
+  simdjson::padded_string docdata =  R"([1,2,3])"_padded;
+  simdjson::ondemand::document doc = parser.iterate(docdata);
+  simdjson::ondemand::array arr = doc.get_array();
+  string_view token = arr.raw_json(); // gives you `[1,2,3]`
+  ASSERT_EQUAL(token, R"([1,2,3])");
+  TEST_SUCCEED();
+}

-void recursive_print_json(ondemand::value element) {
-    bool add_comma;
-    switch (element.type()) {
-    case ondemand::json_type::array:
-      cout << "[";
-      add_comma = false;
-      for (auto child : element.get_array()) {
-        if (add_comma) {
-          cout << ",";
-        }
-        // We need the call to value() to get
-        // an ondemand::value type.
-        recursive_print_json(child.value());
-        add_comma = true;
-      }
-      cout << "]";
-      break;
-    case ondemand::json_type::object:
-      cout << "{";
-      add_comma = false;
-      for (auto field : element.get_object()) {
-        if (add_comma) {
-          cout << ",";
-        }
-        // key() returns the key as it appears in the raw
-        // JSON document, if we want the unescaped key,
-        // we should do field.unescaped_key().
-        cout << "\"" << field.key() << "\": ";
-        recursive_print_json(field.value());
-        add_comma = true;
-      }
-      cout << "}\n";
-      break;
-    case ondemand::json_type::number:
-      // assume it fits in a double
-      cout << element.get_double();
-      break;
-    case ondemand::json_type::string:
-      // get_string() would return escaped string, but
-      // we are happy with unescaped string.
-      cout << "\"" << element.get_raw_json_string() << "\"";
-      break;
-    case ondemand::json_type::boolean:
-      cout << element.get_bool();
-      break;
-    case ondemand::json_type::null:
-      // we check that the value is indeed null
-      // otherwise: an error is thrown.
-      if(element.is_null()) {
-        cout << "null";
-      }
-      break;
+bool gen_raw3() {
+  TEST_START();
+  simdjson::ondemand::parser parser;
+  simdjson::padded_string docdata =  R"({"value":123})"_padded;
+  simdjson::ondemand::document doc = parser.iterate(docdata);
+  simdjson::ondemand::object obj = doc.get_object();
+  string_view token = obj.raw_json(); // gives you `{"value":123}`
+  ASSERT_EQUAL(token, R"({"value":123})");
+  obj.reset(); // revise the object
+  uint64_t x = obj["value"]; // gives me 123
+  ASSERT_EQUAL(x, 123);
+
+  TEST_SUCCEED();
+}
+
+bool at_end() {
+  TEST_START();
+  auto json = R"([1, 2] foo ])"_padded;
+  ondemand::parser parser;
+  ondemand::document doc = parser.iterate(json);
+  ondemand::array array = doc.get_array();
+  for (uint64_t values : array) {
+    std::cout << values << std::endl;
+  }
+  if(!doc.at_end()) {
+    std::cerr << "trailing content at byte index " << doc.current_location() - json.data() << std::endl;
+  }
+  TEST_SUCCEED();
+}
+
+bool number_tests() {
+  TEST_START();
+  ondemand::parser parser;
+  padded_string docdata = R"([1.0, 3, 1, 3.1415,-13231232,9999999999999999999])"_padded;
+  ondemand::document doc = parser.iterate(docdata);
+  ondemand::array arr = doc.get_array();
+  for(ondemand::value val : arr) {
+    std::cout << val << " ";
+    std::cout << "negative: " << val.is_negative() << " ";
+    std::cout << "is_integer: " << val.is_integer() << " ";
+    ondemand::number num = val.get_number();
+    ondemand::number_type t = num.get_number_type();
+    // direct computation without materializing the number:
+    ondemand::number_type dt = val.get_number_type();
+    if(t != dt) { throw std::runtime_error("bug"); }
+    switch(t) {
+      case ondemand::number_type::signed_integer:
+        std::cout  << "integer: " << int64_t(num) << " ";
+        std::cout  << "integer: " << num.get_int64() << std::endl;
+        break;
+      case ondemand::number_type::unsigned_integer:
+        std::cout  << "large 64-bit integer: " << uint64_t(num) << " ";
+        std::cout << "large 64-bit integer: " << num.get_uint64() << std::endl;
+        break;
+      case ondemand::number_type::floating_point_number:
+        std::cout  << "float: " << double(num) << " ";
+        std::cout << "float: " << num.get_double() << std::endl;
+        break;
    }
+  }
+  TEST_SUCCEED();
+}
+
+bool recursive_print_json(ondemand::value element) {
+  TEST_START();
+  bool add_comma;
+  switch (element.type()) {
+  case ondemand::json_type::array:
+    cout << "[";
+    add_comma = false;
+    for (auto child : element.get_array()) {
+      if (add_comma) {
+        cout << ",";
+      }
+      // We need the call to value() to get
+      // an ondemand::value type.
+      recursive_print_json(child.value());
+      add_comma = true;
+    }
+    cout << "]";
+    break;
+  case ondemand::json_type::object:
+    cout << "{";
+    add_comma = false;
+    for (auto field : element.get_object()) {
+      if (add_comma) {
+        cout << ",";
+      }
+      // key() returns the key as it appears in the raw
+      // JSON document, if we want the unescaped key,
+      // we should do field.unescaped_key().
+      cout << "\"" << field.key() << "\": ";
+      recursive_print_json(field.value());
+      add_comma = true;
+    }
+    cout << "}\n";
+    break;
+  case ondemand::json_type::number:
+    // assume it fits in a double
+    cout << element.get_double();
+    break;
+  case ondemand::json_type::string:
+    // get_string() would return escaped string, but
+    // we are happy with unescaped string.
+    cout << "\"" << element.get_raw_json_string() << "\"";
+    break;
+  case ondemand::json_type::boolean:
+    cout << element.get_bool();
+    break;
+  case ondemand::json_type::null:
+    // we check that the value is indeed null
+    // otherwise: an error is thrown.
+    if(element.is_null()) {
+      cout << "null";
+    }
+    break;
+  }
+  TEST_SUCCEED();
 }

 bool basics_treewalk() {
+  TEST_START();
  padded_string json[3] = {R"( [
    { "make": "Toyota", "model": "Camry",  "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
    { "make": "Kia",    "model": "Soul",   "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
@@ -135,77 +177,82 @@ bool basics_treewalk() {
    recursive_print_json(val);
    std::cout << std::endl;
  }
-  return true;
+  TEST_SUCCEED();
 }

-void print_depth_space(ondemand::value element) {
+bool print_depth_space(ondemand::value element) {
+  TEST_START();
  for(auto i = 0; i < element.current_depth(); i++) {
    cout << " ";
  }
+  TEST_SUCCEED();
 }

-void recursive_print_json_breakline(ondemand::value element) {
-    bool add_comma;
-    switch (element.type()) {
-    case ondemand::json_type::array:
-      cout << endl;
-      print_depth_space(element);
-      cout << "[";
-      add_comma = false;
-      for (auto child : element.get_array()) {
-        if (add_comma) {
-          print_depth_space(element);
-          cout << ",";
-        }
-        // We need the call to value() to get
-        // an ondemand::value type.
-        recursive_print_json_breakline(child.value());
-        add_comma = true;
+bool recursive_print_json_breakline(ondemand::value element) {
+  TEST_START();
+  bool add_comma;
+  switch (element.type()) {
+  case ondemand::json_type::array:
+    cout << endl;
+    print_depth_space(element);
+    cout << "[";
+    add_comma = false;
+    for (auto child : element.get_array()) {
+      if (add_comma) {
+        print_depth_space(element);
+        cout << ",";
      }
-      cout << "]";
-      break;
-    case ondemand::json_type::object:
-      cout << endl;
-      print_depth_space(element);
-      cout << "{";
-      add_comma = false;
-      for (auto field : element.get_object()) {
-        if (add_comma) {
-          print_depth_space(element);
-          cout << ",";
-        }
-        // key() returns the key as it appears in the raw
-        // JSON document, if we want the unescaped key,
-        // we should do field.unescaped_key().
-        cout << "\"" << field.key() << "\": ";
-        recursive_print_json_breakline(field.value());
-        add_comma = true;
-      }
-      cout << "}\n";
-      break;
-    case ondemand::json_type::number:
-      // assume it fits in a double
-      cout << element.get_double();
-      break;
-    case ondemand::json_type::string:
-      // get_string() would return escaped string, but
-      // we are happy with unescaped string.
-      cout << "\"" << element.get_raw_json_string() << "\"";
-      break;
-    case ondemand::json_type::boolean:
-      cout << element.get_bool();
-      break;
-    case ondemand::json_type::null:
-      // We check that the value is indeed null
-      // otherwise: an error is thrown.
-      if(element.is_null()) {
-        cout << "null";
-      }
-      break;
+      // We need the call to value() to get
+      // an ondemand::value type.
+      recursive_print_json_breakline(child.value());
+      add_comma = true;
    }
+    cout << "]";
+    break;
+  case ondemand::json_type::object:
+    cout << endl;
+    print_depth_space(element);
+    cout << "{";
+    add_comma = false;
+    for (auto field : element.get_object()) {
+      if (add_comma) {
+        print_depth_space(element);
+        cout << ",";
+      }
+      // key() returns the key as it appears in the raw
+      // JSON document, if we want the unescaped key,
+      // we should do field.unescaped_key().
+      cout << "\"" << field.key() << "\": ";
+      recursive_print_json_breakline(field.value());
+      add_comma = true;
+    }
+    cout << "}\n";
+    break;
+  case ondemand::json_type::number:
+    // assume it fits in a double
+    cout << element.get_double();
+    break;
+  case ondemand::json_type::string:
+    // get_string() would return escaped string, but
+    // we are happy with unescaped string.
+    cout << "\"" << element.get_raw_json_string() << "\"";
+    break;
+  case ondemand::json_type::boolean:
+    cout << element.get_bool();
+    break;
+  case ondemand::json_type::null:
+    // We check that the value is indeed null
+    // otherwise: an error is thrown.
+    if(element.is_null()) {
+      cout << "null";
+    }
+    break;
+  }
+  TEST_SUCCEED();
 }

 bool basics_treewalk_breakline() {
+  TEST_START();
  padded_string json[3] = {R"( [
    { "make": "Toyota", "model": "Camry",  "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
    { "make": "Kia",    "model": "Soul",   "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
@@ -218,7 +265,7 @@ bool basics_treewalk_breakline() {
    recursive_print_json_breakline(val);
    std::cout << std::endl;
  }
-  return true;
+  TEST_SUCCEED();
 }

 bool basics_1() {
@@ -721,7 +768,7 @@ bool json_pointer_multiple() {
 	ASSERT_SUCCESS(cars.count_elements().get(size));
 	double expected[] = {39.9, 31, 30};
 	for (size_t i = 0; i < size; i++) {
-		std::string json_pointer = "/" + std::to_string(i) + "/tire_pressure/1";
+		std::string json_pointer = std::string("/") + std::to_string(i) + std::string("/tire_pressure/1");
 		double x;
 		ASSERT_SUCCESS(cars.at_pointer(json_pointer).get(x));
 		ASSERT_EQUAL(x,expected[i]);
@@ -974,6 +1021,22 @@ int load_example_except() {
  std::cout << identifier << std::endl;
  return EXIT_SUCCESS;
 }
+int load_example_except_morecomplete(void) {
+  TEST_START();
+  simdjson::ondemand::parser parser;
+  simdjson::padded_string json_string;
+  simdjson::ondemand::document doc;
+  try {
+    json_string = padded_string::load("twitter.json");
+    doc = parser.iterate(json_string);
+    uint64_t identifier = doc["statuses"].at(0)["id"];
+    std::cout << identifier << std::endl;
+  } catch (simdjson::simdjson_error &error) {
+    std::cerr << "JSON error: " << error.what() << " near "
+              << doc.current_location() << " in " << json_string << std::endl;
+  }
+  return EXIT_SUCCESS;
+}
 #endif
 bool test_load_example() {
  TEST_START();
@@ -1235,6 +1298,7 @@ bool example1958() {
 bool run() {
  return true
 #if SIMDJSON_EXCEPTIONS
+    && gen_raw1() && gen_raw2() && gen_raw3()
    && at_end()
    && example1956() && example1958()
 //    && basics_1() // Fails because twitter.json isn't in current directory. Compile test only.
@@ -1,5 +1,4 @@
 #include <cinttypes>
-#include <ciso646>
 #include <cmath>
 #include <cstdio>
 #include <cstdlib>
@@ -80,15 +80,6 @@ else :

 atleastminor= (currentv[0] != newversion[0]) or (currentv[1] != newversion[1])

-if(atleastminor):
-    print(colored(0, 255, 0, "This is more than a revision."))
-    releasefile = maindir + os.sep + "RELEASES.md"
-    releasedata = open(releasefile).read()
-    pattern = re.compile("#\s+\d+\.\d+")
-    m = pattern.search(releasedata)
-    if(m == None):
-        print(colored(255, 0, 0, "You are preparing a new minor release and you have not yet updated RELEASES.md."))
-        sys.exit(-1)

 versionfilerel = os.sep + "include" + os.sep + "simdjson" + os.sep + "simdjson_version.h"
 versionfile = maindir + versionfilerel
Author	SHA1	Message	Date
Daniel Lemire	92174c927d	New release	2023-06-15 09:21:07 -04:00
Ng Yong Xiang	b399c0165e	Add comma separated value parsing as an option in iterate_many (#2016 ) * Add comma separated value parsing * Fix failing tests * Make tests work for exceptions * Fix test * Fix try catch making test fail --------- Co-authored-by: Yong Xiang Ng <yxng@drwholdings.com>	2023-06-15 09:11:14 -04:00
Daniel Lemire	445ae024ba	fix: bug identified by Yagiz Nizipli (#2019 )	2023-06-15 09:10:13 -04:00
Daniel Lemire	65f9822542	Adding tests for issue 2017 (#2018 )	2023-06-07 09:57:28 -04:00
Daniel Lemire	74bb7b2533	Adding popcnt	2023-06-05 09:04:36 -04:00
Daniel Lemire	fd97288561	Adding popcnt.	2023-06-02 20:33:15 -04:00
Daniel Lemire	1da0d8ac65	Making sure popcnt is included	2023-06-02 20:32:54 -04:00
Daniel Lemire	85661a2901	Some systems release without NDEBUG set, let us not assert in these cases. (#2014 )	2023-06-01 16:18:36 -04:00
Daniel Lemire	3fcdb65d88	Update README.md	2023-05-26 09:26:17 -04:00
Daniel Lemire	8b00d91a2c	More documentation. (#2011 )	2023-05-26 09:23:21 -04:00
Daniel Lemire	7db3f97ba8	Adding CXX 20 to CI (#2005 ) * Adding CXX 20 to CI * side-stepping new CXX 20 guard. * Going another way * Saving. * Explicit. * Saving... --------- Co-authored-by: Daniel Lemire <dlemire@lemire.me>	2023-05-26 09:23:02 -04:00
Daniel Lemire	412a8f7c4d	Removing iso header. (#2009 ) * Removing iso header. * Fix. * Another one. * Caught another one. --------- Co-authored-by: Daniel Lemire <dlemire@lemire.me>	2023-05-23 16:19:21 -04:00
Daniel Lemire	d4b31c9feb	Update README.md	2023-05-23 16:16:26 -04:00
Daniel Lemire	ecdcf276b2	Update README.md	2023-05-23 16:15:13 -04:00
Daniel Lemire	6d92563280	Fixing issue 1943 (#2006 ) * Fixing issue 1943 * Tweaking.	2023-05-23 11:15:22 -04:00
Daniel Lemire	0998233d58	More documentation. (#2007 )	2023-05-23 11:15:07 -04:00
Daniel Lemire	cb735818ec	Document key queries. (#2008 )	2023-05-23 11:14:55 -04:00
Daniel Lemire	41ee548e3f	Adding godbolt examples. (#2010 ) * Adding godbolt examples. * Updating the examples.	2023-05-23 11:14:41 -04:00
Ng Yong Xiang	e9d5d5088a	Introduce logging levels for conditional logging and log key for NO_SUCH_FIELD error (#2004 ) * Add info and error logging * Add tests for error logging * Add logging for missing field * Update docs for logging usage * Fix style and pass by ref for string format args * Make log_level explicit and simplify get log level from env * Make log level int32_t * Format enum class * Fix ci * Move enum to header * Fix compilation for noexception build in test case * Disable warnings and putenv --------- Co-authored-by: Yong Xiang Ng <yxng@drwholdings.com>	2023-05-22 12:58:55 -04:00
Daniel Lemire	78151e80e5	Update README.md	2023-05-16 12:49:49 -04:00
Daniel Lemire	083a5d7351	Update README.md	2023-05-15 15:03:11 -04:00