Release

* Undoing PR 2004 due to performance concerns.

Fixes https://github.com/simdjson/simdjson/issues/2028
Fixes https://github.com/simdjson/simdjson/issues/2021

* Adding final lines.

---------

Co-authored-by: Daniel Lemire <dlemire@lemire.me>

.clangd

@@ -0,0 +1,2 @@
+CompileFlags:
+    CompilationDatabase: build

.github/workflows/fuzzers.yml

@@ -124,7 +124,7 @@ jobs:
         done
 
     - name: Save the corpus as a github artifact
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v3
       with:
         name: corpus
         path: corpus.tar
@@ -147,7 +147,7 @@ jobs:
       run: tar cf valgrind.tar valgrind-*.txt
 
     - name: Save valgrind output as a github artifact
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v3
       if: always()
       with:
         name: valgrindresults
@@ -155,7 +155,7 @@ jobs:
         if-no-files-found: ignore
 
     - name: Archive any crashes as an artifact
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v3
       if: always()
       with:
         name: crashes

.github/workflows/power-fuzz.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-20.04
     name: Build on ubuntu-20.04 ppc64le
     steps:
-      - uses: actions/checkout@v2.1.0
+      - uses: actions/checkout@v3
       - uses: uraimo/run-on-arch-action@v2.0.5
         name: Run commands
         id: runcmd

.github/workflows/ubuntu22-cxx20.yml

@@ -9,8 +9,8 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/cache@v2
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
         with:
           path: dependencies/.cache
           key: ${{ hashFiles('dependencies/CMakeLists.txt') }}

.github/workflows/vs17-ci-cxx20.yml

@@ -19,7 +19,7 @@ jobs:
           - {gen: Visual Studio 17 2022, arch: x64, shared: OFF}
     steps:
     - name: checkout
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
     - name: Configure
       run: |
         cmake -DSIMDJSON_CXX_STANDARD=20 -G "${{matrix.gen}}" -A ${{matrix.arch}} -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -DBUILD_SHARED_LIBS=${{matrix.shared}} -B build

CMakeLists.txt

@@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.14)
 project(
     simdjson
     # The version number is modified by tools/release.py
-    VERSION 3.2.0
+    VERSION 3.2.1
     DESCRIPTION "Parsing gigabytes of JSON per second"
     HOMEPAGE_URL "https://simdjson.org/"
     LANGUAGES CXX C

Doxyfile

@@ -38,7 +38,7 @@ PROJECT_NAME           = simdjson
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = "3.2.0"
+PROJECT_NUMBER         = "3.2.1"
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a

HACKING.md

@@ -6,16 +6,26 @@ Here is wisdom about how to build, test and run simdjson from within the reposit
 
 If you plan to contribute to simdjson, please read our [CONTRIBUTING](https://github.com/simdjson/simdjson/blob/master/CONTRIBUTING.md) guide.
 
-
+- [Hacking simdjson](#hacking-simdjson)
+  - [Build Quickstart](#build-quickstart)
   - [Design notes](#design-notes)
   - [Developer mode](#developer-mode)
   - [Directory Structure and Source](#directory-structure-and-source)
   - [Runtime Dispatching](#runtime-dispatching)
   - [Regenerating Single-Header Files](#regenerating-single-header-files)
   - [Usage (CMake on 64-bit platforms like Linux, FreeBSD or macOS)](#usage-cmake-on-64-bit-platforms-like-linux-freebsd-or-macos)
-  - [Usage (CMake on 64-bit Windows using Visual Studio 2019)](#usage-cmake-on-64-bit-windows-using-visual-studio-2019)
+  - [Usage (CMake on 64-bit Windows using Visual Studio 2019 or better)](#usage-cmake-on-64-bit-windows-using-visual-studio-2019-or-better)
   - [Various References](#various-references)
 
+Build Quickstart
+------------------------------
+
+```bash
+mkdir build
+cd build
+cmake -D SIMDJSON_DEVELOPER_MODE=ON ..
+cmake --build .
+```
 
 Design notes
 ------------------------------

README.md

@@ -57,6 +57,7 @@ Real-world usage
 - [fastgltf](https://github.com/spnda/fastgltf)
 - [vast](https://github.com/tenzir/vast)
 - [ada-url](https://github.com/ada-url/ada)
+- [fastgron](https://github.com/adamritter/fastgron)
 
 If you are planning to use simdjson in a product, please work from one of our releases.
 

benchmark/json2msgpack/simdjson_dom.h

@@ -15,7 +15,7 @@ struct simdjsondom2msgpack {
    * @param json JSON input
    * @param buf temporary buffer (must be large enough, with simdjson::SIMDJSON_PADDING bytes
    * of padding)
-   * @return std::string_view msgpack output, writting to the temporary buffer
+   * @return std::string_view msgpack output, writing to the temporary buffer
    */
   inline std::string_view to_msgpack(const simdjson::padded_string &json,
                                      uint8_t *buf);

benchmark/json2msgpack/simdjson_ondemand.h

@@ -36,7 +36,7 @@ struct simdjson2msgpack {
    * @param json JSON input
    * @param buf temporary buffer (must be large enough, with simdjson::SIMDJSON_PADDING bytes
    * of padding)
-   * @return std::string_view msgpack output, writting to the temporary buffer
+   * @return std::string_view msgpack output, writing to the temporary buffer
    */
   inline std::string_view to_msgpack(const simdjson::padded_string &json,
                                      uint8_t *buf);

doc/basics.md

@@ -335,8 +335,8 @@ support for users who avoid exceptions. See [the simdjson error handling documen
   `double(element)`. This works for `std::string_view`, double, uint64_t, int64_t, bool,
   ondemand::object and ondemand::array. We also have explicit methods such as `get_string()`, `get_double()`,
   `get_uint64()`, `get_int64()`, `get_bool()`, `get_object()` and `get_array()`. After a cast or an explicit method,
-  the number, string or boolean will be parsed, or the initial `{` or `[` will be verified for ondemand::object and ondemand::array. An exception is thrown if
-  the cast is not possible. Importantly, when getting an ondemand::object or ondemand::array instance, its content is
+  the number, string or boolean will be parsed, or the initial `{` or `[` will be verified for `ondemand::object` and `ondemand::array`. An exception may be thrown if
+  the cast is not possible: there error code is `simdjson::INCORRECT_TYPE` (see [Error Handling](#error-handling)). Importantly, when getting an ondemand::object or ondemand::array instance, its content is
   not validated: you are only guaranteed that the corresponding initial character (`{` or `[`) is present. Thus,
   for example, you could have an ondemand::object instance pointing at the invalid JSON `{ "this is not a valid object" }`: the validation occurs as you access the content.
   The `get_string()` returns a valid UTF-8 string, after
@@ -354,8 +354,8 @@ support for users who avoid exceptions. See [the simdjson error handling documen
   > `get_string()` twice on an object (or to cast an object twice to `std::string_view`).
 * **Field Access:** To get the value of the "foo" field in an object, use `object["foo"]`. This will
   scan through the object looking for the field with the matching string, doing a character-by-character
-  comparison. For efficiency reason, you should avoid looking up the same field repeatedly: e.g., do
-  not do `object["foo"]` followed by `object["foo"]` with the same `object` instance. Keep in mind that On Demand does not buffer or save the result of the parsing: if you repeatedly access `object["foo"]`, then it must repeatedly seek the key and parse the content. The library does not provide a distinct function to check if a key is present, instead we recommend you attempt to access the key: e.g., by doing `ondemand::value val{}; if(!object["foo"].get(val)) {...}`, you have that `val` contains the requested value inside the if clause.  It is your responsability as a user to temporarily keep a reference to the value (`auto v = object["foo"]`), or to consume the content and store it in your own data structures. If you consume an
+  comparison. It may generate the error `simdjson::NO_SUCH_FIELD` if there is no such key in the object, it may throw an exception (see [Error Handling](#error-handling)). For efficiency reason, you should avoid looking up the same field repeatedly: e.g., do
+  not do `object["foo"]` followed by `object["foo"]` with the same `object` instance. Keep in mind that On Demand does not buffer or save the result of the parsing: if you repeatedly access `object["foo"]`, then it must repeatedly seek the key and parse the content. The library does not provide a distinct function to check if a key is present, instead we recommend you attempt to access the key: e.g., by doing `ondemand::value val{}; if(!object["foo"].get(val)) {...}`, you have that `val` contains the requested value inside the if clause.  It is your responsibility as a user to temporarily keep a reference to the value (`auto v = object["foo"]`), or to consume the content and store it in your own data structures. If you consume an
   object twice: `std::string_view(object["foo"]` followed by `std::string_view(object["foo"]` then your code
   is in error. Furthermore, you can only consume one field at a time, on the same object. The
   value instance you get from  `content["bids"]` becomes invalid when you call `content["asks"]`.
@@ -887,10 +887,17 @@ auto error = parser.iterate(json).get(doc);
 if (error) { cerr << error << endl; exit(1); }
 ```
 
-When there is no error, the error code simdjson::SUCCESS is returned: it evaluates as false as a Boolean.
+When there is no error, the error code `simdjson::SUCCESS`is returned: it evaluates as false as a Boolean.
 We have several error codes to indicate errors, they all evaluate to true as a Boolean: your software should not generally not depend on exact
 error codes. We may change the error codes in future releases and the exact error codes could vary depending on your system.
 
+Some errors are recoverable:
+
+* You may get the error `simdjson::INCORRECT_TYPE` after trying to convert a value to an incorrect type: e.g., you expected a number and try to convert the value to a number, but it is an array.
+* You may query a key from an object, but the key is missing in which case you get the error `simdjson::NO_SUCH_FIELD`: e.g., you call `obj["myname"]` and the object does not have a key `"myname"`.
+
+Other errors (e.g., `simdjson::INCOMPLETE_ARRAY_OR_OBJECT`) may indicate a fatal error and often follow from the fact that the document is not valid JSON. In which case, it is no longer possible to continue accessing the document: calling the method `is_alive()` on the document instance returns false. All following accesses will keep returning the same fatal error (e.g., `simdjson::INCOMPLETE_ARRAY_OR_OBJECT`).
+
 When you use the code without exceptions, it is your responsibility to check for error before using the
 result: if there is an error, the result value will not be valid and using it will caused undefined behavior. Most compilers should be able to help you if you activate the right
 set of warnings: they can identify variables that are written to but never otherwise accessed.
@@ -1422,10 +1429,9 @@ This parser can't support a document that big
 
 If your documents are large (e.g., larger than a megabyte), then the `iterate_many` function is maybe ill-suited. It is really meant to support reading efficiently streams of relatively small documents (e.g., a few kilobytes each). If you have larger documents, you should use other functions like `iterate`.
 
+We also provide some support for comma-separated documents and other advanced features.
 See [iterate_many.md](iterate_many.md) for detailed information and design.
 
-
-
 Parsing Numbers Inside Strings
 ------------------------------
 
@@ -1948,6 +1954,8 @@ Performance Tips
 - Given a field `field` in an object, calling `field.key()` is often faster than `field.unescaped_key()` so if you do not need an unescaped `std::string_view` instance, prefer `field.key()`.
 - For release builds, we recommend setting `NDEBUG` pre-processor directive when compiling the `simdjson` library. Importantly, using the optimization flags `-O2` or `-O3` under GCC and LLVM clang does not set the `NDEBUG` directive, you must set it manually (e.g., `-DNDEBUG`).
 - For long streams of JSON documents, consider [`iterate_many`](iterate_many.md) and [`parse_many`](parse_many.md) for better performance.
+- Never seek to access a field twice (e.g., o["data"] and later again o["data"]). Instead capture once an ondemand::value and reuse it.
+- If you must access several different keys in an object, it might be preferable to iterate through all the fields in the object instead, and branch on the field keys.
 - If possible, refer to each object and array in your code once. For example, the following code repeatedly refers to the `"data"` key to create an object...
 	```C++
 	std::string_view make = o["data"]["make"];
@@ -1961,4 +1969,4 @@ Performance Tips
 	std::string_view year = data["year"];
 	std::string_view rating = data["rating"];
   ```
-- To better understand the operation of your On Demand parser, and whether it is performing as well as you think it should be, there is a  logger feature built in to simdjson! To use it, define the pre-processor directive `SIMDJSON_VERBOSE_LOGGING` prior to including the `simdjson.h` header, which enables logging in simdjson. Run your code. It may generate a lot of logging output; adding printouts from your application that show each section may be helpful. The log’s output will show step-by-step information on state, buffer pointer position, depth, and key retrieval status. The log level is set to `INFO` by default, but can be modified with the environment variable `SIMDJSON_LOG_LEVEL`. Setting the environment variable `SIMDJSON_LOG_LEVEL` to `ERROR` will log only errors.
+- To better understand the operation of your On Demand parser, and whether it is performing as well as you think it should be, there is a  logger feature built in to simdjson! To use it, define the pre-processor directive `SIMDJSON_VERBOSE_LOGGING` prior to including the `simdjson.h` header, which enables logging in simdjson. Run your code. It may generate a lot of logging output; adding printouts from your application that show each section may be helpful. The log's output will show step-by-step information on state, buffer pointer position, depth, and key retrieval status. Importantly, unless `SIMDJSON_VERBOSE_LOGGING` is defined, logging is entirely disabled and thus carries no overhead.

doc/iterate_many.md

@@ -129,6 +129,18 @@ Some official formats **(non-exhaustive list)**:
 API
 ---
 
+Example:
+
+```c++
+auto json = R"({ "foo": 1 } { "foo": 2 } { "foo": 3 } )"_padded;
+ondemand::parser parser;
+ondemand::document_stream docs = parser.iterate_many(json);
+for (auto doc : docs) {
+  std::cout << doc["foo"] << std::endl;
+}
+// Prints 1 2 3
+```
+
 See [basics.md](basics.md#newline-delimited-json-ndjson-and-json-lines) for an overview of the API.
 
 ## Use cases
@@ -238,7 +250,38 @@ This will print:
 
 Importantly, you should only call `truncated_bytes()` after iterating through all of the documents since the stream cannot tell whether there are truncated documents at the very end when it may not have accessed that part of the data yet.
 
-Comma separated documents
+Comma-separated documents
 -----------
 
-`iterate_many` also takes in an option to allow parsing of comma separated documents. In this mode, the entire buffer is processed in 1 batch and batch size will be increased to be as large as the JSON passed. Therefore, the capacity of the parser has to be sufficient to support the batch size set.
+We also support comma-separated documents, but with some performance limitations. The `iterate_many` function  takes in an option to allow parsing of comma separated documents (which defaults on false). In this mode, the entire buffer is processed in one batch. Therefore, the total size of the document should not exceed the maximal capacity of the parser (4 GB). This mode also effectively disallow multithreading. It is therefore mostly suitable for not "very large" inputs. In this mode, the batch_size parameter
+is effectively ignored, as it is set to at least the document size.
+
+Example:
+
+```C++
+    auto json = R"( 1, 2, 3, 4, "a", "b", "c", {"hello": "world"} , [1, 2, 3])"_padded;
+    ondemand::parser parser;
+    ondemand::document_stream doc_stream;
+    // We pass '32' as the batch size, but it is a bogus parameter because, since
+    // we pass 'true' to the allow_comma parameter, the batch size will be set to at least
+    // the document size.
+    auto error = parser.iterate_many(json, 32, true).get(doc_stream);
+    if(error) { std::cerr << error << std::endl; return; }
+    for (auto doc : doc_stream) {
+        std::cout << doc.type() << std::endl;
+    }
+ ```
+
+ This will print:
+
+```
+number
+number
+number
+number
+string
+string
+string
+object
+array
+```

include/simdjson/arm64/numberparsing.h

@@ -3,10 +3,11 @@
 
 namespace simdjson {
 namespace SIMDJSON_IMPLEMENTATION {
-namespace {
+namespace numberparsing {
 
 // we don't have SSE, so let us use a scalar function
 // credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/
+/** @private */
 static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) {
   uint64_t val;
   std::memcpy(&val, chars, sizeof(uint64_t));
@@ -15,7 +16,7 @@ static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars
   return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32);
 }
 
-} // unnamed namespace
+} // namespace numberparsing
 } // namespace SIMDJSON_IMPLEMENTATION
 } // namespace simdjson
 

include/simdjson/fallback/numberparsing.h

@@ -10,8 +10,9 @@ void found_float(double result, const uint8_t *buf);
 
 namespace simdjson {
 namespace SIMDJSON_IMPLEMENTATION {
-namespace {
+namespace numberparsing {
 // credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/
+/** @private */
 static simdjson_inline uint32_t parse_eight_digits_unrolled(const char *chars) {
   uint64_t val;
   memcpy(&val, chars, sizeof(uint64_t));
@@ -19,11 +20,12 @@ static simdjson_inline uint32_t parse_eight_digits_unrolled(const char *chars) {
   val = (val & 0x00FF00FF00FF00FF) * 6553601 >> 16;
   return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32);
 }
+/** @private */
 static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) {
   return parse_eight_digits_unrolled(reinterpret_cast<const char *>(chars));
 }
 
-} // unnamed namespace
+} // namespace numberparsing
 } // namespace SIMDJSON_IMPLEMENTATION
 } // namespace simdjson
 

include/simdjson/generic/numberparsing.h

@@ -3,8 +3,9 @@
 
 namespace simdjson {
 namespace SIMDJSON_IMPLEMENTATION {
+/// @private
+namespace numberparsing {
 
-namespace ondemand {
 /**
  * The type of a JSON number
  */
@@ -13,13 +14,6 @@ enum class number_type {
     signed_integer,          /// a signed integer that fits in a 64-bit word using two's complement
     unsigned_integer         /// a positive integer larger or equal to 1<<63
 };
-}
-
-namespace {
-/// @private
-namespace numberparsing {
-
-
 
 #ifdef JSON_TEST_NUMBERS
 #define INVALID_NUMBER(SRC) (found_invalid_number((SRC)), NUMBER_ERROR)
@@ -34,6 +28,7 @@ namespace numberparsing {
 #endif
 
 namespace {
+
 // Convert a mantissa, an exponent and a sign bit into an ieee64 double.
 // The real_exponent needs to be in [0, 2046] (technically real_exponent = 2047 would be acceptable).
 // The mantissa should be in [0,1<<53). The bit at index (1ULL << 52) while be zeroed.
@@ -45,7 +40,7 @@ simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool
     std::memcpy(&d, &mantissa, sizeof(d));
     return d;
 }
-}
+
 // Attempts to compute i * 10^(power) exactly; and if "negative" is
 // true, negate the result.
 // This function will only work in some cases, when it does not work, success is
@@ -62,10 +57,11 @@ simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative,
 #endif
 #if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0)
   // We cannot be certain that x/y is rounded to nearest.
-  if (0 <= power && power <= 22 && i <= 9007199254740991) {
+  if (0 <= power && power <= 22 && i <= 9007199254740991)
 #else
-  if (-22 <= power && power <= 22 && i <= 9007199254740991) {
+  if (-22 <= power && power <= 22 && i <= 9007199254740991)
 #endif
+  {
     // convert the integer into a double. This is lossless since
     // 0 <= i <= 2^53 - 1.
     d = double(i);
@@ -317,6 +313,7 @@ static bool parse_float_fallback(const uint8_t *ptr, double *outDouble) {
   // to handle that max may be a macro on windows).
   return !(*outDouble > (std::numeric_limits<double>::max)() || *outDouble < std::numeric_limits<double>::lowest());
 }
+
 static bool parse_float_fallback(const uint8_t *ptr, const uint8_t *end_ptr, double *outDouble) {
   *outDouble = simdjson::internal::from_chars(reinterpret_cast<const char *>(ptr), reinterpret_cast<const char *>(end_ptr));
   // We do not accept infinite values.
@@ -350,16 +347,6 @@ simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) {
           0x3333333333333333);
 }
 
-template<typename W>
-error_code slow_float_parsing(simdjson_unused const uint8_t * src, W writer) {
-  double d;
-  if (parse_float_fallback(src, &d)) {
-    writer.append_double(d);
-    return SUCCESS;
-  }
-  return INVALID_NUMBER(src);
-}
-
 template<typename I>
 SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later
 simdjson_inline bool parse_digit(const uint8_t c, I &i) {
@@ -372,7 +359,7 @@ simdjson_inline bool parse_digit(const uint8_t c, I &i) {
   return true;
 }
 
-simdjson_inline error_code parse_decimal(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) {
+simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) {
   // we continue with the fiction that we have an integer. If the
   // floating point number is representable as x * 10^z for some integer
   // z that fits in 53 bits, then we will be able to convert back the
@@ -460,6 +447,20 @@ simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t d
   return digit_count - size_t(start - start_digits);
 }
 
+} // unnamed namespace
+
+/** @private */
+template<typename W>
+error_code slow_float_parsing(simdjson_unused const uint8_t * src, W writer) {
+  double d;
+  if (parse_float_fallback(src, &d)) {
+    writer.append_double(d);
+    return SUCCESS;
+  }
+  return INVALID_NUMBER(src);
+}
+
+/** @private */
 template<typename W>
 simdjson_inline error_code write_float(const uint8_t *const src, bool negative, uint64_t i, const uint8_t * start_digits, size_t digit_count, int64_t exponent, W &writer) {
   // If we frequently had to deal with long strings of digits,
@@ -531,7 +532,7 @@ simdjson_unused simdjson_inline simdjson_result<int64_t> parse_integer_in_string
 simdjson_unused simdjson_inline simdjson_result<double> parse_double_in_string(const uint8_t * const src) noexcept { return 0; }
 simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept  { return false; }
 simdjson_unused simdjson_inline simdjson_result<bool> is_integer(const uint8_t * src) noexcept  { return false; }
-simdjson_unused simdjson_inline simdjson_result<ondemand::number_type> get_number_type(const uint8_t * src) noexcept { return ondemand::number_type::signed_integer; }
+simdjson_unused simdjson_inline simdjson_result<number_type> get_number_type(const uint8_t * src) noexcept { return number_type::signed_integer; }
 #else
 
 // parse the number at src
@@ -573,7 +574,7 @@ simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) {
   if ('.' == *p) {
     is_float = true;
     ++p;
-    SIMDJSON_TRY( parse_decimal(src, p, i, exponent) );
+    SIMDJSON_TRY( parse_decimal_after_separator(src, p, i, exponent) );
     digit_count = int(p - start_digits); // used later to guard against overflows
   }
   if (('e' == *p) || ('E' == *p)) {
@@ -1061,7 +1062,7 @@ simdjson_unused simdjson_inline simdjson_result<bool> is_integer(const uint8_t *
   return false;
 }
 
-simdjson_unused simdjson_inline simdjson_result<ondemand::number_type> get_number_type(const uint8_t * src) noexcept {
+simdjson_unused simdjson_inline simdjson_result<number_type> get_number_type(const uint8_t * src) noexcept {
   bool negative = (*src == '-');
   src += uint8_t(negative);
   const uint8_t *p = src;
@@ -1070,20 +1071,20 @@ simdjson_unused simdjson_inline simdjson_result<ondemand::number_type> get_numbe
   if (jsoncharutils::is_structural_or_whitespace(*p)) {
     // We have an integer.
     // If the number is negative and valid, it must be a signed integer.
-    if(negative) { return ondemand::number_type::signed_integer; }
+    if(negative) { return number_type::signed_integer; }
     // We want values larger or equal to 9223372036854775808 to be unsigned
     // integers, and the other values to be signed integers.
     int digit_count = int(p - src);
     if(digit_count >= 19) {
       const uint8_t * smaller_big_integer = reinterpret_cast<const uint8_t *>("9223372036854775808");
       if((digit_count >= 20) || (memcmp(src, smaller_big_integer, 19) >= 0)) {
-        return ondemand::number_type::unsigned_integer;
+        return number_type::unsigned_integer;
       }
     }
-    return ondemand::number_type::signed_integer;
+    return number_type::signed_integer;
   }
   // Hopefully, we have 'e' or 'E' or '.'.
-  return ondemand::number_type::floating_point_number;
+  return number_type::floating_point_number;
 }
 
 // Never read at src_end or beyond
@@ -1245,10 +1246,20 @@ simdjson_unused simdjson_inline simdjson_result<double> parse_double_in_string(c
   }
   return d;
 }
-} //namespace {}
+
+} // unnamed namespace
 #endif // SIMDJSON_SKIPNUMBERPARSING
 
+inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept {
+    switch (type) {
+        case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break;
+        case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break;
+        case number_type::floating_point_number: out << "floating-point number (binary64)"; break;
+        default: SIMDJSON_UNREACHABLE();
+    }
+    return out;
+}
+
 } // namespace numberparsing
-} // unnamed namespace
 } // namespace SIMDJSON_IMPLEMENTATION
 } // namespace simdjson

include/simdjson/generic/ondemand.h

@@ -10,6 +10,9 @@ namespace ondemand {
 /** Represents the depth of a JSON value (number of nested arrays/objects). */
 using depth_t = int32_t;
 
+/** @copydoc simdjson::SIMDJSON_IMPLEMENTATION::numberparsing::number_type */
+using number_type = simdjson::SIMDJSON_IMPLEMENTATION::numberparsing::number_type;
+
 } // namespace ondemand
 } // namespace SIMDJSON_IMPLEMENTATION
 } // namespace simdjson

include/simdjson/generic/ondemand/document-inl.h

@@ -456,7 +456,7 @@ simdjson_inline simdjson_result<bool> simdjson_result<SIMDJSON_IMPLEMENTATION::o
   return first.is_integer();
 }
 
-simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number_type> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::get_number_type() noexcept {
+simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::numberparsing::number_type> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::get_number_type() noexcept {
   if (error()) { return error(); }
   return first.get_number_type();
 }
@@ -739,7 +739,7 @@ simdjson_inline simdjson_result<bool> simdjson_result<SIMDJSON_IMPLEMENTATION::o
   if (error()) { return error(); }
   return first.is_integer();
 }
-simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number_type> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::get_number_type() noexcept {
+simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::numberparsing::number_type> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::get_number_type() noexcept {
   if (error()) { return error(); }
   return first.get_number_type();
 }

include/simdjson/generic/ondemand/document.h

@@ -730,7 +730,7 @@ public:
   simdjson_inline bool at_end() const noexcept;
   simdjson_inline bool is_negative() noexcept;
   simdjson_inline simdjson_result<bool> is_integer() noexcept;
-  simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number_type> get_number_type() noexcept;
+  simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::numberparsing::number_type> get_number_type() noexcept;
   simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number> get_number() noexcept;
   /** @copydoc simdjson_inline std::string_view document::raw_json_token() const noexcept */
   simdjson_inline simdjson_result<std::string_view> raw_json_token() noexcept;
@@ -795,7 +795,7 @@ public:
   simdjson_inline simdjson_result<int32_t> current_depth() const noexcept;
   simdjson_inline simdjson_result<bool> is_negative() noexcept;
   simdjson_inline simdjson_result<bool> is_integer() noexcept;
-  simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number_type> get_number_type() noexcept;
+  simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::numberparsing::number_type> get_number_type() noexcept;
   simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number> get_number() noexcept;
   /** @copydoc simdjson_inline std::string_view document_reference::raw_json_token() const noexcept */
   simdjson_inline simdjson_result<std::string_view> raw_json_token() noexcept;

include/simdjson/generic/ondemand/json_iterator.h

@@ -298,8 +298,8 @@ protected:
   friend class raw_json_string;
   friend class parser;
   friend class value_iterator;
-  friend simdjson_inline void logger::log_line(const json_iterator &iter, const char *title_prefix, const char *title, std::string_view detail, int delta, int depth_delta, logger::log_level level) noexcept;
-  friend simdjson_inline void logger::log_line(const json_iterator &iter, token_position index, depth_t depth, const char *title_prefix, const char *title, std::string_view detail, logger::log_level level) noexcept;
+  friend simdjson_inline void logger::log_line(const json_iterator &iter, const char *title_prefix, const char *title, std::string_view detail, int delta, int depth_delta) noexcept;
+  friend simdjson_inline void logger::log_line(const json_iterator &iter, token_position index, depth_t depth, const char *title_prefix, const char *title, std::string_view detail) noexcept;
 }; // json_iterator
 
 } // namespace ondemand

include/simdjson/generic/ondemand/json_type-inl.h

@@ -15,15 +15,6 @@ inline std::ostream& operator<<(std::ostream& out, json_type type) noexcept {
     return out;
 }
 
-inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept {
-    switch (type) {
-        case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break;
-        case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break;
-        case number_type::floating_point_number: out << "floating-point number (binary64)"; break;
-        default: SIMDJSON_UNREACHABLE();
-    }
-    return out;
-}
 #if SIMDJSON_EXCEPTIONS
 inline std::ostream& operator<<(std::ostream& out, simdjson_result<json_type> &type) noexcept(false) {
     return out << type.value();

include/simdjson/generic/ondemand/json_type.h

@@ -84,11 +84,11 @@ protected:
    */
   friend class value_iterator;
   template<typename W>
+  friend error_code numberparsing::slow_float_parsing(simdjson_unused const uint8_t * src, W writer);
+  template<typename W>
   friend error_code numberparsing::write_float(const uint8_t *const src, bool negative, uint64_t i, const uint8_t * start_digits, size_t digit_count, int64_t exponent, W &writer);
   template<typename W>
   friend error_code numberparsing::parse_number(const uint8_t *const src, W &writer);
-  template<typename W>
-  friend error_code numberparsing::slow_float_parsing(simdjson_unused const uint8_t * src, W writer);
   /** Store a signed 64-bit value to the number. */
   simdjson_inline void append_s64(int64_t value) noexcept;
   /** Store an unsigned 64-bit value to the number. */
@@ -120,7 +120,6 @@ protected:
  * @param type The json_type.
  */
 inline std::ostream& operator<<(std::ostream& out, json_type type) noexcept;
-inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept;
 
 #if SIMDJSON_EXCEPTIONS
 /**

include/simdjson/generic/ondemand/logger-inl.h

@@ -1,4 +1,3 @@
-#include <memory>
 namespace simdjson {
 namespace SIMDJSON_IMPLEMENTATION {
 namespace ondemand {
@@ -19,68 +18,36 @@ static inline char printable_char(char c) {
   }
 }
 
-static inline log_level get_log_level_from_env()
-{
-  SIMDJSON_PUSH_DISABLE_WARNINGS
-  SIMDJSON_DISABLE_DEPRECATED_WARNING // Disable CRT_SECURE warning on MSVC: manually verified this is safe
-  char *lvl = getenv("SIMDJSON_LOG_LEVEL");
-  SIMDJSON_POP_DISABLE_WARNINGS
-  if (lvl && simdjson_strcasecmp(lvl, "ERROR") == 0) { return log_level::LOG_ERROR; }
-  return log_level::LOG_INFO;
-}
-
-static inline log_level log_threshold()
-{
-  static log_level threshold = get_log_level_from_env();
-  return threshold;
-}
-
-static inline bool should_log(log_level level)
-{
-  return level >= log_threshold();
-}
-
-template<typename... Args>
-inline std::string string_format(const std::string& format, const Args&... args)
-{
-  int size_s = std::snprintf(nullptr, 0, format.c_str(), args...) + 1;
-  auto size = static_cast<size_t>(size_s);
-  if (size <= 0) return std::string();
-  std::unique_ptr<char[]> buf(new char[size]);
-  std::snprintf(buf.get(), size, format.c_str(), args...);
-  return std::string(buf.get(), buf.get() + size - 1);
-}
-
 inline void log_event(const json_iterator &iter, const char *type, std::string_view detail, int delta, int depth_delta) noexcept {
-  log_line(iter, "", type, detail, delta, depth_delta, log_level::LOG_INFO);
+  log_line(iter, "", type, detail, delta, depth_delta);
 }
 
 inline void log_value(const json_iterator &iter, token_position index, depth_t depth, const char *type, std::string_view detail) noexcept {
-  log_line(iter, index, depth, "", type, detail, log_level::LOG_INFO);
+  log_line(iter, index, depth, "", type, detail);
 }
 inline void log_value(const json_iterator &iter, const char *type, std::string_view detail, int delta, int depth_delta) noexcept {
-  log_line(iter, "", type, detail, delta, depth_delta, log_level::LOG_INFO);
+  log_line(iter, "", type, detail, delta, depth_delta);
 }
 
 inline void log_start_value(const json_iterator &iter, token_position index, depth_t depth, const char *type, std::string_view detail) noexcept {
-  log_line(iter, index, depth, "+", type, detail, log_level::LOG_INFO);
+  log_line(iter, index, depth, "+", type, detail);
   if (LOG_ENABLED) { log_depth++; }
 }
 inline void log_start_value(const json_iterator &iter, const char *type, int delta, int depth_delta) noexcept {
-  log_line(iter, "+", type, "", delta, depth_delta, log_level::LOG_INFO);
+  log_line(iter, "+", type, "", delta, depth_delta);
   if (LOG_ENABLED) { log_depth++; }
 }
 
 inline void log_end_value(const json_iterator &iter, const char *type, int delta, int depth_delta) noexcept {
   if (LOG_ENABLED) { log_depth--; }
-  log_line(iter, "-", type, "", delta, depth_delta, log_level::LOG_INFO);
+  log_line(iter, "-", type, "", delta, depth_delta);
 }
 
 inline void log_error(const json_iterator &iter, const char *error, const char *detail, int delta, int depth_delta) noexcept {
-  log_line(iter, "ERROR: ", error, detail, delta, depth_delta, log_level::LOG_ERROR);
+  log_line(iter, "ERROR: ", error, detail, delta, depth_delta);
 }
 inline void log_error(const json_iterator &iter, token_position index, depth_t depth, const char *error, const char *detail) noexcept {
-  log_line(iter, index, depth, "ERROR: ", error, detail, log_level::LOG_ERROR);
+  log_line(iter, index, depth, "ERROR: ", error, detail);
 }
 
 inline void log_event(const value_iterator &iter, const char *type, std::string_view detail, int delta, int depth_delta) noexcept {
@@ -104,7 +71,7 @@ inline void log_error(const value_iterator &iter, const char *error, const char
 }
 
 inline void log_headers() noexcept {
-  if (LOG_ENABLED && simdjson_unlikely(should_log(log_level::LOG_INFO))) {
+  if (LOG_ENABLED) {
     // Technically a static variable is not thread-safe, but if you are using threads
     // and logging... well...
     static bool displayed_hint{false};
@@ -154,11 +121,11 @@ inline void log_headers() noexcept {
   }
 }
 
-inline void log_line(const json_iterator &iter, const char *title_prefix, const char *title, std::string_view detail, int delta, int depth_delta, log_level level) noexcept {
-  log_line(iter, iter.position()+delta, depth_t(iter.depth()+depth_delta), title_prefix, title, detail, level);
+inline void log_line(const json_iterator &iter, const char *title_prefix, const char *title, std::string_view detail, int delta, int depth_delta) noexcept {
+  log_line(iter, iter.position()+delta, depth_t(iter.depth()+depth_delta), title_prefix, title, detail);
 }
-inline void log_line(const json_iterator &iter, token_position index, depth_t depth, const char *title_prefix, const char *title, std::string_view detail, log_level level) noexcept {
-  if (LOG_ENABLED && simdjson_unlikely(should_log(level))) {
+inline void log_line(const json_iterator &iter, token_position index, depth_t depth, const char *title_prefix, const char *title, std::string_view detail) noexcept {
+  if (LOG_ENABLED) {
     const int indent = depth*2;
     const auto buf = iter.token.buf;
     printf("| %*s%s%-*s ",

include/simdjson/generic/ondemand/logger.h

@@ -5,13 +5,11 @@ namespace ondemand {
 class json_iterator;
 class value_iterator;
 
+// Logging should be free unless SIMDJSON_VERBOSE_LOGGING is set. Importantly, it is critical
+// that the call to the log functions be side-effect free. Thus, for example, you should not
+// create temporary std::string instances.
 namespace logger {
 
-enum class log_level : int32_t {
-  LOG_INFO = 0,
-  LOG_ERROR = 1
-};
-
 #if SIMDJSON_VERBOSE_LOGGING
   static constexpr const bool LOG_ENABLED = true;
 #else
@@ -21,11 +19,9 @@ enum class log_level : int32_t {
 // We do not want these functions to be 'really inlined' since real inlining is
 // for performance purposes and if you are using the loggers, you do not care about
 // performance (or should not).
-template<typename... Args>
-static inline std::string string_format(const std::string& format, const Args&... args);
 static inline void log_headers() noexcept;
-static inline void log_line(const json_iterator &iter, token_position index, depth_t depth, const char *title_prefix, const char *title, std::string_view detail, log_level level) noexcept;
-static inline void log_line(const json_iterator &iter, const char *title_prefix, const char *title, std::string_view detail, int delta, int depth_delta, log_level level) noexcept;
+static inline void log_line(const json_iterator &iter, token_position index, depth_t depth, const char *title_prefix, const char *title, std::string_view detail) noexcept;
+static inline void log_line(const json_iterator &iter, const char *title_prefix, const char *title, std::string_view detail, int delta, int depth_delta) noexcept;
 static inline void log_event(const json_iterator &iter, const char *type, std::string_view detail="", int delta=0, int depth_delta=0) noexcept;
 static inline void log_value(const json_iterator &iter, token_position index, depth_t depth, const char *type, std::string_view detail="") noexcept;
 static inline void log_value(const json_iterator &iter, const char *type, std::string_view detail="", int delta=-1, int depth_delta=0) noexcept;

include/simdjson/generic/ondemand/object-inl.h

@@ -5,23 +5,13 @@ namespace ondemand {
 simdjson_inline simdjson_result<value> object::find_field_unordered(const std::string_view key) & noexcept {
   bool has_value;
   SIMDJSON_TRY( iter.find_field_unordered_raw(key).get(has_value) );
-  if (!has_value) {
-    auto key_str = std::string(key.data(), key.size());
-    auto msg = logger::string_format("Cannot find key: %s", key_str.c_str());
-    logger::log_error(iter, msg.c_str());
-    return NO_SUCH_FIELD;
-  }
+  if (!has_value) { return NO_SUCH_FIELD; }
   return value(iter.child());
 }
 simdjson_inline simdjson_result<value> object::find_field_unordered(const std::string_view key) && noexcept {
   bool has_value;
   SIMDJSON_TRY( iter.find_field_unordered_raw(key).get(has_value) );
-  if (!has_value) {
-    auto key_str = std::string(key.data(), key.size());
-    auto msg = logger::string_format("Cannot find key: %s", key_str.c_str());
-    logger::log_error(iter, msg.c_str());
-    return NO_SUCH_FIELD;
-  }
+  if (!has_value) { return NO_SUCH_FIELD; }
   return value(iter.child());
 }
 simdjson_inline simdjson_result<value> object::operator[](const std::string_view key) & noexcept {
@@ -33,23 +23,13 @@ simdjson_inline simdjson_result<value> object::operator[](const std::string_view
 simdjson_inline simdjson_result<value> object::find_field(const std::string_view key) & noexcept {
   bool has_value;
   SIMDJSON_TRY( iter.find_field_raw(key).get(has_value) );
-  if (!has_value) {
-    auto key_str = std::string(key.data(), key.size());
-    auto msg = logger::string_format("Cannot find key: %s", key_str.c_str());
-    logger::log_error(iter, msg.c_str());
-    return NO_SUCH_FIELD;
-  }
+  if (!has_value) { return NO_SUCH_FIELD; }
   return value(iter.child());
 }
 simdjson_inline simdjson_result<value> object::find_field(const std::string_view key) && noexcept {
   bool has_value;
   SIMDJSON_TRY( iter.find_field_raw(key).get(has_value) );
-  if (!has_value) {
-    auto key_str = std::string(key.data(), key.size());
-    auto msg = logger::string_format("Cannot find key: %s", key_str.c_str());
-    logger::log_error(iter, msg.c_str());
-    return NO_SUCH_FIELD;
-  }
+  if (!has_value) { return NO_SUCH_FIELD; }
   return value(iter.child());
 }
 

include/simdjson/generic/ondemand/parser.h

@@ -212,6 +212,11 @@ public:
    *                   spot is cache-related: small enough to fit in cache, yet big enough to
    *                   parse as many documents as possible in one tight loop.
    *                   Defaults to 10MB, which has been a reasonable sweet spot in our tests.
+   * @param allow_comma_separated (defaults on false) This allows a mode where the documents are
+   *                   separated by commas instead of whitespace. It comes with a performance
+   *                   penalty because the entire document is indexed at once (and the document must be
+   *                   less than 4 GB), and there is no multithreading. In this mode, the batch_size parameter
+   *                   is effectively ignored, as it is set to at least the document size.
    * @return The stream, or an error. An empty input will yield 0 documents rather than an EMPTY error. Errors:
    *         - MEMALLOC if the parser does not have enough capacity and memory allocation fails
    *         - CAPACITY if the parser does not have enough capacity and batch_size > max_capacity.

include/simdjson/generic/ondemand/token_iterator.h

@@ -116,8 +116,8 @@ protected:
   friend class json_iterator;
   friend class value_iterator;
   friend class object;
-  friend simdjson_inline void logger::log_line(const json_iterator &iter, const char *title_prefix, const char *title, std::string_view detail, int delta, int depth_delta, logger::log_level lvl) noexcept;
-  friend simdjson_inline void logger::log_line(const json_iterator &iter, token_position index, depth_t depth, const char *title_prefix, const char *title, std::string_view detail, logger::log_level lvl) noexcept;
+  friend simdjson_inline void logger::log_line(const json_iterator &iter, const char *title_prefix, const char *title, std::string_view detail, int delta, int depth_delta) noexcept;
+  friend simdjson_inline void logger::log_line(const json_iterator &iter, token_position index, depth_t depth, const char *title_prefix, const char *title, std::string_view detail) noexcept;
 };
 
 } // namespace ondemand

include/simdjson/generic/ondemand/value-inl.h

@@ -357,7 +357,7 @@ simdjson_inline simdjson_result<bool> simdjson_result<SIMDJSON_IMPLEMENTATION::o
   if (error()) { return error(); }
   return first.is_integer();
 }
-simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number_type> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::get_number_type() noexcept {
+simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::numberparsing::number_type> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::get_number_type() noexcept {
   if (error()) { return error(); }
   return first.get_number_type();
 }

include/simdjson/generic/ondemand/value.h

@@ -688,7 +688,7 @@ public:
   simdjson_inline simdjson_result<bool> is_scalar() noexcept;
   simdjson_inline simdjson_result<bool> is_negative() noexcept;
   simdjson_inline simdjson_result<bool> is_integer() noexcept;
-  simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number_type> get_number_type() noexcept;
+  simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::numberparsing::number_type> get_number_type() noexcept;
   simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number> get_number() noexcept;
 
   /** @copydoc simdjson_inline std::string_view value::raw_json_token() const noexcept */

include/simdjson/generic/ondemand/value_iterator-inl.h

@@ -584,7 +584,7 @@ simdjson_inline simdjson_result<bool> value_iterator::is_root_integer(bool check
   return answer;
 }
 
-simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number_type> value_iterator::get_root_number_type(bool check_trailing) noexcept {
+simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::numberparsing::number_type> value_iterator::get_root_number_type(bool check_trailing) noexcept {
   auto max_len = peek_start_length();
   auto json = peek_root_scalar("number");
   // Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/,

include/simdjson/haswell/numberparsing.h

@@ -3,8 +3,9 @@
 
 namespace simdjson {
 namespace SIMDJSON_IMPLEMENTATION {
-namespace {
+namespace numberparsing {
 
+/** @private */
 static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) {
   // this actually computes *16* values so we are being wasteful.
   const __m128i ascii0 = _mm_set1_epi8('0');
@@ -23,7 +24,7 @@ static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars
       t4); // only captures the sum of the first 8 digits, drop the rest
 }
 
-} // unnamed namespace
+} // namespace numberparsing
 } // namespace SIMDJSON_IMPLEMENTATION
 } // namespace simdjson
 

include/simdjson/icelake/numberparsing.h

@@ -3,7 +3,7 @@
 
 namespace simdjson {
 namespace SIMDJSON_IMPLEMENTATION {
-namespace {
+namespace numberparsing {
 
 static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) {
   // this actually computes *16* values so we are being wasteful.
@@ -23,7 +23,7 @@ static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars
       t4); // only captures the sum of the first 8 digits, drop the rest
 }
 
-} // unnamed namespace
+} // namespace numberparsing
 } // namespace SIMDJSON_IMPLEMENTATION
 } // namespace simdjson
 

include/simdjson/ppc64/numberparsing.h

@@ -9,12 +9,12 @@
 
 namespace simdjson {
 namespace SIMDJSON_IMPLEMENTATION {
-namespace {
+namespace numberparsing {
 
 // we don't have appropriate instructions, so let us use a scalar function
 // credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/
-static simdjson_inline uint32_t
-parse_eight_digits_unrolled(const uint8_t *chars) {
+/** @private */
+static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) {
   uint64_t val;
   std::memcpy(&val, chars, sizeof(uint64_t));
 #ifdef __BIG_ENDIAN__
@@ -29,7 +29,7 @@ parse_eight_digits_unrolled(const uint8_t *chars) {
   return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32);
 }
 
-} // unnamed namespace
+} // namespace numberparsing
 } // namespace SIMDJSON_IMPLEMENTATION
 } // namespace simdjson
 

include/simdjson/simdjson_version.h

@@ -4,7 +4,7 @@
 #define SIMDJSON_SIMDJSON_VERSION_H
 
 /** The version of simdjson being used (major.minor.revision) */
-#define SIMDJSON_VERSION "3.2.0"
+#define SIMDJSON_VERSION "3.2.1"
 
 namespace simdjson {
 enum {
@@ -19,7 +19,7 @@ enum {
   /**
    * The revision (major.minor.REVISION) of simdjson being used.
    */
-  SIMDJSON_VERSION_REVISION = 0
+  SIMDJSON_VERSION_REVISION = 1
 };
 } // namespace simdjson
 

include/simdjson/westmere/numberparsing.h

@@ -3,8 +3,9 @@
 
 namespace simdjson {
 namespace SIMDJSON_IMPLEMENTATION {
-namespace {
+namespace numberparsing {
 
+/** @private */
 static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) {
   // this actually computes *16* values so we are being wasteful.
   const __m128i ascii0 = _mm_set1_epi8('0');
@@ -23,7 +24,7 @@ static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars
       t4); // only captures the sum of the first 8 digits, drop the rest
 }
 
-} // unnamed namespace
+} // namespace numberparsing
 } // namespace SIMDJSON_IMPLEMENTATION
 } // namespace simdjson
 

singleheader/amalgamate.py

@@ -126,8 +126,10 @@ def dofile(fid, prepath, filename):
 # Forcing it to be UTC is difficult, because it needs to be portable
 # between gnu date and busybox date.
 try:
-    timestamp = subprocess.run(['git', 'show', '-s', '--format=%ci', 'HEAD'],
-                           stdout=subprocess.PIPE).stdout.decode('utf-8').strip()
+    proc = subprocess.run(['git', 'show', '-s', '--format=%ci', 'HEAD'],
+                           stdout=subprocess.PIPE)
+    print("the commandline is {}".format(proc.args))
+    timestamp = proc.stdout.decode('utf-8').strip()
 except:
     print("git not found, timestamp based on current time")
     timestamp = str(datetime.datetime.now())

singleheader/simdjson.cpp

@@ -1,4 +1,4 @@
-/* auto-generated on 2023-06-15 09:11:14 -0400. Do not edit! */
+/* auto-generated on 2023-07-06 21:34:14 -0400. Do not edit! */
 /* begin file src/simdjson.cpp */
 #include "simdjson.h"
 

tests/ondemand/ondemand_readme_examples.cpp

@@ -1037,6 +1037,22 @@ int load_example_except_morecomplete(void) {
   }
   return EXIT_SUCCESS;
 }
+
+bool allow_comma_separated_example() {
+  TEST_START();
+  auto json = R"( 1, 2, 3, 4, "a", "b", "c", {"hello": "world"} , [1, 2, 3])"_padded;
+  ondemand::parser parser;
+  ondemand::document_stream doc_stream;
+  // We pass '32' as the batch size, but it is a bogus parameter because, since
+  // we pass 'true' to the allow_comma parameter, the batch size will be set to at least
+  // the document size.
+  auto error = parser.iterate_many(json, 32, true).get(doc_stream);
+  if(error) { std::cerr << error << std::endl; return false; }
+  for (auto doc : doc_stream) {
+    std::cout << doc.type() << std::endl;
+  }
+  TEST_SUCCEED();
+}
 #endif
 bool test_load_example() {
   TEST_START();
@@ -1067,7 +1083,6 @@ bool current_location_tape_error() {
   TEST_SUCCEED();
 }
 
-
 bool current_location_user_error() {
   TEST_START();
   auto json = R"( [1,2,3] )"_padded;
@@ -1301,6 +1316,7 @@ bool run() {
     && gen_raw1() && gen_raw2() && gen_raw3()
     && at_end()
     && example1956() && example1958()
+    && allow_comma_separated_example()
 //    && basics_1() // Fails because twitter.json isn't in current directory. Compile test only.
     &&  basics_treewalk()
     &&  basics_treewalk_breakline()

tools/release.py

@@ -23,8 +23,9 @@ def toversionstring(major, minor, rev):
 
 def topaddedversionstring(major, minor, rev):
     return str(major)+str(minor).zfill(3)+str(rev).zfill(3)
-
+print("Calling git rev-parse --abbrev-ref HEAD")
 pipe = subprocess.Popen(["git", "rev-parse", "--abbrev-ref", "HEAD"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+print("the commandline is {}".format(pipe.args))
 branchresult = pipe.communicate()[0].decode().strip()
 
 if(branchresult != "master"):
@@ -34,8 +35,9 @@ ret = subprocess.call(["git", "remote", "update"])
 
 if(ret != 0):
     sys.exit(ret)
-
+print("Calling git log HEAD.. --oneline")
 pipe = subprocess.Popen(["git", "log", "HEAD..", "--oneline"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+print("the commandline is {}".format(pipe.args))
 uptodateresult = pipe.communicate()[0].decode().strip()
 
 if(len(uptodateresult) != 0):
@@ -151,12 +153,17 @@ for line in fileinput.input(doxyfile, inplace=1, backup='.bak'):
 print("modified "+doxyfile+", a backup was made")
 
 
+print("running amalgamate.py")
+cp = subprocess.run(["python3", maindir+ os.sep + "singleheader/amalgamate.py"], stdout=subprocess.DEVNULL)  # doesn't capture output
+print("the commandline is {}".format(cp.args))
 
-cp = subprocess.run(["python3", "amalgamate.py"], stdout=subprocess.DEVNULL, cwd=maindir+ os.sep + "singleheader")  # doesn't capture output
 if(cp.returncode != 0):
     print("Failed to run amalgamate")
 
+print("running doxygen")
 cp = subprocess.run(["doxygen"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, cwd=maindir)  # doesn't capture output
+print("the commandline is {}".format(cp.args))
+
 if(cp.returncode != 0):
     print("Failed to run doxygen")