version bump

* Allowing users to write directly to std::optional<std::string>

* better fallback

* do not force the cast to std::string

* missing header

* removing abort

.github/workflows/ubuntu20-sani.yml

@@ -32,7 +32,6 @@ jobs:
         with:
           path: dependencies/.cache
           key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
-          ctest --output-on-failure -LE explicitonly -j
       - name: Use cmake with undefined sanitizer
         run: |
           mkdir builddebugundefsani &&

CMakeLists.txt

@@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.14)
 project(
     simdjson
     # The version number is modified by tools/release.py
-    VERSION 3.3.0
+    VERSION 3.6.0
     DESCRIPTION "Parsing gigabytes of JSON per second"
     HOMEPAGE_URL "https://simdjson.org/"
     LANGUAGES CXX C
@@ -20,8 +20,8 @@ string(
 # ---- Options, variables ----
 
 # These version numbers are modified by tools/release.py
-set(SIMDJSON_LIB_VERSION "16.0.0" CACHE STRING "simdjson library version")
-set(SIMDJSON_LIB_SOVERSION "16" CACHE STRING "simdjson library soversion")
+set(SIMDJSON_LIB_VERSION "19.0.0" CACHE STRING "simdjson library version")
+set(SIMDJSON_LIB_SOVERSION "19" CACHE STRING "simdjson library soversion")
 
 option(SIMDJSON_BUILD_STATIC_LIB "Build simdjson_static library along with simdjson" OFF)
 

Doxyfile

@@ -38,7 +38,7 @@ PROJECT_NAME           = simdjson
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = "3.3.0"
+PROJECT_NUMBER         = "3.6.0"
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a

doc/basics.md

@@ -30,6 +30,7 @@ An overview of what you need to know to use simdjson, with examples.
   - [Dynamic Number Types](#dynamic-number-types)
   - [Raw Strings](#raw-strings)
   - [General Direct Access to the Raw JSON String](#general-direct-access-to-the-raw-json-string)
+  - [Storing Directly into an Existing String Instance](#storing-directly-into-an-existing-string-instance)
   - [Thread Safety](#thread-safety)
   - [Standard Compliance](#standard-compliance)
   - [Backwards Compatibility](#backwards-compatibility)
@@ -179,8 +180,8 @@ strcpy(json, "[1]");
 ondemand::document doc = parser.iterate(json, strlen(json), sizeof(json));
 ```
 
-The simdjson library will also accept `std::string` instances, as long as the `capacity()` of
-the string exceeds the `size()` by at least `SIMDJSON_PADDING`. You can increase the `capacity()` with the `reserve()` function of your strings.
+The simdjson library will also accept `std::string` instances. If the provided
+reference is non-const, it will allocate padding as needed.
 
 You can copy your data directly on a `simdjson::padded_string` as follows:
 
@@ -1653,7 +1654,7 @@ JSON string to a user-provided buffer:
 
 General Direct Access to the Raw JSON String
 --------------------------------
-If your value is a string, the `raw_json_string` gives you direct access to the unprocess
+If your value is a string, the `raw_json_string` you with `get_raw_json_string()` gives you direct access to the unprocessed
 string. The simdjson library allows you to have access to the raw underlying JSON
 more generally.
 
@@ -1722,6 +1723,82 @@ obj.reset(); // revise the object
 uint64_t x = obj["value"]; // gives me 123
 ```
 
+You can use `raw_json()` with the values inside an array and object. When
+calling `raw_json()` on an untyped value, it acts as `raw_json()` when the
+value is an array or an object. Otherwise, it acts as `raw_json_token()`.
+It is useful if you do not care for the type of the value and just wants a
+string representation.
+
+```C++
+  auto json = R"( [1,2,"fds", {"a":1}, [1,344]] )"_padded;
+  ondemand::parser parser;
+  ondemand::document doc = parser.iterate(json);
+  size_t counter = 0;
+  for(auto array: doc) {
+    std::string_view raw = array.raw_json();
+    // will capture "1", "2", "\"fds\"", "{\"a\":1}", "[1,344]"
+  }
+```
+
+```C++
+  auto json = R"( {"key1":1,"key2":2,"key3":"fds", "key4":{"a":1}, "key5":[1,344]} )"_padded;
+  ondemand::parser parser;
+  ondemand::document doc = parser.iterate(json);
+  size_t counter = 0;
+  for(auto key_value: doc.get_object()) {
+    std::string_view raw = key_value.value().raw_json();
+    // will capture "1", "2", "\"fds\"", "{\"a\":1}", "[1,344]"
+  }
+```
+
+
+Storing Directly into an Existing String Instance
+-----------------------------------------------------
+
+The simdjson library favours  the use of `std::string_view` instances because
+it tends to lead to better performance due to causing fewer memory allocations.
+However, they are cases where you need to store a string result in an `std::string``
+instance. You can do so with a templated version of the `to_string()` method which takes as
+a parameter a reference to an `std::string`.
+
+```C++
+  auto json = R"({
+  "name": "Daniel",
+  "age": 42
+})"_padded;
+  ondemand::parser parser;
+  ondemand::document doc = parser.iterate(json);
+  std::string name;
+  doc["name"].get_string(name);
+```
+
+The same routine can be written without exceptions handling:
+
+```C++
+  std::string name;
+  auto err = doc["name"].get_string(name);
+  if(err) { /* handle error */ }
+```
+
+The `std::string` instance, once created, is independent. Unlike our `std::string_view` instances,
+it does not point at data that is within our `parser` instance. The same caveat applies: you should
+only consume a JSON string once.
+
+Because `get_string()` is a template that requires a type that can be assigned an `std::string`, you
+can use it with features such as `std::optional`:
+
+```C++
+  auto json = R"({ "foo1": "3.1416" } )"_padded;
+  ondemand::parser parser;
+  ondemand::document doc = parser.iterate(json);
+  std::optional<std::string> value;
+  if(doc["foo1"].get_string(value)) { /* error */ }
+  // value was populated with "3.1416"
+```
+
+You should be mindful of the trade-off: allocating multiple
+`std::string` instances can become expensive.
+
 Thread Safety
 -------------
 

include/simdjson/dom/document_stream.h

@@ -224,8 +224,7 @@ private:
    * Parse the next document found in the buffer previously given to document_stream.
    *
    * The content should be a valid JSON document encoded as UTF-8. If there is a
-   * UTF-8 BOM, the caller is responsible for omitting it, UTF-8 BOM are
-   * discouraged.
+   * UTF-8 BOM, the parser skips it.
    *
    * You do NOT need to pre-allocate a parser.  This function takes care of
    * pre-allocating a capacity defined by the batch_size defined when creating the

include/simdjson/dom/element-inl.h

@@ -410,6 +410,12 @@ inline simdjson_result<element> element::at_key(std::string_view key) const noex
 inline simdjson_result<element> element::at_key_case_insensitive(std::string_view key) const noexcept {
   return get<object>().at_key_case_insensitive(key);
 }
+inline bool element::operator<(const element &other) const noexcept {
+  return tape.json_index < other.tape.json_index;
+}
+inline bool element::operator==(const element &other) const noexcept {
+  return tape.json_index == other.tape.json_index;
+}
 
 inline bool element::dump_raw_tape(std::ostream &out) const noexcept {
   SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914

include/simdjson/dom/element.h

@@ -211,7 +211,11 @@ public:
   inline simdjson_result<T> get() const noexcept {
     // Unless the simdjson library provides an inline implementation, calling this method should
     // immediately fail.
-    static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library.");
+    static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library. "
+      "The supported types are Boolean (bool), numbers (double, uint64_t, int64_t), "
+      "strings (std::string_view, const char *), arrays (dom::array) and objects (dom::object). "
+      "We recommand you use get_double(), get_bool(), get_uint64(), get_int64(), "
+      "get_object(), get_array() or get_string() instead of the get template.");
   }
 
   /**
@@ -451,6 +455,22 @@ public:
    */
   inline simdjson_result<element> at_key_case_insensitive(std::string_view key) const noexcept;
 
+  /**
+   * operator< defines a total order for element allowing to use them in
+   * ordered C++ STL containers
+   *
+   * @return TRUE if the key appears before the other one in the tape
+   */
+  inline bool operator<(const element &other) const noexcept;
+
+  /**
+   * operator== allows to verify if two element values reference the
+   * same JSON item
+   *
+   * @return TRUE if the two values references the same JSON element
+   */
+  inline bool operator==(const element &other) const noexcept;
+
   /** @private for debugging. Prints out the root element. */
   inline bool dump_raw_tape(std::ostream &out) const noexcept;
 

include/simdjson/dom/parser-inl.h

@@ -12,6 +12,7 @@
 #include "simdjson/dom/element-inl.h"
 
 #include <climits>
+#include <cstring> /* memcmp */
 
 namespace simdjson {
 namespace dom {
@@ -120,8 +121,14 @@ inline simdjson_result<element> parser::parse_into_document(document& provided_d
       _loaded_bytes_capacity = len;
     }
     std::memcpy(static_cast<void *>(loaded_bytes.get()), buf, len);
+    buf = reinterpret_cast<const uint8_t*>(loaded_bytes.get());
   }
-  _error = implementation->parse(realloc_if_needed ? reinterpret_cast<const uint8_t*>(loaded_bytes.get()): buf, len, provided_doc);
+
+  if((len >= 3) && (std::memcmp(buf, "\xEF\xBB\xBF", 3) == 0)) {
+    buf += 3;
+    len -= 3;
+  }
+  _error = implementation->parse(buf, len, provided_doc);
 
   if (_error) { return _error; }
 
@@ -158,6 +165,10 @@ simdjson_inline simdjson_result<element> parser::parse(const padded_string_view
 
 inline simdjson_result<document_stream> parser::parse_many(const uint8_t *buf, size_t len, size_t batch_size) noexcept {
   if(batch_size < MINIMAL_BATCH_SIZE) { batch_size = MINIMAL_BATCH_SIZE; }
+  if((len >= 3) && (std::memcmp(buf, "\xEF\xBB\xBF", 3) == 0)) {
+    buf += 3;
+    len -= 3;
+  }
   return document_stream(*this, buf, len, batch_size);
 }
 inline simdjson_result<document_stream> parser::parse_many(const char *buf, size_t len, size_t batch_size) noexcept {

include/simdjson/dom/parser.h

@@ -254,6 +254,8 @@ public:
    * And, possibly, no document many have been parsed when the `parser.load_many(path)` function
    * returned.
    *
+   * If there is a UTF-8 BOM, the parser skips it.
+   *
    * ### Format
    *
    * The file must contain a series of one or more JSON documents, concatenated into a single
@@ -346,6 +348,8 @@ public:
    *     cout << std::string(doc["title"]) << endl;
    *   }
    *
+   * If there is a UTF-8 BOM, the parser skips it.
+   *
    * ### Format
    *
    * The buffer must contain a series of one or more JSON documents, concatenated into a single

include/simdjson/generic/ondemand/document-inl.h

@@ -128,6 +128,10 @@ simdjson_inline simdjson_result<double> document::get_double_in_string() noexcep
 simdjson_inline simdjson_result<std::string_view> document::get_string(bool allow_replacement) noexcept {
   return get_root_value_iterator().get_root_string(true, allow_replacement);
 }
+template <typename string_type>
+simdjson_inline error_code document::get_string(string_type& receiver, bool allow_replacement) noexcept {
+  return get_root_value_iterator().get_root_string(receiver, true, allow_replacement);
+}
 simdjson_inline simdjson_result<std::string_view> document::get_wobbly_string() noexcept {
   return get_root_value_iterator().get_root_wobbly_string(true);
 }
@@ -397,6 +401,11 @@ simdjson_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLE
   if (error()) { return error(); }
   return first.get_string(allow_replacement);
 }
+template <typename string_type>
+simdjson_inline error_code simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::get_string(string_type& receiver, bool allow_replacement) noexcept {
+  if (error()) { return error(); }
+  return first.get_string(receiver, allow_replacement);
+}
 simdjson_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::get_wobbly_string() noexcept {
   if (error()) { return error(); }
   return first.get_wobbly_string();
@@ -583,6 +592,8 @@ simdjson_inline simdjson_result<int64_t> document_reference::get_int64_in_string
 simdjson_inline simdjson_result<double> document_reference::get_double() noexcept { return doc->get_root_value_iterator().get_root_double(false); }
 simdjson_inline simdjson_result<double> document_reference::get_double_in_string() noexcept { return doc->get_root_value_iterator().get_root_double(false); }
 simdjson_inline simdjson_result<std::string_view> document_reference::get_string(bool allow_replacement) noexcept { return doc->get_root_value_iterator().get_root_string(false, allow_replacement); }
+template <typename string_type>
+simdjson_inline error_code document_reference::get_string(string_type& receiver, bool allow_replacement) noexcept { return doc->get_root_value_iterator().get_root_string(receiver, false, allow_replacement); }
 simdjson_inline simdjson_result<std::string_view> document_reference::get_wobbly_string() noexcept { return doc->get_root_value_iterator().get_root_wobbly_string(false); }
 simdjson_inline simdjson_result<raw_json_string> document_reference::get_raw_json_string() noexcept { return doc->get_root_value_iterator().get_root_raw_json_string(false); }
 simdjson_inline simdjson_result<bool> document_reference::get_bool() noexcept { return doc->get_root_value_iterator().get_root_bool(false); }
@@ -719,6 +730,11 @@ simdjson_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLE
   if (error()) { return error(); }
   return first.get_string(allow_replacement);
 }
+template <typename string_type>
+simdjson_inline error_code simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::get_string(string_type& receiver, bool allow_replacement) noexcept {
+  if (error()) { return error(); }
+  return first.get_string(receiver, allow_replacement);
+}
 simdjson_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::get_wobbly_string() noexcept {
   if (error()) { return error(); }
   return first.get_wobbly_string();

include/simdjson/generic/ondemand/document.h

@@ -100,6 +100,21 @@ public:
    * @returns INCORRECT_TYPE if the JSON value is not a string.
    */
   simdjson_inline simdjson_result<std::string_view> get_string(bool allow_replacement = false) noexcept;
+  /**
+   * Attempts to fill the provided std::string reference with the parsed value of the current string.
+   *
+   * The string is guaranteed to be valid UTF-8.
+   *
+   * Important: a value should be consumed once. Calling get_string() twice on the same value
+   * is an error.
+   *
+   * Performance: This method may be slower than get_string() or get_string(bool) because it may need to allocate memory.
+   * We recommend you avoid allocating an std::string unless you need to.
+   *
+   * @returns INCORRECT_TYPE if the JSON value is not a string. Otherwise, we return SUCCESS.
+   */
+  template <typename string_type>
+  simdjson_inline error_code get_string(string_type& receiver, bool allow_replacement = false) noexcept;
   /**
    * Cast this JSON value to a string.
    *
@@ -160,13 +175,19 @@ public:
   template<typename T> simdjson_inline simdjson_result<T> get() & noexcept {
     // Unless the simdjson library provides an inline implementation, calling this method should
     // immediately fail.
-    static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library.");
+    static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library. "
+      "The supported types are ondemand::object, ondemand::array, raw_json_string, std::string_view, uint64_t, "
+      "int64_t, double, and bool. We recommend you use get_double(), get_bool(), get_uint64(), get_int64(), "
+      " get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template.");
   }
   /** @overload template<typename T> simdjson_result<T> get() & noexcept */
   template<typename T> simdjson_inline simdjson_result<T> get() && noexcept {
     // Unless the simdjson library provides an inline implementation, calling this method should
     // immediately fail.
-    static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library.");
+    static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library. "
+      "The supported types are ondemand::object, ondemand::array, raw_json_string, std::string_view, uint64_t, "
+      "int64_t, double, and bool. We recommend you use get_double(), get_bool(), get_uint64(), get_int64(), "
+      " get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template.");
   }
 
   /**
@@ -618,6 +639,8 @@ public:
   simdjson_inline simdjson_result<double> get_double() noexcept;
   simdjson_inline simdjson_result<double> get_double_in_string() noexcept;
   simdjson_inline simdjson_result<std::string_view> get_string(bool allow_replacement = false) noexcept;
+  template <typename string_type>
+  simdjson_inline error_code get_string(string_type& receiver, bool allow_replacement = false) noexcept;
   simdjson_inline simdjson_result<std::string_view> get_wobbly_string() noexcept;
   simdjson_inline simdjson_result<raw_json_string> get_raw_json_string() noexcept;
   simdjson_inline simdjson_result<bool> get_bool() noexcept;
@@ -687,6 +710,8 @@ public:
   simdjson_inline simdjson_result<double> get_double() noexcept;
   simdjson_inline simdjson_result<double> get_double_in_string() noexcept;
   simdjson_inline simdjson_result<std::string_view> get_string(bool allow_replacement = false) noexcept;
+  template <typename string_type>
+  simdjson_inline error_code get_string(string_type& receiver, bool allow_replacement = false) noexcept;
   simdjson_inline simdjson_result<std::string_view> get_wobbly_string() noexcept;
   simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::raw_json_string> get_raw_json_string() noexcept;
   simdjson_inline simdjson_result<bool> get_bool() noexcept;
@@ -759,6 +784,8 @@ public:
   simdjson_inline simdjson_result<double> get_double() noexcept;
   simdjson_inline simdjson_result<double> get_double_in_string() noexcept;
   simdjson_inline simdjson_result<std::string_view> get_string(bool allow_replacement = false) noexcept;
+  template <typename string_type>
+  simdjson_inline error_code get_string(string_type& receiver, bool allow_replacement = false) noexcept;
   simdjson_inline simdjson_result<std::string_view> get_wobbly_string() noexcept;
   simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::raw_json_string> get_raw_json_string() noexcept;
   simdjson_inline simdjson_result<bool> get_bool() noexcept;

include/simdjson/generic/ondemand/document_stream.h

@@ -241,8 +241,7 @@ private:
    * Parse the next document found in the buffer previously given to document_stream.
    *
    * The content should be a valid JSON document encoded as UTF-8. If there is a
-   * UTF-8 BOM, the caller is responsible for omitting it, UTF-8 BOM are
-   * discouraged.
+   * UTF-8 BOM, the parser skips it.
    *
    * You do NOT need to pre-allocate a parser.  This function takes care of
    * pre-allocating a capacity defined by the batch_size defined when creating the

include/simdjson/generic/ondemand/json_iterator.h

@@ -254,6 +254,7 @@ public:
    */
   simdjson_inline simdjson_result<std::string_view> unescape(raw_json_string in, bool allow_replacement) noexcept;
   simdjson_inline simdjson_result<std::string_view> unescape_wobbly(raw_json_string in) noexcept;
+
   simdjson_inline void reenter_child(token_position position, depth_t child_depth) noexcept;
 
   simdjson_inline error_code consume_character(char c) noexcept;

include/simdjson/generic/ondemand/parser-inl.h

@@ -46,6 +46,8 @@ simdjson_warn_unused simdjson_inline error_code parser::allocate(size_t new_capa
 simdjson_warn_unused simdjson_inline simdjson_result<document> parser::iterate(padded_string_view json) & noexcept {
   if (json.padding() < SIMDJSON_PADDING) { return INSUFFICIENT_PADDING; }
 
+  json.remove_utf8_bom();
+
   // Allocate if needed
   if (capacity() < json.length() || !string_buf) {
     SIMDJSON_TRY( allocate(json.length(), max_depth()) );
@@ -68,6 +70,13 @@ simdjson_warn_unused simdjson_inline simdjson_result<document> parser::iterate(s
   return iterate(padded_string_view(json, allocated));
 }
 
+simdjson_warn_unused simdjson_inline simdjson_result<document> parser::iterate(std::string &json) & noexcept {
+  if(json.capacity() - json.size() < SIMDJSON_PADDING) {
+    json.reserve(json.size() + SIMDJSON_PADDING);
+  }
+  return iterate(padded_string_view(json));
+}
+
 simdjson_warn_unused simdjson_inline simdjson_result<document> parser::iterate(const std::string &json) & noexcept {
   return iterate(padded_string_view(json));
 }
@@ -89,6 +98,8 @@ simdjson_warn_unused simdjson_inline simdjson_result<document> parser::iterate(c
 simdjson_warn_unused simdjson_inline simdjson_result<json_iterator> parser::iterate_raw(padded_string_view json) & noexcept {
   if (json.padding() < SIMDJSON_PADDING) { return INSUFFICIENT_PADDING; }
 
+  json.remove_utf8_bom();
+
   // Allocate if needed
   if (capacity() < json.length()) {
     SIMDJSON_TRY( allocate(json.length(), max_depth()) );
@@ -101,6 +112,10 @@ simdjson_warn_unused simdjson_inline simdjson_result<json_iterator> parser::iter
 
 inline simdjson_result<document_stream> parser::iterate_many(const uint8_t *buf, size_t len, size_t batch_size, bool allow_comma_separated) noexcept {
   if(batch_size < MINIMAL_BATCH_SIZE) { batch_size = MINIMAL_BATCH_SIZE; }
+  if((len >= 3) && (std::memcmp(buf, "\xEF\xBB\xBF", 3) == 0)) {
+    buf += 3;
+    len -= 3;
+  }
   if(allow_comma_separated && batch_size < len) { batch_size = len; }
   return document_stream(*this, buf, len, batch_size, allow_comma_separated);
 }

include/simdjson/generic/ondemand/parser.h

@@ -59,7 +59,7 @@ public:
    * It is expected that the content is a valid UTF-8 file, containing a valid JSON document.
    * Otherwise the iterate method may return an error. In particular, the whole input should be
    * valid: we do not attempt to tolerate incorrect content either before or after a JSON
-   * document.
+   * document. If there is a UTF-8 BOM, the parser skips it.
    *
    * ### IMPORTANT: Validate what you use
    *
@@ -107,6 +107,8 @@ public:
   /** @overload simdjson_result<document> iterate(padded_string_view json) & noexcept */
   simdjson_warn_unused simdjson_result<document> iterate(const std::string &json) & noexcept;
   /** @overload simdjson_result<document> iterate(padded_string_view json) & noexcept */
+  simdjson_warn_unused simdjson_result<document> iterate(std::string &json) & noexcept;
+  /** @overload simdjson_result<document> iterate(padded_string_view json) & noexcept */
   simdjson_warn_unused simdjson_result<document> iterate(const simdjson_result<padded_string> &json) & noexcept;
   /** @overload simdjson_result<document> iterate(padded_string_view json) & noexcept */
   simdjson_warn_unused simdjson_result<document> iterate(const simdjson_result<padded_string_view> &json) & noexcept;
@@ -186,6 +188,7 @@ public:
    * arrays or objects) MUST be separated with ASCII whitespace.
    *
    * The characters inside a JSON document, and between JSON documents, must be valid Unicode (UTF-8).
+   * If there is a UTF-8 BOM, the parser skips it.
    *
    * The documents must not exceed batch_size bytes (by default 1MB) or they will fail to parse.
    * Setting batch_size to excessively large or excessively small values may impact negatively the

include/simdjson/generic/ondemand/value-inl.h

@@ -47,6 +47,10 @@ simdjson_inline simdjson_result<raw_json_string> value::get_raw_json_string() no
 simdjson_inline simdjson_result<std::string_view> value::get_string(bool allow_replacement) noexcept {
   return iter.get_string(allow_replacement);
 }
+template <typename string_type>
+simdjson_inline error_code value::get_string(string_type& receiver, bool allow_replacement) noexcept {
+  return iter.get_string(receiver, allow_replacement);
+}
 simdjson_inline simdjson_result<std::string_view> value::get_wobbly_string() noexcept {
   return iter.get_wobbly_string();
 }
@@ -193,6 +197,26 @@ simdjson_inline std::string_view value::raw_json_token() noexcept {
   return std::string_view(reinterpret_cast<const char*>(iter.peek_start()), iter.peek_start_length());
 }
 
+simdjson_inline simdjson_result<std::string_view> value::raw_json() noexcept {
+  json_type t;
+  SIMDJSON_TRY(type().get(t));
+  switch (t)
+  {
+    case json_type::array: {
+      ondemand::array array;
+      SIMDJSON_TRY(get_array().get(array));
+      return array.raw_json();
+    }
+    case json_type::object: {
+      ondemand::object object;
+      SIMDJSON_TRY(get_object().get(object));
+      return object.raw_json();
+    }
+    default:
+      return raw_json_token();
+  }
+}
+
 simdjson_inline simdjson_result<const char *> value::current_location() noexcept {
   return iter.json_iter().current_location();
 }
@@ -319,6 +343,11 @@ simdjson_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLE
   if (error()) { return error(); }
   return first.get_string(allow_replacement);
 }
+template <typename string_type>
+simdjson_inline error_code simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::get_string(string_type& receiver, bool allow_replacement) noexcept {
+  if (error()) { return error(); }
+  return first.get_string(receiver, allow_replacement);
+}
 simdjson_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::get_wobbly_string() noexcept {
   if (error()) { return error(); }
   return first.get_wobbly_string();
@@ -419,6 +448,11 @@ simdjson_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLE
   return first.raw_json_token();
 }
 
+simdjson_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::raw_json() noexcept {
+  if (error()) { return error(); }
+  return first.raw_json();
+}
+
 simdjson_inline simdjson_result<const char *> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::current_location() noexcept {
   if (error()) { return error(); }
   return first.current_location();

include/simdjson/generic/ondemand/value.h

@@ -38,7 +38,10 @@ public:
   template<typename T> simdjson_inline simdjson_result<T> get() noexcept {
     // Unless the simdjson library provides an inline implementation, calling this method should
     // immediately fail.
-    static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library.");
+    static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library. "
+      "The supported types are ondemand::object, ondemand::array, raw_json_string, std::string_view, uint64_t, "
+      "int64_t, double, and bool. We recommend you use get_double(), get_bool(), get_uint64(), get_int64(), "
+      " get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template.");
   }
 
   /**
@@ -132,6 +135,21 @@ public:
    */
   simdjson_inline simdjson_result<std::string_view> get_string(bool allow_replacement = false) noexcept;
 
+  /**
+   * Attempts to fill the provided std::string reference with the parsed value of the current string.
+   *
+   * The string is guaranteed to be valid UTF-8.
+   *
+   * Important: a value should be consumed once. Calling get_string() twice on the same value
+   * is an error.
+   *
+   * Performance: This method may be slower than get_string() or get_string(bool) because it may need to allocate memory.
+   * We recommend you avoid allocating an std::string unless you need to.
+   *
+   * @returns INCORRECT_TYPE if the JSON value is not a string. Otherwise, we return SUCCESS.
+   */
+  template <typename string_type>
+  simdjson_inline error_code get_string(string_type& receiver, bool allow_replacement = false) noexcept;
 
   /**
    * Cast this JSON value to a "wobbly" string.
@@ -475,9 +493,21 @@ public:
    * - true
    * - false
    * - null
+   *
+   * See also value::raw_json().
    */
   simdjson_inline std::string_view raw_json_token() noexcept;
 
+  /**
+   * Get a string_view pointing at this value in the JSON document.
+   * If this element is an array or an object, it consumes the array or the object
+   * and returns a string_view instance corresponding to the
+   * array as represented in JSON. It points inside the original document.
+   * If this element is a scalar (string, number, Boolean, null), it returns what
+   * raw_json_token() would return.
+   */
+  simdjson_inline simdjson_result<std::string_view> raw_json() noexcept;
+
   /**
    * Returns the current location in the document if in bounds.
    */
@@ -602,6 +632,8 @@ public:
   simdjson_inline simdjson_result<double> get_double() noexcept;
   simdjson_inline simdjson_result<double> get_double_in_string() noexcept;
   simdjson_inline simdjson_result<std::string_view> get_string(bool allow_replacement = false) noexcept;
+  template <typename string_type>
+  simdjson_inline error_code get_string(string_type& receiver, bool allow_replacement = false) noexcept;
   simdjson_inline simdjson_result<std::string_view> get_wobbly_string() noexcept;
   simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::raw_json_string> get_raw_json_string() noexcept;
   simdjson_inline simdjson_result<bool> get_bool() noexcept;
@@ -694,6 +726,7 @@ public:
 
   /** @copydoc simdjson_inline std::string_view value::raw_json_token() const noexcept */
   simdjson_inline simdjson_result<std::string_view> raw_json_token() noexcept;
+  simdjson_inline simdjson_result<std::string_view> raw_json() noexcept;
 
   /** @copydoc simdjson_inline simdjson_result<const char *> current_location() noexcept */
   simdjson_inline simdjson_result<const char *> current_location() noexcept;

include/simdjson/generic/ondemand/value_iterator-inl.h

@@ -513,6 +513,14 @@ simdjson_warn_unused simdjson_inline simdjson_result<bool> value_iterator::parse
 simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> value_iterator::get_string(bool allow_replacement) noexcept {
   return get_raw_json_string().unescape(json_iter(), allow_replacement);
 }
+template <typename string_type>
+simdjson_warn_unused simdjson_inline error_code value_iterator::get_string(string_type& receiver, bool allow_replacement) noexcept {
+  std::string_view content;
+  auto err = get_string(allow_replacement).get(content);
+  if (err) { return err; }
+  receiver = content;
+  return SUCCESS;
+}
 simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> value_iterator::get_wobbly_string() noexcept {
   return get_raw_json_string().unescape_wobbly(json_iter());
 }
@@ -636,6 +644,14 @@ simdjson_inline simdjson_result<number> value_iterator::get_root_number(bool che
 simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> value_iterator::get_root_string(bool check_trailing, bool allow_replacement) noexcept {
   return get_root_raw_json_string(check_trailing).unescape(json_iter(), allow_replacement);
 }
+template <typename string_type>
+simdjson_warn_unused simdjson_inline error_code value_iterator::get_root_string(string_type& receiver, bool check_trailing, bool allow_replacement) noexcept {
+  std::string_view content;
+  auto err = get_root_string(check_trailing, allow_replacement).get(content);
+  if (err) { return err; }
+  receiver = content;
+  return SUCCESS;
+}
 simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> value_iterator::get_root_wobbly_string(bool check_trailing) noexcept {
   return get_root_raw_json_string(check_trailing).unescape_wobbly(json_iter());
 }

include/simdjson/generic/ondemand/value_iterator.h

@@ -296,6 +296,8 @@ public:
    */
 
   simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> get_string(bool allow_replacement) noexcept;
+  template <typename string_type>
+  simdjson_warn_unused simdjson_inline error_code get_string(string_type& receiver, bool allow_replacement) noexcept;
   simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> get_wobbly_string() noexcept;
   simdjson_warn_unused simdjson_inline simdjson_result<raw_json_string> get_raw_json_string() noexcept;
   simdjson_warn_unused simdjson_inline simdjson_result<uint64_t> get_uint64() noexcept;
@@ -312,7 +314,9 @@ public:
   simdjson_warn_unused simdjson_inline simdjson_result<number> get_number() noexcept;
 
   simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> get_root_string(bool check_trailing, bool allow_replacement) noexcept;
-    simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> get_root_wobbly_string(bool check_trailing) noexcept;
+  template <typename string_type>
+  simdjson_warn_unused simdjson_inline error_code get_root_string(string_type& receiver, bool check_trailing, bool allow_replacement) noexcept;
+  simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> get_root_wobbly_string(bool check_trailing) noexcept;
   simdjson_warn_unused simdjson_inline simdjson_result<raw_json_string> get_root_raw_json_string(bool check_trailing) noexcept;
   simdjson_warn_unused simdjson_inline simdjson_result<uint64_t> get_root_uint64(bool check_trailing) noexcept;
   simdjson_warn_unused simdjson_inline simdjson_result<uint64_t> get_root_uint64_in_string(bool check_trailing) noexcept;

include/simdjson/padded_string_view-inl.h

@@ -2,9 +2,10 @@
 #define SIMDJSON_PADDED_STRING_VIEW_INL_H
 
 #include "simdjson/padded_string_view.h"
-
 #include "simdjson/error-inl.h"
 
+#include <cstring> /* memcmp */
+
 namespace simdjson {
 
 inline padded_string_view::padded_string_view(const char* s, size_t len, size_t capacity) noexcept
@@ -31,6 +32,16 @@ inline size_t padded_string_view::capacity() const noexcept { return _capacity;
 
 inline size_t padded_string_view::padding() const noexcept { return capacity() - length(); }
 
+inline bool padded_string_view::remove_utf8_bom() noexcept {
+  if(length() < 3) { return false; }
+  if (std::memcmp(data(), "\xEF\xBB\xBF", 3) == 0) {
+    remove_prefix(3);
+    _capacity -= 3;
+    return true;
+  }
+  return false;
+}
+
 #if SIMDJSON_EXCEPTIONS
 inline std::ostream& operator<<(std::ostream& out, simdjson_result<padded_string_view> &s) noexcept(false) { return out << s.value(); }
 #endif

include/simdjson/padded_string_view.h

@@ -54,6 +54,13 @@ public:
   /** The number of allocated bytes. */
   inline size_t capacity() const noexcept;
 
+  /**
+   * Remove the UTF-8 Byte Order Mark (BOM) if it exists.
+   *
+   * @return whether a BOM was found and removed
+   */
+  inline bool remove_utf8_bom() noexcept;
+
   /** The amount of padding on the string (capacity() - length()) */
   inline size_t padding() const noexcept;
 

include/simdjson/portability.h

@@ -58,11 +58,8 @@
 
 #if SIMDJSON_IS_32BITS
 #ifndef SIMDJSON_NO_PORTABILITY_WARNING
-#pragma message("The simdjson library is designed \
-for 64-bit processors and it seems that you are not \
-compiling for a known 64-bit platform. All fast kernels \
-will be disabled and performance may be poor. Please \
-use a 64-bit target such as x64, 64-bit ARM or 64-bit PPC.")
+// In the future, we should allow programmers
+// to get warning.
 #endif // SIMDJSON_NO_PORTABILITY_WARNING
 #endif // SIMDJSON_IS_32BITS
 

include/simdjson/simdjson_version.h

@@ -4,7 +4,7 @@
 #define SIMDJSON_SIMDJSON_VERSION_H
 
 /** The version of simdjson being used (major.minor.revision) */
-#define SIMDJSON_VERSION "3.3.0"
+#define SIMDJSON_VERSION "3.6.0"
 
 namespace simdjson {
 enum {
@@ -15,7 +15,7 @@ enum {
   /**
    * The minor version (major.MINOR.revision) of simdjson being used.
    */
-  SIMDJSON_VERSION_MINOR = 3,
+  SIMDJSON_VERSION_MINOR = 6,
   /**
    * The revision (major.minor.REVISION) of simdjson being used.
    */

singleheader/simdjson.cpp

@@ -1,4 +1,4 @@
-/* auto-generated on . Do not edit! */
+/* auto-generated on 2023-10-30 12:51:02 -0400. Do not edit! */
 /* including simdjson.cpp:  */
 /* begin file simdjson.cpp */
 #define SIMDJSON_SRC_SIMDJSON_CPP
@@ -131,11 +131,8 @@
 
 #if SIMDJSON_IS_32BITS
 #ifndef SIMDJSON_NO_PORTABILITY_WARNING
-#pragma message("The simdjson library is designed \
-for 64-bit processors and it seems that you are not \
-compiling for a known 64-bit platform. All fast kernels \
-will be disabled and performance may be poor. Please \
-use a 64-bit target such as x64, 64-bit ARM or 64-bit PPC.")
+// In the future, we should allow programmers
+// to get warning.
 #endif // SIMDJSON_NO_PORTABILITY_WARNING
 #endif // SIMDJSON_IS_32BITS
 

tests/dom/basictests.cpp

@@ -66,6 +66,16 @@ namespace number_tests {
     return true;
   }
 
+  bool bomskip() {
+    TEST_START();
+    simdjson::dom::parser parser;
+    simdjson::padded_string docdata = "\xEF\xBB\xBF{\"score\":0.8825149536132812}"_padded;
+    double score;
+    ASSERT_SUCCESS(parser.parse(docdata)["score"].get_double().get(score));
+    ASSERT_EQUAL(score, 0.8825149536132812);
+    TEST_SUCCEED();
+  }
+
   bool issue2017() {
     TEST_START();
     simdjson::dom::parser parser;
@@ -386,7 +396,8 @@ namespace number_tests {
   }
 
   bool run() {
-    return issue2017() &&
+    return bomskip() &&
+           issue2017() &&
            truncated_borderline() &&
            specific_tests() &&
            ground_truth() &&
@@ -1658,6 +1669,7 @@ namespace validate_tests {
     }
     return true;
   }
+
   bool test_validate() {
     std::cout << "Running " << __func__ << std::endl;
     const std::string test = R"({ "foo" : 1, "bar" : [ 1, 2, 3 ], "baz": { "a": 1, "b": 2, "c": 3 } })";
@@ -1666,6 +1678,7 @@ namespace validate_tests {
     }
     return true;
   }
+
   bool test_range() {
     std::cout << "Running " << __func__ << std::endl;
     for(size_t len = 0; len <= 128; len++) {
@@ -1683,6 +1696,7 @@ namespace validate_tests {
     }
     return true;
   }
+
   bool test_issue1169() {
     std::cout << "Running " << __func__ << std::endl;
     std::vector<uint8_t> source(64,' ');
@@ -1693,6 +1707,7 @@ namespace validate_tests {
     }
     return true;
   }
+
   bool test_issue1169_long() {
     std::cout << "Running " << __func__ << std::endl;
     for(size_t len = 1; len <= 128; len++) {
@@ -1702,6 +1717,7 @@ namespace validate_tests {
     }
     return true;
   }
+
   bool test_random() {
     std::cout << "Running " << __func__ << std::endl;
     std::vector<uint8_t> source(64,' ');
@@ -1763,6 +1779,7 @@ namespace minify_tests {
     }
     return true;
   }
+
   // this is meant to test buffer overflows.
   bool test_various_lengths2() {
     std::cout << "Running " << __func__ << std::endl;
@@ -1781,6 +1798,7 @@ namespace minify_tests {
     }
     return true;
   }
+
   bool test_single_quote() {
     std::cout << "Running " << __func__ << std::endl;
     const std::string test = "\"";
@@ -1801,12 +1819,14 @@ namespace minify_tests {
     const std::string minified(R"({"foo":1,"bar":[1,2,0.11111111111111113],"baz":{"a":3.1415926535897936,"b":2,"c":3.141592653589794}})");
     return check_minification(test.c_str(), test.size(), minified.c_str(), minified.size());
   }
+
   bool test_minify_array() {
     std::cout << "Running " << __func__ << std::endl;
     std::string test("[ 1,    2,    3]");
     std::string minified("[1,2,3]");
     return check_minification(test.c_str(), test.size(), minified.c_str(), minified.size());
   }
+
   bool test_minify_object() {
     std::cout << "Running " << __func__ << std::endl;
     std::string test(R"({ "foo   " : 1, "b  ar" : [ 1, 2, 3 ], "baz": { "a": 1, "b": 2, "c": 3 } })");
@@ -1849,6 +1869,7 @@ namespace format_tests {
     s << doc;
     return assert_minified(s);
   }
+
   bool print_minify_parser_parse() {
     std::cout << "Running " << __func__ << std::endl;
     dom::parser parser;
@@ -1868,6 +1889,7 @@ namespace format_tests {
     s << value;
     return assert_minified(s, "1");
   }
+
   bool print_minify_element() {
     std::cout << "Running " << __func__ << std::endl;
     dom::parser parser;
@@ -1887,6 +1909,7 @@ namespace format_tests {
     s << array;
     return assert_minified(s, "[1,2,0.11111111111111113]");
   }
+
   bool print_minify_array() {
     std::cout << "Running " << __func__ << std::endl;
     dom::parser parser;
@@ -1906,6 +1929,7 @@ namespace format_tests {
     s << object;
     return assert_minified(s, R"({"a":3.1415926535897936,"b":2,"c":3.141592653589794})");
   }
+
   bool print_minify_object() {
     std::cout << "Running " << __func__ << std::endl;
     dom::parser parser;
@@ -1925,6 +1949,7 @@ namespace format_tests {
     s << parser.parse(DOCUMENT);
     return assert_minified(s);
   }
+
   bool print_minify_parser_parse_exception() {
     std::cout << "Running " << __func__ << std::endl;
     dom::parser parser;
@@ -1940,6 +1965,7 @@ namespace format_tests {
     s << parser.parse(DOCUMENT)["foo"];
     return assert_minified(s, "1");
   }
+
   bool print_minify_element_result_exception() {
     std::cout << "Running " << __func__ << std::endl;
     dom::parser parser;
@@ -1956,6 +1982,7 @@ namespace format_tests {
     s << value;
     return assert_minified(s, "1");
   }
+
   bool print_minify_element_exception() {
     std::cout << "Running " << __func__ << std::endl;
     dom::parser parser;
@@ -1972,6 +1999,7 @@ namespace format_tests {
     s << parser.parse(DOCUMENT)["bar"].get_array();
     return assert_minified(s, "[1,2,0.11111111111111113]");
   }
+
   bool print_minify_array_result_exception() {
     std::cout << "Running " << __func__ << std::endl;
     dom::parser parser;

tests/dom/document_stream_tests.cpp

@@ -397,6 +397,33 @@ namespace document_stream_tests {
     std::cout << "number of documents " << count << std::endl;
     return count == 1;
   }
+
+  bool skipbom() {
+    std::cout << "Running " << __func__ << std::endl;
+    simdjson::dom::parser parser;
+    auto json = "\xEF\xBB\xBF{\"hello\": \"world\"}"_padded;
+    simdjson::dom::document_stream stream;
+    ASSERT_SUCCESS(parser.parse_many(json).get(stream));
+    size_t count = 0;
+    for (auto doc : stream) {
+        if(doc.error()) {
+          std::cerr << "Unexpected error: " << doc.error() << std::endl;
+          return false;
+        }
+        std::string expected = R"({"hello":"world"})";
+        simdjson::dom::element this_document;
+        ASSERT_SUCCESS(doc.get(this_document));
+
+        std::string answer = simdjson::minify(this_document);
+        if(answer != expected) {
+          std::cout << this_document << std::endl;
+          return false;
+        }
+        count += 1;
+    }
+    std::cout << "number of documents " << count << std::endl;
+    return count == 1;
+  }
 #if SIMDJSON_EXCEPTIONS
   bool single_document_exceptions() {
     std::cout << "Running " << __func__ << std::endl;
@@ -913,7 +940,8 @@ namespace document_stream_tests {
   }
 
   bool run() {
-    return fuzzaccess() &&
+    return skipbom() &&
+           fuzzaccess() &&
            baby_fuzzer() &&
            issue1649() &&
            adversarial_single_document_array() &&

tests/dom/document_tests.cpp

@@ -51,6 +51,10 @@ namespace document_tests {
     simdjson::dom::array array;
     ASSERT_SUCCESS( parser.parse(smalljson).get(array) );
     ASSERT_EQUAL( array.size(), 3 );
+    ASSERT_EQUAL( *array.begin() < *array.end(), true );
+    ASSERT_EQUAL( *array.end() < *array.begin(), false );
+    ASSERT_EQUAL( *array.begin() == *array.begin(), true );
+    ASSERT_EQUAL( *array.begin() == *array.end(), false );
     return true;
   }
   bool count_object_example() {
@@ -60,6 +64,10 @@ namespace document_tests {
     simdjson::dom::object object;
     ASSERT_SUCCESS( parser.parse(smalljson).get(object) );
     ASSERT_EQUAL( object.size(), 3 );
+    ASSERT_EQUAL( (*object.begin()).value < (*object.end()).value, true );
+    ASSERT_EQUAL( (*object.end()).value < (*object.begin()).value, false );
+    ASSERT_EQUAL( (*object.begin()).value == (*object.begin()).value, true );
+    ASSERT_EQUAL( (*object.begin()).value == (*object.end()).value, false );
     return true;
   }
   bool padded_with_open_bracket() {
@@ -232,4 +240,4 @@ int main(int argc, char *argv[]) {
   } else {
     return EXIT_FAILURE;
   }
-}
+}

tests/ondemand/CMakeLists.txt

@@ -23,6 +23,7 @@ add_cpp_test(ondemand_ordering_tests         LABELS ondemand acceptance per_impl
 add_cpp_test(ondemand_parse_api_tests        LABELS ondemand acceptance per_implementation)
 add_cpp_test(ondemand_readme_examples        LABELS ondemand acceptance per_implementation)
 add_cpp_test(ondemand_scalar_tests           LABELS ondemand acceptance per_implementation)
+add_cpp_test(ondemand_to_string              LABELS ondemand acceptance per_implementation)
 add_cpp_test(ondemand_twitter_tests          LABELS ondemand acceptance per_implementation)
 add_cpp_test(ondemand_wrong_type_error_tests LABELS ondemand acceptance per_implementation)
 add_cpp_test(ondemand_iterate_many_csv       LABELS ondemand acceptance per_implementation)

tests/ondemand/ondemand_document_stream_tests.cpp

@@ -322,6 +322,43 @@ namespace document_stream_tests {
         TEST_SUCCEED();
     }
 
+    bool skipbom() {
+        TEST_START();
+        auto json = "\xEF\xBB\xBF[1,[1,2]] {\"a\":1,\"b\":2} {\"o\":{\"1\":1,\"2\":2}} [1,2,3]"_padded;
+        ondemand::parser parser;
+        ondemand::document_stream stream;
+        ASSERT_SUCCESS(parser.iterate_many(json).get(stream));
+        std::string_view expected[4] = {"[1,[1,2]]", "{\"a\":1,\"b\":2}", "{\"o\":{\"1\":1,\"2\":2}}", "[1,2,3]"};
+        size_t counter{0};
+        auto i = stream.begin();
+        int64_t x;
+
+        ASSERT_EQUAL(i.source(),expected[counter++]);
+        ASSERT_SUCCESS( (*i).at_pointer("/1/1").get(x) );
+        ASSERT_EQUAL(x,2);
+        ++i;
+
+        ASSERT_EQUAL(i.source(),expected[counter++]);
+        simdjson_result<ondemand::document_reference> xxx = *i;
+        ASSERT_SUCCESS( xxx.find_field("a").get(x) );
+        ASSERT_EQUAL(x,1);
+        ++i;
+
+        ASSERT_EQUAL(i.source(),expected[counter++]);
+        ASSERT_SUCCESS( (*i).at_pointer("/o/2").get(x) );
+        ASSERT_EQUAL(x,2);
+        ++i;
+
+        ASSERT_EQUAL(i.source(),expected[counter++]);
+        ASSERT_SUCCESS( (*i).at_pointer("/2").get(x) );
+        ASSERT_EQUAL(x,3);
+        ++i;
+
+        if (i != stream.end()) { return false; }
+
+        TEST_SUCCEED();
+    }
+
     bool atoms_json() {
         TEST_START();
         auto json = R"(5 true 20.3 "string" )"_padded;
@@ -803,6 +840,7 @@ namespace document_stream_tests {
 
     bool run() {
         return
+            skipbom() &&
             issue1977() &&
             string_with_trailing() &&
             uint64_with_trailing() &&

tests/ondemand/ondemand_misc_tests.cpp

@@ -16,6 +16,18 @@ namespace misc_tests {
     ASSERT_FALSE(b);
     TEST_SUCCEED();
   }
+  bool skipbom() {
+    auto error_phrase = "\xEF\xBB\xBF false"_padded;
+    TEST_START();
+    ondemand::parser parser;
+    ondemand::document doc;
+    ASSERT_SUCCESS(parser.iterate(error_phrase).get(doc));
+    bool b;
+    ASSERT_SUCCESS( doc.get_bool().get(b));
+    ASSERT_FALSE(b);
+    TEST_SUCCEED();
+  }
+
 
   bool issue1981_failure() {
     auto error_phrase = R"(falseA)"_padded;
@@ -593,6 +605,7 @@ namespace misc_tests {
 
   bool run() {
     return
+           skipbom() &&
            issue1981_success() &&
            issue1981_failure() &&
            replacement_char() &&

tests/ondemand/ondemand_parse_api_tests.cpp

@@ -144,14 +144,20 @@ namespace parse_api_tests {
     }
 
     {
-      std::string json = "12";
+      std::string json = "12345642314123421321321321321321312321321321321312";
       json.shrink_to_fit();
       cout << "- string, 0 padding" << endl;
-      ASSERT_ERROR( parser.iterate(json), INSUFFICIENT_PADDING );
+      ASSERT_SUCCESS( parser.iterate(json) );
+    }
+
+    {
+      std::string json = "12345642314123421321321321321321312321321321321312";
+      json.shrink_to_fit();
+      cout << "- string, 0 padding" << endl;
+      ASSERT_ERROR( parser.iterate((const std::string&)json), INSUFFICIENT_PADDING );
       // It's actually kind of hard to allocate "just enough" capacity, since the string tends
       // to grow more than you tell it to.
     }
-
     TEST_SUCCEED();
   }
 

tests/ondemand/ondemand_readme_examples.cpp

@@ -1,6 +1,8 @@
 #include "simdjson.h"
 #include "test_ondemand.h"
-
+#if __cpp_lib_optional >= 201606L
+#include <optional>
+#endif
 using namespace std;
 using namespace simdjson;
 using error_code=simdjson::error_code;
@@ -20,7 +22,39 @@ bool string2() {
 }
 
 
+bool to_string_example_no_except() {
+    TEST_START();
+  auto json = R"({
+  "name": "Daniel",
+  "age": 42
+})"_padded;
+  ondemand::parser parser;
+  ondemand::document doc;
+  auto err = parser.iterate(json).get(doc);
+  if(err) { return false; }
+  std::string name;
+  err = doc["name"].get_string(name);
+  if(err) { return false; }
+  TEST_SUCCEED();
+}
+
 #if SIMDJSON_EXCEPTIONS
+
+
+bool to_string_example() {
+    TEST_START();
+  auto json = R"({
+  "name": "Daniel",
+  "age": 42
+})"_padded;
+  ondemand::parser parser;
+  ondemand::document doc = parser.iterate(json);
+  std::string name;
+  doc["name"].get_string(name);
+  ASSERT_EQUAL(name, "Daniel");
+  TEST_SUCCEED();
+}
+
 bool gen_raw1() {
   TEST_START();
   simdjson::ondemand::parser parser;
@@ -141,6 +175,76 @@ bool examplecrt() {
   TEST_SUCCEED();
 }
 
+
+bool examplecrt_realloc() {
+  TEST_START();
+  std::string unpadded_input_json = R"([
+	{ "monitor": [
+		{ "id": "monitor",		"type": "toggle",		"label": "monitor"			},
+		{ "id": "profile",		"type": "selector",		"label": "collection"		},
+		{ "id": "overlay",		"type": "selector",		"label": "overlay"			},
+		{ "id": "zoom",			"type": "toggleSlider",	"label": "zoom"				}
+	] },
+
+	{ "crt": [
+		{ "id": "system",		"type": "multi",		"label": "system",		"choices": "PAL, NTSC"	},
+		{ "type": "spacer" },
+		{ "id": "brightness",	"type": "slider",		"icon": "brightness"		},
+		{ "id": "contrast",		"type": "slider",		"icon": "contrast"			},
+		{ "id": "saturation",	"type": "slider",		"icon": "saturation"		},
+		{ "type": "spacer" },
+		{ "id": "overscan",		"type": "toggleSlider",	"label": "overscan"			},
+		{ "type": "spacer" },
+		{ "id": "emulation",	"type": "toggle",		"label": "CRT emulation"	},
+		{ "type": "spacer" },
+		{ "id": "curve",		"type": "toggleSlider",	"label": "curve"			},
+		{ "id": "bleed",		"type": "toggleSlider",	"label": "bleed"			},
+		{ "id": "vignette",		"type": "toggleSlider",	"label": "vignette"			},
+		{ "id": "scanlines",	"type": "toggleSlider",	"label": "scanlines"		},
+		{ "id": "gridlines",	"type": "toggleSlider",	"label": "gridlines"		},
+		{ "id": "glow",			"type": "toggleSlider",	"label": "glow"				},
+		{ "id": "flicker",		"type": "toggleSlider",	"label": "flicker"			},
+		{ "id": "noise",		"type": "toggleSlider",	"label": "noise"			},
+    {}
+	] }
+])";
+  unpadded_input_json.shrink_to_fit();
+  auto parser = ondemand::parser{};
+  auto doc = parser.iterate(unpadded_input_json);
+  auto root_array = doc.get_array();
+  // the root should be an object, not an array, but that's the JSON we are
+  // given.
+  for (ondemand::object node : root_array) {
+    // We know that we are going to have just one element in the object.
+    for (auto field : node) {
+      std::cout << "\n\ntop level:" << field.key() << std::endl;
+      // You can get a proper std::string_view for the key with:
+      // std::string_view key = field.unescaped_key();
+      // and second for-range loop to get child-elements here
+      for (ondemand::object inner_object : field.value()) {
+        auto i = inner_object.begin();
+        if (i == inner_object.end()) {
+          std::cout << "empty object" << std::endl;
+          continue;
+        } else {
+          for (; i != inner_object.end(); ++i) {
+            auto inner_field = *i;
+            std::cout << '"' << inner_field.key()
+                      << "\" : " << inner_field.value() << ", ";
+            // You can get proper std::string_view for the key and value with:
+            // std::string_view inner_key = field.unescaped_key();
+            // std::string_view value_str = field.value();
+          }
+        }
+        std::cout << std::endl;
+      }
+      // You can break here if you only want just the first element.
+      // break;
+    }
+  }
+  TEST_SUCCEED();
+}
+
 bool number_tests() {
   TEST_START();
   ondemand::parser parser;
@@ -1377,10 +1481,60 @@ bool example1958() {
   }
   return true;
 }
+
+
+bool to_optional() {
+  TEST_START();
+  auto json = R"({ "foo1": "3.1416" } )"_padded;
+  ondemand::parser parser;
+  ondemand::document doc = parser.iterate(json);
+#if __cpp_lib_optional >= 201606L
+  std::optional<std::string> value;
+  ASSERT_SUCCESS(doc["foo1"].get_string(value));
+  std::cout << value.value() << std::endl;
+#else
+  std::string value;
+  ASSERT_SUCCESS(doc["foo1"].get_string(value));
+  std::cout << value << std::endl;
+#endif
+  TEST_SUCCEED();
+}
+
+bool value_raw_json_array() {
+  TEST_START();
+  auto json = R"( [1,2,"fds", {"a":1}, [1,344]] )"_padded;
+  ondemand::parser parser;
+  ondemand::document doc = parser.iterate(json);
+  std::string_view expected[] = {"1", "2", "\"fds\"", "{\"a\":1}", "[1,344]"};
+  size_t counter = 0;
+  for(auto array: doc) {
+    std::string_view raw = array.raw_json();
+    ASSERT_EQUAL(raw, expected[counter++]);
+  }
+  TEST_SUCCEED();
+}
+
+
+bool value_raw_json_object() {
+  TEST_START();
+  auto json = R"( {"key1":1,"key2":2,"key3":"fds", "key4":{"a":1}, "key5":[1,344]} )"_padded;
+  ondemand::parser parser;
+  ondemand::document doc = parser.iterate(json);
+  std::string_view expected[] = {"1", "2", "\"fds\"", "{\"a\":1}", "[1,344]"};
+  size_t counter = 0;
+  for(auto key_value: doc.get_object()) {
+    std::string_view raw = key_value.value().raw_json();
+    ASSERT_EQUAL(raw, expected[counter++]);
+  }
+  TEST_SUCCEED();
+}
+
 #endif
 bool run() {
   return true
 #if SIMDJSON_EXCEPTIONS
+    && to_optional()
+    && value_raw_json_array() && value_raw_json_object()
     && gen_raw1() && gen_raw2() && gen_raw3()
     && at_end()
     && example1956() && example1958()
@@ -1420,11 +1574,14 @@ bool run() {
     && current_location_user_error()
     && current_location_out_of_bounds()
     && current_location_no_error()
+    && to_string_example_no_except()
   #if SIMDJSON_EXCEPTIONS
+    && to_string_example()
     && raw_string()
     && number_tests()
     && current_location_tape_error_with_except()
     && examplecrt()
+    && examplecrt_realloc()
   #endif
   ;
 }

tests/ondemand/ondemand_to_string.cpp

@@ -0,0 +1,210 @@
+#include "simdjson.h"
+#include "test_ondemand.h"
+
+using namespace simdjson;
+
+namespace json_package_tests {
+using namespace std;
+bool baby() {
+    TEST_START();
+  auto json = R"({
+  "name": "Daniel",
+  "age": 42
+})"_padded;
+  ondemand::parser parser;
+  ondemand::document doc;
+  ASSERT_SUCCESS(parser.iterate(json).get(doc));
+
+  simdjson::ondemand::object main_object;
+  ASSERT_SUCCESS(doc.get_object().get(main_object));
+  std::string name;
+  ASSERT_SUCCESS(main_object["name"].get_string(name));
+  ASSERT_EQUAL(name, "Daniel");
+  uint64_t age;
+  ASSERT_SUCCESS(main_object["age"].get_uint64().get(age));
+  ASSERT_EQUAL(age, 42);
+  TEST_SUCCEED();
+}
+
+bool thirtysecondsofcode() {
+  TEST_START();
+  auto json = R"({
+  "name": "30-seconds-of-code",
+  "private": true,
+  "version": "10.0.0",
+  "description": "30 seconds of code website.",
+  "exports": "./index.js",
+  "author": "chalarangelo",
+  "type": "module",
+  "devDependencies": {
+    "@jsiqle/core": "^3.0.0",
+    "astro": "^3.2.0",
+    "chalk": "^5.3.0",
+    "eslint": "^8.50.0",
+    "eslint-config-prettier": "^9.0.0",
+    "front-matter": "^4.0.2",
+    "fs-extra": "^11.1.1",
+    "glob": "^10.3.10",
+    "hast-util-to-html": "^9.0.0",
+    "js-yaml": "^4.1.0",
+    "mdast-util-to-hast": "^13.0.2",
+    "prettier": "^3.0.3",
+    "prettier-plugin-astro": "^0.12.0",
+    "prismjs": "^1.29.0",
+    "remark": "^15.0.1",
+    "remark-gfm": "^4.0.0",
+    "sass": "^1.68.0",
+    "sharp": "^0.32.6",
+    "unist-util-select": "^5.0.0",
+    "unist-util-visit": "^5.0.0",
+    "unist-util-visit-parents": "^6.0.1",
+    "webfonts-generator": "^0.4.0"
+  },
+  "imports": {
+    "#blocks/*": "./src/blocks/*.js",
+    "#components/*": "./src/components/*.astro",
+    "#layouts/*": "./src/layouts/*.astro",
+    "#settings/*": "./src/settings/*.js",
+    "#prefabs": "./src/prefabs/index.js",
+    "#utils": "./src/utils/index.js",
+    "#utils/search": "./src/utils/search.js"
+  },
+  "scripts": {
+    "predev": "NODE_ENV=development node ./src/scripts/develop.js",
+    "dev": "astro dev --port 8000",
+    "start": "astro dev --port 8000",
+    "prebuild": "NODE_ENV=production node ./src/scripts/build.js",
+    "build": "astro build",
+    "preview": "astro preview --port 9000",
+    "watch": "NODE_ENV=development node ./src/scripts/watch.js",
+    "console": "NODE_ENV=production node ./src/scripts/console.js",
+    "create": "NODE_ENV=production node ./src/scripts/create.js",
+    "icons": "NODE_ENV=production node ./src/scripts/icons.js",
+    "manifest": "NODE_ENV=production node ./src/scripts/manifest.js"
+  },
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/30-seconds/30-seconds-of-code"
+  },
+  "bugs": {
+    "url": "https://github.com/30-seconds/30-seconds-of-code/issues"
+  },
+  "browserslist": [
+    "> 0.5% and last 4 versions and not dead and not ie>0 and not op_mini all and not and_uc>0 and not edge<79"
+  ],
+  "engines": {
+    "node": ">=18.14.2"
+  }
+})"_padded;
+  ondemand::parser parser;
+  ondemand::document doc;
+  ASSERT_SUCCESS(parser.iterate(json).get(doc));
+
+  simdjson::ondemand::object main_object;
+  ASSERT_SUCCESS(doc.get_object().get(main_object));
+
+  simdjson::ondemand::raw_json_string key;
+  simdjson::ondemand::value value;
+
+  for (auto field : main_object) {
+    // Throw error if getting key or value fails.
+    ASSERT_SUCCESS(field.key().get(key));
+    ASSERT_SUCCESS(field.value().get(value));
+
+    if (key == "name") {
+      std::string name;
+      ASSERT_SUCCESS(value.get_string(name));
+      ASSERT_EQUAL(name, "30-seconds-of-code");
+    } else if (key == "main") {
+      std::string main;
+      ASSERT_SUCCESS(value.get_string(main));
+      // unused
+    } else if (key == "exports") {
+      simdjson::ondemand::json_type exports_type;
+      if (!value.type().get(exports_type)) {
+        std::string_view exports;
+        switch (exports_type) {
+        case simdjson::ondemand::json_type::object: {
+          simdjson::ondemand::object exports_object;
+          if (!value.get_object().get(exports_object) &&
+              !exports_object.raw_json().get(exports)) {
+            // unused
+          }
+          break;
+        }
+        case simdjson::ondemand::json_type::array: {
+          simdjson::ondemand::array exports_array;
+          if (!value.get_array().get(exports_array) &&
+              !exports_array.raw_json().get(exports)) {
+            // unused
+          }
+          break;
+        }
+        case simdjson::ondemand::json_type::string: {
+          if (!value.get_string().get(exports)) {
+            ASSERT_EQUAL(exports, "./index.js");
+          }
+          break;
+        }
+        default:
+          break;
+        }
+      }
+    } else if (key == "imports") {
+      simdjson::ondemand::json_type imports_type;
+      if (!value.type().get(imports_type)) {
+        std::string_view imports;
+        switch (imports_type) {
+        case simdjson::ondemand::json_type::object: {
+          simdjson::ondemand::object imports_object;
+          if (!value.get_object().get(imports_object) &&
+              !imports_object.raw_json().get(imports)) {
+            ASSERT_EQUAL(imports, R"({
+    "#blocks/*": "./src/blocks/*.js",
+    "#components/*": "./src/components/*.astro",
+    "#layouts/*": "./src/layouts/*.astro",
+    "#settings/*": "./src/settings/*.js",
+    "#prefabs": "./src/prefabs/index.js",
+    "#utils": "./src/utils/index.js",
+    "#utils/search": "./src/utils/search.js"
+  })");
+          }
+          break;
+        }
+        case simdjson::ondemand::json_type::array: {
+          simdjson::ondemand::array imports_array;
+          if (!value.get_array().get(imports_array) &&
+              !imports_array.raw_json().get(imports)) {
+            // unused
+          }
+          break;
+        }
+        case simdjson::ondemand::json_type::string: {
+          if (!value.get_string().get(imports)) {
+            // unused
+          }
+          break;
+        }
+        default:
+          break;
+        }
+      }
+    } else if (key == "type") {
+      std::string_view type;
+      if (!value.get_string().get(type) &&
+          (type == "commonjs" || type == "module")) {
+        ASSERT_EQUAL(type, "module");
+      }
+    }
+  }
+  TEST_SUCCEED();
+}
+
+bool run() { return thirtysecondsofcode() && baby(); }
+
+} // namespace json_package_tests
+
+int main(int argc, char *argv[]) {
+  return test_main(argc, argv, json_package_tests::run);
+}

tools/release.py

@@ -167,8 +167,6 @@ print("the commandline is {}".format(cp.args))
 if(cp.returncode != 0):
     print("Failed to run doxygen")
 
-#ipe = subprocess.Popen(["doxygen"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=maindir)
-#doxygenresult = pipe.communicate()[0].decode().strip()
 
 pattern = re.compile("https://simdjson.org/api/(\d+\.\d+\.\d+)/index.html")
 readmefile = maindir + os.sep + "README.md"