Fixing typo

.github/workflows/documentation.yml

@@ -1,8 +1,9 @@
 name: Doxygen GitHub Pages
 
 on:
-  release:
-    types: [created]
+  push:
+    branches:
+      - master
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
 

.github/workflows/vs17-arm-ci.yml

@@ -18,4 +18,4 @@ jobs:
        - name: Use cmake
          run: |
            cmake -A ${{ matrix.arch }} -DCMAKE_SYSTEM_VERSION="10.0.22621.0" -DCMAKE_CROSSCOMPILING=1 -DSIMDJSON_DEVELOPER_MODE=ON -D SIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_EXCEPTIONS=OFF -B build  &&
-           cmake --build build --verbose
+           cmake --build build --verbose

CMakeLists.txt

@@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.14)
 project(
     simdjson
     # The version number is modified by tools/release.py
-    VERSION 3.11.2
+    VERSION 3.10.1
     DESCRIPTION "Parsing gigabytes of JSON per second"
     HOMEPAGE_URL "https://simdjson.org/"
     LANGUAGES CXX C
@@ -20,8 +20,8 @@ string(
 # ---- Options, variables ----
 
 # These version numbers are modified by tools/release.py
-set(SIMDJSON_LIB_VERSION "24.0.0" CACHE STRING "simdjson library version")
-set(SIMDJSON_LIB_SOVERSION "24" CACHE STRING "simdjson library soversion")
+set(SIMDJSON_LIB_VERSION "23.0.0" CACHE STRING "simdjson library version")
+set(SIMDJSON_LIB_SOVERSION "23" CACHE STRING "simdjson library soversion")
 
 option(SIMDJSON_BUILD_STATIC_LIB "Build simdjson_static library along with simdjson (only makes sense if BUILD_SHARED_LIBS=ON)" OFF)
 if(SIMDJSON_BUILD_STATIC_LIB AND NOT BUILD_SHARED_LIBS)

Doxyfile

@@ -38,7 +38,7 @@ PROJECT_NAME           = simdjson
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = "3.11.2"
+PROJECT_NUMBER         = "3.10.1"
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a

README.md

@@ -31,7 +31,6 @@ Table of Contents
 * [Documentation](#documentation)
 * [Godbolt](#godbolt)
 * [Performance results](#performance-results)
-* [Packages](#packages)
 * [Bindings and Ports of simdjson](#bindings-and-ports-of-simdjson)
 * [About simdjson](#about-simdjson)
 * [Funding](#funding)
@@ -144,9 +143,6 @@ speed for [synthetic files over various sizes generated with a script](https://g
 For NDJSON files, we can exceed 3 GB/s with [our  multithreaded parsing functions](https://github.com/simdjson/simdjson/blob/master/doc/parse_many.md).
 
 
-Packages
-------------------------------
-[![Packaging status](https://repology.org/badge/vertical-allrepos/simdjson.svg)](https://repology.org/project/simdjson/versions)
 
 
 Bindings and Ports of simdjson

doc/basics.md

@@ -209,25 +209,6 @@ std::string data = "my data";
 simdjson::padded_string my_padded_data(data); // copies to a padded buffer
 ```
 
-You can then parse the JSON data from the `simdjson::padded_string` instance:
-
-
-```c++
-ondemand::document doc = parser.iterate(my_padded_data);
-```
-
-Whenever you pass an `std::string` reference to `parser::iterate`,
-the parser will access the bytes beyond the end of
-the string but before the end of the allocated memory (`std::string::capacity()`).
-If you are using a sanitizer that checks for reading uninitialized bytes or `std::string`'s
-container-overflow checks, you may encounter sanitizer warnings.
-You can safely ignore these warnings. Or you can call `simdjson::pad(std::string&)` to pad the
-string with `SIMDJSON_PADDING` spaces: this function returns a `simdjson::padding_string_view` which can be be passed to the parser's iterator function:
-
-```c++
-std::string json = "[1]";
-ondemand::document doc = parser.iterate(simdjson::pad(json));
-```
 
 We recommend against creating many `std::string` or many `std::padding_string` instances in your application to store your JSON data.
 Consider reusing the same buffers and limiting memory allocations.
@@ -557,7 +538,7 @@ support for users who avoid exceptions. See [the simdjson error handling documen
   For this purpose, `array` instances have a `count_elements` method. Users should be
   aware that the `count_elements` method can be costly since it requires scanning the
   whole array. You should only call `count_elements` as a last resort as it may
-  require scanning the document twice or more. In the spirit of On-Demand, the `count_elements` function does not validate the values in the array. You may use it as follows if your document is itself an array:
+  require scanning the document twice or more. You may use it as follows if your document is itself an array:
 
   ```C++
   auto cars_json = R"( [ 40.1, 39.9, 37.7, 40.4 ] )"_padded;

doc/dom.md

@@ -60,26 +60,6 @@ std::string data = "my data";
 simdjson::padded_string my_padded_data(data); // copies to a padded buffer
 ```
 
-You can then parse the JSON document from the `simdjson::padded_string` instance:
-
-```cpp
-simdjson::dom::parser parser;
-simdjson::dom::element doc = parser.parse(my_padded_data);
-```
-
-Whenever you pass an `std::string` reference to `parser::parse`,
-the parser will access the bytes beyond the end of
-the string but before the end of the allocated memory (`std::string::capacity()`).
-If you are using a sanitizer that checks for reading uninitialized bytes or `std::string`'s
-container-overflow checks, you may encounter sanitizer warnings.
-You can safely ignore these warnings. Or you can call `simdjson::pad(std::string&)` to pad the
-string with `SIMDJSON_PADDING` spaces: this function returns a `simdjson::padding_string_view` which can be be passed to the parser's iterator function:
-
-```c++
-std::string json = "[1]";
-dom::element doc = parser.parse(simdjson::pad(json));
-```
-
 The parsed document resulting from the `parser.load` and `parser.parse` calls depends on the `parser` instance. Thus the `parser` instance must remain in scope. Furthermore, you must have at most one parsed document in play per `parser` instance.
 You cannot copy a `parser` instance, you may only move it.
 

include/simdjson/arm64/bitmanipulation.h

@@ -19,7 +19,7 @@ SIMDJSON_NO_SANITIZE_UNDEFINED
 // See issue https://github.com/simdjson/simdjson/issues/1965
 SIMDJSON_NO_SANITIZE_MEMORY
 simdjson_inline int trailing_zeroes(uint64_t input_num) {
-#if SIMDJSON_REGULAR_VISUAL_STUDIO
+#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
   unsigned long ret;
   // Search the mask data from least significant bit (LSB)
   // to the most significant bit (MSB) for a set bit (1).
@@ -43,7 +43,7 @@ simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) {
 SIMDJSON_NO_SANITIZE_UNDEFINED
 /* result might be undefined when input_num is zero */
 simdjson_inline int leading_zeroes(uint64_t input_num) {
-#if SIMDJSON_REGULAR_VISUAL_STUDIO
+#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
   unsigned long leading_zero = 0;
   // Search the mask data from most significant bit (MSB)
   // to least significant bit (LSB) for a set bit (1).
@@ -96,7 +96,7 @@ simdjson_inline uint64_t zero_leading_bit(uint64_t rev_bits, int leading_zeroes)
 #endif
 
 simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t *result) {
-#if SIMDJSON_REGULAR_VISUAL_STUDIO
+#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
   *result = value1 + value2;
   return *result < value1;
 #else

include/simdjson/arm64/implementation.h

@@ -23,6 +23,7 @@ public:
   ) const noexcept final;
   simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final;
   simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final;
+  simdjson_warn_unused size_t write_string_escaped(const std::string_view input, char *out) const noexcept final;
 };
 
 } // namespace arm64

include/simdjson/arm64/simd.h

@@ -12,7 +12,7 @@ namespace arm64 {
 namespace {
 namespace simd {
 
-#if SIMDJSON_REGULAR_VISUAL_STUDIO
+#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
 namespace {
 // Start of private section with Visual Studio workaround
 
@@ -121,7 +121,7 @@ namespace {
     // We return uint32_t instead of uint16_t because that seems to be more efficient for most
     // purposes (cutting it down to uint16_t costs performance in some compilers).
     simdjson_inline uint32_t to_bitmask() const {
-#if SIMDJSON_REGULAR_VISUAL_STUDIO
+#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
       const uint8x16_t bit_mask =  simdjson_make_uint8x16_t(0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80,
                                                    0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80);
 #else
@@ -134,6 +134,12 @@ namespace {
       tmp = vpaddq_u8(tmp, tmp);
       return vgetq_lane_u16(vreinterpretq_u16_u8(tmp), 0);
     }
+    // Returns 4-bit out of each byte, alternating between the high 4 bits and low
+    // bits result it is 64 bit.
+    simdjson_inline uint64_t to_bitmask64() const {
+      return vget_lane_u64(
+          vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(*this), 4)), 0);
+    }
     simdjson_inline bool any() const { return vmaxvq_u32(vreinterpretq_u32_u8(*this)) != 0; }
   };
 
@@ -152,7 +158,7 @@ namespace {
     // Splat constructor
     simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {}
     // Member-by-member initialization
-#if SIMDJSON_REGULAR_VISUAL_STUDIO
+#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
     simdjson_inline simd8(
       uint8_t v0,  uint8_t v1,  uint8_t v2,  uint8_t v3,  uint8_t v4,  uint8_t v5,  uint8_t v6,  uint8_t v7,
       uint8_t v8,  uint8_t v9,  uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15
@@ -210,7 +216,7 @@ namespace {
 
     // Bit-specific operations
     simdjson_inline simd8<bool> any_bits_set(simd8<uint8_t> bits) const { return vtstq_u8(*this, bits); }
-    simdjson_inline bool any_bits_set_anywhere() const { return this->max_val() != 0; }
+    simdjson_inline bool any_bits_set_anywhere() const { return vmaxvq_u32(vreinterpretq_u32_u8(*this)) != 0; }
     simdjson_inline bool any_bits_set_anywhere(simd8<uint8_t> bits) const { return (*this & bits).any_bits_set_anywhere(); }
     template<int N>
     simdjson_inline simd8<uint8_t> shr() const { return vshrq_n_u8(*this, N); }
@@ -223,7 +229,12 @@ namespace {
       return lookup_table.apply_lookup_16_to(*this);
     }
 
-
+    // Returns 4-bit out of each byte, alternating between the high 4 bits and low
+    // bits result it is 64 bit.
+    simdjson_inline uint64_t to_bitmask64() const {
+      return vget_lane_u64(
+          vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(*this), 4)), 0);
+    }
     // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset).
     // Passing a 0 value for mask would be equivalent to writing out every byte to output.
     // Only the first 16 - count_ones(mask) bytes of the result are significant but 16 bytes
@@ -246,7 +257,7 @@ namespace {
       uint64x2_t shufmask64 = {thintable_epi8[mask1], thintable_epi8[mask2]};
       uint8x16_t shufmask = vreinterpretq_u8_u64(shufmask64);
       // we increment by 0x08 the second half of the mask
-#if SIMDJSON_REGULAR_VISUAL_STUDIO
+#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
       uint8x16_t inc = simdjson_make_uint8x16_t(0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08);
 #else
       uint8x16_t inc = {0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08};
@@ -276,7 +287,7 @@ namespace {
       uint8x8_t compactmask1 = vcreate_u8(thintable_epi8[mask1]);
       uint8x8_t compactmask2 = vcreate_u8(thintable_epi8[mask2]);
       // we increment by 0x08 the second half of the mask
-#if SIMDJSON_REGULAR_VISUAL_STUDIO
+#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
       uint8x8_t inc = simdjson_make_uint8x8_t(0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08);
 #else
       uint8x8_t inc = {0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08};
@@ -328,7 +339,7 @@ namespace {
     // Array constructor
     simdjson_inline simd8(const int8_t* values) : simd8(load(values)) {}
     // Member-by-member initialization
-#if SIMDJSON_REGULAR_VISUAL_STUDIO
+#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
     simdjson_inline simd8(
       int8_t v0,  int8_t v1,  int8_t v2,  int8_t v3, int8_t v4,  int8_t v5,  int8_t v6,  int8_t v7,
       int8_t v8,  int8_t v9,  int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15
@@ -449,7 +460,7 @@ namespace {
     }
 
     simdjson_inline uint64_t to_bitmask() const {
-#if SIMDJSON_REGULAR_VISUAL_STUDIO
+#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
       const uint8x16_t bit_mask = simdjson_make_uint8x16_t(
         0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80,
         0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80

include/simdjson/arm64/stringparsing_defs.h

@@ -46,6 +46,32 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin
   };
 }
 
+struct escaping {
+  static constexpr uint32_t BYTES_PROCESSED = 16;
+  simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst);
+
+  simdjson_inline bool has_escape() { return escape_bits != 0; }
+  simdjson_inline int escape_index() { return trailing_zeroes(escape_bits) / 4; }
+
+  uint64_t escape_bits;
+}; // struct escaping
+
+
+
+simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) {
+  static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes");
+  simd8<uint8_t> v(src);
+  v.store(dst);
+  simd8<bool> is_quote = (v == '"');
+  simd8<bool> is_backslash = (v == '\\');
+  simd8<bool> is_control = (v < 32);
+  return {
+    (is_backslash | is_quote | is_control).to_bitmask64()
+  };
+}
+
+
+
 } // unnamed namespace
 } // namespace arm64
 } // namespace simdjson

include/simdjson/concepts.h

@@ -20,14 +20,14 @@ namespace details {
     };                                                                         \
   };
 
-SIMDJSON_IMPL_CONCEPT(emplace_back, emplace_back)
-SIMDJSON_IMPL_CONCEPT(emplace, emplace)
-SIMDJSON_IMPL_CONCEPT(push_back, push_back)
-SIMDJSON_IMPL_CONCEPT(add, add)
-SIMDJSON_IMPL_CONCEPT(push, push)
-SIMDJSON_IMPL_CONCEPT(append, append)
-SIMDJSON_IMPL_CONCEPT(insert, insert)
-SIMDJSON_IMPL_CONCEPT(op_append, operator+=)
+SIMDJSON_IMPL_CONCEPT(emplace_back, emplace_back);
+SIMDJSON_IMPL_CONCEPT(emplace, emplace);
+SIMDJSON_IMPL_CONCEPT(push_back, push_back);
+SIMDJSON_IMPL_CONCEPT(add, add);
+SIMDJSON_IMPL_CONCEPT(push, push);
+SIMDJSON_IMPL_CONCEPT(append, append);
+SIMDJSON_IMPL_CONCEPT(insert, insert);
+SIMDJSON_IMPL_CONCEPT(op_append, operator+=);
 
 #undef SIMDJSON_IMPL_CONCEPT
 } // namespace details

include/simdjson/dom/element.h

@@ -372,8 +372,6 @@ public:
    *         - INCORRECT_TYPE if this is not an object
    */
   inline simdjson_result<element> operator[](const char *key) const noexcept;
-  simdjson_result<element> operator[](int) const noexcept = delete;
-
 
   /**
    * Get the value associated with the given JSON pointer.  We use the RFC 6901
@@ -542,7 +540,6 @@ public:
 
   simdjson_inline simdjson_result<dom::element> operator[](std::string_view key) const noexcept;
   simdjson_inline simdjson_result<dom::element> operator[](const char *key) const noexcept;
-  simdjson_result<dom::element> operator[](int) const noexcept = delete;
   simdjson_inline simdjson_result<dom::element> at_pointer(const std::string_view json_pointer) const noexcept;
   simdjson_inline simdjson_result<dom::element> at_path(const std::string_view json_path) const noexcept;
   [[deprecated("For standard compliance, use at_pointer instead, and prefix your pointers with a slash '/', see RFC6901 ")]]

include/simdjson/dom/object.h

@@ -145,7 +145,6 @@ public:
    *         - INCORRECT_TYPE if this is not an object
    */
   inline simdjson_result<element> operator[](const char *key) const noexcept;
-  simdjson_result<element> operator[](int) const noexcept = delete;
 
   /**
    * Get the value associated with the given JSON pointer. We use the RFC 6901
@@ -259,7 +258,6 @@ public:
 
   inline simdjson_result<dom::element> operator[](std::string_view key) const noexcept;
   inline simdjson_result<dom::element> operator[](const char *key) const noexcept;
-  simdjson_result<dom::element> operator[](int) const noexcept = delete;
   inline simdjson_result<dom::element> at_pointer(std::string_view json_pointer) const noexcept;
   inline simdjson_result<dom::element> at_path(std::string_view json_path) const noexcept;
   inline simdjson_result<dom::element> at_key(std::string_view key) const noexcept;

include/simdjson/dom/parser.h

@@ -202,22 +202,6 @@ public:
    *   simdjson::dom::parser parser;
    *   simdjson::dom::element element = parser.parse(padded_json_copy.get(), json_len, false);
    *
-   * ### std::string references
-   *
-   * If you pass a mutable std::string reference (std::string&), the parser will seek to extend
-   * its capacity to SIMDJSON_PADDING bytes beyond the end of the string.
-   *
-   * Whenever you pass an std::string reference, the parser will access the bytes beyond the end of
-   * the string but before the end of the allocated memory (std::string::capacity()).
-   * If you are using a sanitizer that checks for reading uninitialized bytes or std::string's
-   * container-overflow checks, you may encounter sanitizer warnings.
-   * You can safely ignore these warnings. Or you can call simdjson::pad(std::string&) to pad the
-   * string with SIMDJSON_PADDING spaces: this function returns a simdjson::padding_string_view
-   * which can be be passed to the parser's parse function:
-   *
-   *    std::string json = R"({ "foo": 1 } { "foo": 2 } { "foo": 3 } )";
-   *    element doc = parser.parse(simdjson::pad(json));
-   *
    * ### Parser Capacity
    *
    * If the parser's current capacity is less than len, it will allocate enough capacity

include/simdjson/error.h

@@ -49,7 +49,8 @@ enum error_code {
   SCALAR_DOCUMENT_AS_VALUE,   ///< A scalar document is treated as a value.
   OUT_OF_BOUNDS,              ///< Attempted to access location outside of document.
   TRAILING_CONTENT,           ///< Unexpected trailing content in the JSON input
-  NUM_ERROR_CODES
+  OUT_OF_CAPACITY,            ///< The capacity was exceeded, we cannot allocate enough memory.
+  NUM_ERROR_CODES             ///< Placeholder for end of error code list.
 };
 
 /**
@@ -86,7 +87,7 @@ struct simdjson_error : public std::exception {
    */
   simdjson_error(error_code error) noexcept : _error{error} { }
   /** The error message */
-  const char *what() const noexcept override { return error_message(error()); }
+  const char *what() const noexcept { return error_message(error()); }
   /** The error code */
   error_code error() const noexcept { return _error; }
 private:

include/simdjson/fallback/implementation.h

@@ -26,6 +26,7 @@ public:
   ) const noexcept final;
   simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final;
   simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final;
+  simdjson_warn_unused size_t write_string_escaped(const std::string_view input, char *out) const noexcept final;
 };
 
 } // namespace fallback

include/simdjson/fallback/stringparsing_defs.h

@@ -29,6 +29,24 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin
   return { src[0] };
 }
 
+
+struct escaping {
+  static constexpr uint32_t BYTES_PROCESSED = 1;
+  simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst);
+
+  simdjson_inline bool has_escape() { return escape_bits; }
+  simdjson_inline int escape_index() { return 0; }
+
+  bool escape_bits;
+}; // struct escaping
+
+
+
+simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) {
+  dst[0] = src[0];
+  return { (src[0] == '\\') || (src[0] == '"') || (src[0] < 32) };
+}
+
 } // unnamed namespace
 } // namespace fallback
 } // namespace simdjson

include/simdjson/generic/ondemand/amalgamated.h

@@ -45,4 +45,8 @@
 #include "simdjson/generic/ondemand/token_iterator-inl.h"
 #include "simdjson/generic/ondemand/value_iterator-inl.h"
 
+// JSON builder, ideally they should not be part of the ondemand directory
+// but it is convenient for now to have them here.
+#include "simdjson/generic/ondemand/json_string_builder.h"
+#include "simdjson/generic/ondemand/json_string_builder-inl.h"
 

include/simdjson/generic/ondemand/array.h

@@ -44,8 +44,7 @@ public:
    * calling this function, if successful, the array is 'rewinded' at its
    * beginning as if it had never been accessed. If the JSON is malformed (e.g.,
    * there is a missing comma), then an error is returned and it is no longer
-   * safe to continue. Note that count_elements() does not validate the JSON values,
-   * only the structure of the array.
+   * safe to continue.
    *
    * To check that an array is empty, it is more performant to use
    * the is_empty() method.

include/simdjson/generic/ondemand/document.h

@@ -354,8 +354,7 @@ public:
    * calling this function, if successful, the array is 'rewinded' at its
    * beginning as if it had never been accessed. If the JSON is malformed (e.g.,
    * there is a missing comma), then an error is returned and it is no longer
-   * safe to continue. Note that count_elements() does not validate the JSON values,
-   * only the structure of the array.
+   * safe to continue.
    */
   simdjson_inline simdjson_result<size_t> count_elements() & noexcept;
    /**
@@ -467,7 +466,6 @@ public:
   simdjson_inline simdjson_result<value> operator[](std::string_view key) & noexcept;
   /** @overload simdjson_inline simdjson_result<value> find_field_unordered(std::string_view key) & noexcept; */
   simdjson_inline simdjson_result<value> operator[](const char *key) & noexcept;
- simdjson_result<value> operator[](int) & noexcept = delete;
 
   /**
    * Get the type of this JSON value. It does not validate or consume the value.
@@ -854,7 +852,6 @@ public:
   simdjson_inline simdjson_result<value> find_field(const char *key) & noexcept;
   simdjson_inline simdjson_result<value> operator[](std::string_view key) & noexcept;
   simdjson_inline simdjson_result<value> operator[](const char *key) & noexcept;
-  simdjson_result<value> operator[](int) & noexcept = delete;
   simdjson_inline simdjson_result<value> find_field_unordered(std::string_view key) & noexcept;
   simdjson_inline simdjson_result<value> find_field_unordered(const char *key) & noexcept;
 
@@ -933,7 +930,6 @@ public:
   simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field(const char *key) & noexcept;
   simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> operator[](std::string_view key) & noexcept;
   simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> operator[](const char *key) & noexcept;
-  simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> operator[](int) & noexcept = delete;
   simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field_unordered(std::string_view key) & noexcept;
   simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field_unordered(const char *key) & noexcept;
   simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::json_type> type() noexcept;
@@ -1011,7 +1007,6 @@ public:
   simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field(const char *key) & noexcept;
   simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> operator[](std::string_view key) & noexcept;
   simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> operator[](const char *key) & noexcept;
-  simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> operator[](int) & noexcept = delete;
   simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field_unordered(std::string_view key) & noexcept;
   simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field_unordered(const char *key) & noexcept;
   simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::json_type> type() noexcept;

include/simdjson/generic/ondemand/json_string_builder-inl.h

@@ -0,0 +1,281 @@
+/**
+ * This file is part of the builder API. It is temporarily in the ondemand directory
+ * but we will move it to a builder directory later.
+ */
+#include <type_traits>
+#ifndef SIMDJSON_GENERIC_BUILDER_INL_H
+
+#ifndef SIMDJSON_CONDITIONAL_INCLUDE
+#define SIMDJSON_GENERIC_BUILDER_INL_H
+#include "simdjson/generic/builder/json_string_builder.h"
+#endif // SIMDJSON_CONDITIONAL_INCLUDE
+
+namespace simdjson {
+namespace SIMDJSON_IMPLEMENTATION {
+namespace builder {
+
+simdjson_inline string_builder::string_builder(size_t initial_capacity) :
+  buffer(new (std::nothrow) char[initial_capacity]),
+  position(0), capacity(buffer.get() != nullptr ? initial_capacity : 0),
+  is_valid(buffer.get() != nullptr) {}
+
+simdjson_inline bool string_builder::capacity_check(size_t upcoming_bytes) {
+  // We use the convention that when is_valid is false, then the capacity and
+  // the position are 0.
+  // Most of the time, this function will return true.
+  if (simdjson_likely(upcoming_bytes <= capacity - position)) { return true; }
+  // check for overflow, most of the time there is no overflow
+  if (simdjson_likely(position + upcoming_bytes < position)) { return false; }
+  // We will rarely get here.
+  grow_buffer((std::max)(capacity * 2, position + upcoming_bytes));
+  // If the buffer allocation failed, we set is_valid to false.
+  return is_valid;
+}
+
+simdjson_inline void string_builder::grow_buffer(size_t desired_capacity) {
+  if (!is_valid) { return; }
+  std::unique_ptr<char[]> new_buffer(new (std::nothrow) char[desired_capacity]);
+  if (new_buffer.get() == nullptr) {
+    set_valid(false);
+    return;
+  }
+  std::memcpy(new_buffer.get(), buffer.get(), position);
+  buffer.swap(new_buffer);
+  capacity = desired_capacity;
+}
+
+simdjson_inline void string_builder::set_valid(bool valid) noexcept {
+  if(!valid) {
+    is_valid = false;
+    capacity = 0;
+    position = 0;
+    buffer.reset();
+  } else {
+    is_valid = true;
+  }
+}
+
+simdjson_inline size_t string_builder::size() const noexcept {
+  return position;
+}
+
+
+simdjson_inline void string_builder::append(char c) noexcept {
+    if(capacity_check(1)) {
+      buffer.get()[position++] = c;
+    }
+}
+
+
+simdjson_inline void string_builder::append_null() noexcept {
+  constexpr char null_literal[] = "null";
+  constexpr size_t null_len = sizeof(null_literal) - 1;
+  if(capacity_check(null_len)) {
+    std::memcpy(buffer.get() + position, null_literal, null_len);
+    position += null_len;
+  }
+}
+
+simdjson_inline void string_builder::clear()  noexcept {
+  position = 0;
+  // if it was invalid, we should try to repair it
+  if(!is_valid) {
+    capacity = 0;
+    buffer.reset();
+    is_valid = true;
+  }
+}
+
+namespace internal {
+
+// We could specialize further for 32-bit integers.
+int int_log2(uint32_t x) { return (63 - leading_zeroes(x | 1)); }
+
+int fast_digit_count_32(uint32_t x) {
+  static uint64_t table[] = {
+      4294967296,  8589934582,  8589934582,  8589934582,  12884901788,
+      12884901788, 12884901788, 17179868184, 17179868184, 17179868184,
+      21474826480, 21474826480, 21474826480, 21474826480, 25769703776,
+      25769703776, 25769703776, 30063771072, 30063771072, 30063771072,
+      34349738368, 34349738368, 34349738368, 34349738368, 38554705664,
+      38554705664, 38554705664, 41949672960, 41949672960, 41949672960,
+      42949672960, 42949672960};
+  return uint32_t((x + table[int_log2(x)]) >> 32);
+}
+
+int int_log2(uint64_t x) { return 63 - leading_zeroes(x | 1); }
+
+int digit_count_64(uint64_t x) {
+  static uint64_t table[] = {9,
+                             99,
+                             999,
+                             9999,
+                             99999,
+                             999999,
+                             9999999,
+                             99999999,
+                             999999999,
+                             9999999999,
+                             99999999999,
+                             999999999999,
+                             9999999999999,
+                             99999999999999,
+                             999999999999999ULL,
+                             9999999999999999ULL,
+                             99999999999999999ULL,
+                             999999999999999999ULL,
+                             9999999999999999999ULL};
+  int y = (19 * int_log2(x) >> 6);
+  y += x > table[y];
+  return y + 1;
+}
+
+template<typename number_type,
+         typename = typename std::enable_if<std::is_unsigned<number_type>::value>::type>
+simdjson_inline size_t digit_count(number_type v) noexcept {
+  static_assert(sizeof(number_type) == 8
+   || sizeof(number_type) == 4
+   || sizeof(number_type) == 2
+   || sizeof(number_type) == 1, "We only support 8-bit, 16-bit, 32-bit and 64-bit numbers");
+  if (sizeof(number_type) <= 4) {
+    return fast_digit_count_32(v);
+  } else {
+    return digit_count_64(v);
+  }
+}
+
+} // internal
+
+template<typename number_type, typename>
+simdjson_inline void string_builder::append(number_type v) noexcept {
+  static_assert(std::is_same<number_type, bool>::value
+    || std::is_integral<number_type>::value || std::is_floating_point<number_type>::value, "Unsupported number type");
+  // If C++17 is available, we can 'if constexpr' here.
+  if constexpr (std::is_same<number_type, bool>::value) {
+    if (v) {
+      constexpr char true_literal[] = "true";
+      constexpr size_t true_len = sizeof(true_literal) - 1;
+      if(capacity_check(true_len)) {
+        std::memcpy(buffer.get() + position, true_literal, true_len);
+        position += true_len;
+      }
+    } else {
+      constexpr char false_literal[] = "false";
+      constexpr size_t false_len = sizeof(false_literal) - 1;
+      if(capacity_check(false_len)) {
+        std::memcpy(buffer.get() + position, false_literal, false_len);
+        position += false_len;
+      }
+    }
+  } else if constexpr (std::is_unsigned<number_type>::value) {
+    constexpr size_t max_number_size = 20;
+    if(capacity_check(max_number_size)) {
+      using unsigned_type = typename std::make_unsigned<number_type>::type;
+      unsigned_type pv = static_cast<unsigned_type>(v);
+      size_t dc = internal::digit_count(pv);
+      char *write_pointer = buffer.get() + position + dc - 1;
+      // optimization opportunity: if v is large, we can do better.
+      while(pv >= 10) {
+        *write_pointer-- = char('0' + (pv % 10));
+        pv /= 10;
+      }
+      *write_pointer = char('0' + pv);
+      position += dc;
+    }
+  } else if constexpr (std::is_integral<number_type>::value) {
+    constexpr size_t max_number_size = 20;
+    if(capacity_check(max_number_size)) {
+      using unsigned_type = typename std::make_unsigned<number_type>::type;
+      bool negative = v < 0;
+      unsigned_type pv = static_cast<unsigned_type>(negative ? -v : v);
+      size_t dc = internal::digit_count(pv);
+      if(negative) {
+        buffer.get()[position++] = '-';
+      }
+      char *write_pointer = buffer.get() + position + dc - 1;
+      // optimization opportunity: if v is large, we can do better.
+      while(pv >= 10) {
+        *write_pointer-- = char('0' + (pv % 10));
+        pv /= 10;
+      }
+      *write_pointer = char('0' + pv);
+      position += dc;
+    }
+  } else if constexpr (std::is_floating_point<number_type>::value) {
+    constexpr size_t max_number_size = 24;
+    if(capacity_check(max_number_size)) {
+      // We could specialize for float.
+      char *end = simdjson::internal::to_chars(buffer.get() + position, nullptr, double(v));
+      position = end - buffer.get();
+    }
+  }
+}
+
+simdjson_inline void string_builder::escape_and_append(std::string_view input)  noexcept {
+  // escaping might turn a control character into \x00xx so 6 characters.
+  if(capacity_check(6 * input.size())) {
+    position += simdjson::write_string_escaped(input, buffer.get() + position);
+  }
+}
+
+simdjson_inline void string_builder::escape_and_append_with_quotes(std::string_view input)  noexcept {
+  // escaping might turn a control character into \x00xx so 6 characters.
+  if(capacity_check(2 + 6 * input.size())) {
+    buffer.get()[position++] = '"';
+    position += simdjson::write_string_escaped(input, buffer.get() + position);
+    buffer.get()[position++] = '"';
+  }
+}
+
+simdjson_inline void string_builder::append_raw(const char *c)  noexcept {
+  if(capacity_check(1)) {
+    buffer.get()[position++] = *c;
+  }
+}
+
+simdjson_inline void string_builder::append_raw(std::string_view input)  noexcept {
+  if(capacity_check(input.size())) {
+    std::memcpy(buffer.get() + position, input.data(), input.size());
+    position += input.size();
+  }
+}
+
+simdjson_inline void string_builder::append_raw(const char *str, size_t len)  noexcept {
+  if(capacity_check(len)) {
+    std::memcpy(buffer.get() + position, str, len);
+    position += len;
+  }
+}
+
+#if SIMDJSON_EXCEPTIONS
+simdjson_inline string_builder::operator std::string() const noexcept(false) {
+  return std::string(std::string_view());
+}
+
+simdjson_inline string_builder::operator std::string_view() const noexcept(false) {
+  return view();
+}
+#endif
+
+simdjson_inline simdjson_result<std::string_view> string_builder::view() const noexcept {
+  if (!is_valid) { return simdjson::OUT_OF_CAPACITY; }
+  return std::string_view(buffer.get(), position);
+}
+
+simdjson_inline simdjson_result<const char *> string_builder::c_str() noexcept {
+  if(capacity_check(1)) {
+    buffer.get()[position] = '\0';
+    return buffer.get();
+  }
+  return simdjson::OUT_OF_CAPACITY;
+}
+
+simdjson_inline bool string_builder::validate_unicode() const noexcept {
+  return simdjson::validate_utf8(buffer.get(), position);
+}
+
+} // namespace builder
+} // namespace SIMDJSON_IMPLEMENTATION
+} // namespace simdjson
+
+#endif // SIMDJSON_GENERIC_BUILDER_INL_H

include/simdjson/generic/ondemand/json_string_builder.h

@@ -0,0 +1,167 @@
+/**
+ * This file is part of the builder API. It is temporarily in the ondemand directory
+ * but we will move it to a builder directory later.
+ */
+#ifndef SIMDJSON_GENERIC_BUILDER_H
+
+#ifndef SIMDJSON_CONDITIONAL_INCLUDE
+#define SIMDJSON_GENERIC_BUILDER_H
+#include "simdjson/generic/implementation_simdjson_result_base.h"
+#endif // SIMDJSON_CONDITIONAL_INCLUDE
+
+namespace simdjson {
+namespace SIMDJSON_IMPLEMENTATION {
+namespace builder {
+
+/**
+ * A builder for JSON strings representing documents. This is a low-level
+ * builder that is not meant to be used directly by end-users. Though it
+ * supports atomic types (Booleans, strings), it does not support composed
+ * types (arrays and objects).
+ *
+ * Ultimately, this class should support kernel-specific optimizations. E.g.,
+ * it may make use of SIMD instructions to escape strings faster.
+ */
+class string_builder {
+public:
+  simdjson_inline string_builder(size_t initial_capacity = 1024);
+
+  /**
+   * Append number (includes Booleans). Booleans are mapped to the strings
+   * false and true. Numbers are converted to strings abiding by the JSON standard.
+   * Floating-point numbers are converted to the shortest string that 'correctly'
+   * represents the number.
+   */
+  template<typename number_type,
+         typename = typename std::enable_if<std::is_arithmetic<number_type>::value>::type>
+  simdjson_inline void append(number_type v) noexcept;
+
+  /**
+   * Append character c.
+   */
+  simdjson_inline void append(char c)  noexcept;
+
+  /**
+   * Append the string 'null'.
+   */
+  simdjson_inline void append_null()  noexcept;
+
+  /**
+   * Clear the content.
+   */
+  simdjson_inline void clear()  noexcept;
+
+  /**
+   * Append the std::string_view, after escaping it.
+   * There is no UTF-8 validation.
+   */
+  simdjson_inline void escape_and_append(std::string_view input)  noexcept;
+
+  /**
+   * Append the std::string_view surrounded by double quotes, after escaping it.
+   * There is no UTF-8 validation.
+   */
+  simdjson_inline void escape_and_append_with_quotes(std::string_view input)  noexcept;
+
+  /**
+   * Append the C string directly, without escaping.
+   * There is no UTF-8 validation.
+   */
+  simdjson_inline void append_raw(const char *c)  noexcept;
+
+  /**
+   * Append the std::string_view directly, without escaping.
+   * There is no UTF-8 validation.
+   */
+  simdjson_inline void append_raw(std::string_view input)  noexcept;
+
+  /**
+   * Append len characters from str.
+   * There is no UTF-8 validation.
+   */
+  simdjson_inline void append_raw(const char *str, size_t len) noexcept;
+#if SIMDJSON_EXCEPTIONS
+  /**
+   * Creates an std::string from the written JSON buffer.
+   * Throws if memory allocation failed
+   *
+   * The result may not be valid UTF-8 if some of your content was not valid UTF-8.
+   * Use validate_unicode() to check the content if needed.
+   */
+  simdjson_inline operator std::string() const noexcept(false);
+
+  /**
+   * Creates an std::string_view from the written JSON buffer.
+   * Throws if memory allocation failed.
+   *
+   * The result may not be valid UTF-8 if some of your content was not valid UTF-8.
+   * Use validate_unicode() to check the content if needed.
+   */
+  simdjson_inline operator std::string_view() const noexcept(false);
+#endif
+
+  /**
+   * Returns a view on the written JSON buffer. Returns an error
+   * if memory allocation failed.
+   *
+   * The result may not be valid UTF-8 if some of your content was not valid UTF-8.
+   * Use validate_unicode() to check the content.
+   */
+  simdjson_inline simdjson_result<std::string_view> view() const noexcept;
+
+  /**
+   * Appends the null character to the buffer and returns
+   * a pointer to the beginning of the written JSON buffer.
+   * Returns an error if memory allocation failed.
+   * The result is null-terminated.
+   *
+   * The result may not be valid UTF-8 if some of your content was not valid UTF-8.
+   * Use validate_unicode() to check the content.
+   */
+  simdjson_inline simdjson_result<const char *> c_str() noexcept;
+
+  /**
+   * Return true if the content is valid UTF-8.
+   */
+  simdjson_inline bool validate_unicode() const noexcept;
+
+  /**
+   * Returns the current size of the written JSON buffer.
+   * If an error occurred, returns 0.
+   */
+  simdjson_inline size_t size() const noexcept;
+
+private:
+  /**
+   * Returns true if we can write at least upcoming_bytes bytes.
+   * The underlying buffer is reallocated if needed. It is designed
+   * to be called before writing to the buffer. It should be fast.
+   */
+  simdjson_inline bool capacity_check(size_t upcoming_bytes);
+
+  /**
+   * Grow the buffer to at least desired_capacity bytes.
+   * If the allocation fails, is_valid is set to false. We expect
+   * that this function would not be repeatedly called.
+   */
+  simdjson_inline void grow_buffer(size_t desired_capacity);
+
+  /**
+   * We use this helper function to make sure that is_valid is kept consistent.
+   */
+  simdjson_inline void set_valid(bool valid) noexcept;
+
+
+  std::unique_ptr<char[]> buffer{};
+  size_t position{0};
+  size_t capacity{0};
+  bool is_valid{true};
+};
+
+
+
+}
+}
+} // namespace simdjson
+
+#endif // SIMDJSON_GENERIC_BUILDER_H

include/simdjson/generic/ondemand/parser.h

@@ -84,22 +84,6 @@ public:
    * using a sanitizer that verifies that no uninitialized byte is read, then you should initialize the
    * SIMDJSON_PADDING bytes to avoid runtime warnings.
    *
-   * ### std::string references
-   *
-   * If you pass a mutable std::string reference (std::string&), the parser will seek to extend
-   * its capacity to SIMDJSON_PADDING bytes beyond the end of the string.
-   *
-   * Whenever you pass an std::string reference, the parser will access the bytes beyond the end of
-   * the string but before the end of the allocated memory (std::string::capacity()).
-   * If you are using a sanitizer that checks for reading uninitialized bytes or std::string's
-   * container-overflow checks, you may encounter sanitizer warnings.
-   * You can safely ignore these warnings. Or you can call simdjson::pad(std::string&) to pad the
-   * string with SIMDJSON_PADDING spaces: this function returns a simdjson::padding_string_view
-   * which can be be passed to the parser's iterate function:
-   *
-   *    std::string json = R"({ "foo": 1 } { "foo": 2 } { "foo": 3 } )";
-   *    document doc = parser.iterate(simdjson::pad(json));
-   *
    * @param json The JSON to parse.
    * @param len The length of the JSON.
    * @param capacity The number of bytes allocated in the JSON (must be at least len+SIMDJSON_PADDING).

include/simdjson/generic/ondemand/value.h

@@ -163,17 +163,6 @@ public:
    * Important: a value should be consumed once. Calling get_string() twice on the same value
    * is an error.
    *
-   * In some instances, you may want to allow replacement of invalid Unicode sequences.
-   * You may do so by passing the allow_replacement parameter as true. In the following
-   * example, the string "431924697b\udff0L\u0001Y" is not valid Unicode. By passing true
-   * to get_string, we allow the replacement of the invalid Unicode sequences with the Unicode
-   * replacement character (U+FFFD).
-   *
-   *   simdjson::ondemand::parser parser;
-   *   auto json = R"({"deviceId":"431924697b\udff0L\u0001Y"})"_padded;
-   *   simdjson::ondemand::document doc = parser.iterate(json);
-   *   auto view = doc["deviceId"].get_string(true);
-   *
    * @returns An UTF-8 string. The string is stored in the parser and will be invalidated the next
    *          time it parses a document or when it is destroyed.
    * @returns INCORRECT_TYPE if the JSON value is not a string.
@@ -425,7 +414,6 @@ public:
   simdjson_inline simdjson_result<value> operator[](std::string_view key) noexcept;
   /** @overload simdjson_inline simdjson_result<value> find_field_unordered(std::string_view key) noexcept; */
   simdjson_inline simdjson_result<value> operator[](const char *key) noexcept;
-  simdjson_result<value> operator[](int) noexcept = delete;
 
   /**
    * Get the type of this JSON value. It does not validate or consume the value.
@@ -793,7 +781,6 @@ public:
   simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> operator[](std::string_view key) noexcept;
   /** @overload simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field_unordered(std::string_view key) noexcept; */
   simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> operator[](const char *key) noexcept;
-  simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> operator[](int) noexcept = delete;
 
   /**
    * Get the type of this JSON value.

include/simdjson/haswell/implementation.h

@@ -28,6 +28,7 @@ public:
   ) const noexcept final;
   simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final;
   simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final;
+  simdjson_warn_unused size_t write_string_escaped(const std::string_view input, char *out) const noexcept final;
 };
 
 } // namespace haswell

include/simdjson/haswell/stringparsing_defs.h

@@ -41,6 +41,31 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin
   };
 }
 
+
+struct escaping {
+  static constexpr uint32_t BYTES_PROCESSED = 32;
+  simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst);
+
+  simdjson_inline bool has_escape() { return escape_bits != 0; }
+  simdjson_inline int escape_index() { return trailing_zeroes(escape_bits); }
+
+  uint64_t escape_bits;
+}; // struct escaping
+
+
+
+simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) {
+  static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes");
+  simd8<uint8_t> v(src);
+  v.store(dst);
+  simd8<bool> is_quote = (v == '"');
+  simd8<bool> is_backslash = (v == '\\');
+  simd8<bool> is_control = (v < 32);
+  return {
+    uint64_t((is_backslash | is_quote | is_control).to_bitmask())
+  };
+}
+
 } // unnamed namespace
 } // namespace haswell
 } // namespace simdjson

include/simdjson/icelake/implementation.h

@@ -28,6 +28,7 @@ public:
   ) const noexcept final;
   simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final;
   simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final;
+  simdjson_warn_unused size_t write_string_escaped(const std::string_view input, char *out) const noexcept final;
 };
 
 } // namespace icelake

include/simdjson/icelake/simd.h

@@ -77,7 +77,6 @@ namespace simd {
     friend simdjson_really_inline uint64_t operator==(const simd8<T> lhs, const simd8<T> rhs) {
       return _mm512_cmpeq_epi8_mask(lhs, rhs);
     }
-
     static const int SIZE = sizeof(base<T>::value);
 
     template<int N=1>

include/simdjson/icelake/stringparsing_defs.h

@@ -41,6 +41,35 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin
   };
 }
 
+
+
+struct escaping {
+  static constexpr uint32_t BYTES_PROCESSED = 64;
+  simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst);
+
+  simdjson_inline bool has_escape() { return escape_bits != 0; }
+  simdjson_inline int escape_index() { return trailing_zeroes(uint64_t(escape_bits)); }
+
+  __mmask64 escape_bits;
+}; // struct escaping
+
+
+
+simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) {
+  static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes");
+  simd8<uint8_t> v(src);
+  v.store(dst);
+  __mmask64 is_quote = _mm512_cmpeq_epi8_mask(v, _mm512_set1_epi8('"'));
+  __mmask64 is_backslash = _mm512_cmpeq_epi8_mask(v, _mm512_set1_epi8('\\'));
+  __mmask64 is_control = _mm512_cmplt_epi8_mask(v, _mm512_set1_epi8(32));
+  return {
+    (is_backslash | is_quote | is_control)
+  };
+}
+
+
+
+
 } // unnamed namespace
 } // namespace icelake
 } // namespace simdjson

include/simdjson/implementation.h

@@ -26,6 +26,15 @@ simdjson_inline simdjson_warn_unused bool validate_utf8(const std::string_view s
   return validate_utf8(sv.data(), sv.size());
 }
 
+/**
+ * Write the string to the output buffer while escaping double-quote, backlash and ascii control characters.
+ *
+ * @param input the string_view to escape
+ * @param out output buffer (for escaped string): to be safe, it should have 6 * input.size() allocated bytes.
+ * @return number of bytes written
+ */
+simdjson_warn_unused size_t write_string_escaped(const std::string_view input, char *out) noexcept;
+
 /**
  * Validate the UTF-8 string.
  *
@@ -127,6 +136,14 @@ public:
    */
   simdjson_warn_unused virtual bool validate_utf8(const char *buf, size_t len) const noexcept = 0;
 
+  /**
+   * Write the string to the output buffer while escaping double-quote, backlash and ascii control characters.
+   *
+   * @param input the string_view to escape
+   * @param out output buffer (for escaped string): to be safe, it should have 6 * input.size() allocated bytes.
+   * @return number of bytes written
+   */
+  simdjson_warn_unused virtual size_t write_string_escaped(const std::string_view input, char *out) const noexcept = 0;
 protected:
   /** @private Construct an implementation with the given name and description. For subclasses. */
   simdjson_inline implementation(

include/simdjson/lasx/implementation.h

@@ -23,6 +23,7 @@ public:
   ) const noexcept final;
   simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final;
   simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final;
+  simdjson_warn_unused size_t write_string_escaped(const std::string_view input, char *out) const noexcept final;
 };
 
 } // namespace lasx

include/simdjson/lasx/stringparsing_defs.h

@@ -40,6 +40,31 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin
   };
 }
 
+
+struct escaping {
+  static constexpr uint32_t BYTES_PROCESSED = 16;
+  simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst);
+
+  simdjson_inline bool has_escape() { return escape_bits != 0; }
+  simdjson_inline int escape_index() { return trailing_zeroes(escape_bits); }
+
+  uint64_t escape_bits;
+}; // struct escaping
+
+
+
+simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) {
+  static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes");
+  simd8<uint8_t> v(src);
+  v.store(dst);
+  simd8<bool> is_quote = (v == '"');
+  simd8<bool> is_backslash = (v == '\\');
+  simd8<bool> is_control = (v < 32);
+  return {
+    (is_backslash | is_quote | is_control).to_bitmask()
+  };
+}
+
 } // unnamed namespace
 } // namespace lasx
 } // namespace simdjson

include/simdjson/lsx/implementation.h

@@ -23,6 +23,7 @@ public:
   ) const noexcept final;
   simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final;
   simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final;
+  simdjson_warn_unused size_t write_string_escaped(const std::string_view input, char *out) const noexcept final;
 };
 
 } // namespace lsx

include/simdjson/lsx/stringparsing_defs.h

@@ -46,6 +46,31 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin
   };
 }
 
+
+struct escaping {
+  static constexpr uint32_t BYTES_PROCESSED = 16;
+  simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst);
+
+  simdjson_inline bool has_escape() { return escape_bits != 0; }
+  simdjson_inline int escape_index() { return trailing_zeroes(escape_bits); }
+
+  uint64_t escape_bits;
+}; // struct escaping
+
+
+
+simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) {
+  static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes");
+  simd8<uint8_t> v(src);
+  v.store(dst);
+  simd8<bool> is_quote = (v == '"');
+  simd8<bool> is_backslash = (v == '\\');
+  simd8<bool> is_control = (v < 32);
+  return {
+    (is_backslash | is_quote | is_control).to_bitmask()
+  };
+}
+
 } // unnamed namespace
 } // namespace lsx
 } // namespace simdjson

include/simdjson/ondemand.h

@@ -8,6 +8,10 @@ namespace simdjson {
    * @copydoc simdjson::builtin::ondemand
    */
   namespace ondemand = builtin::ondemand;
+  /**
+   * @copydoc simdjson::builtin::builder
+   */
+  namespace builder = builtin::builder;
 } // namespace simdjson
 
 #endif // SIMDJSON_ONDEMAND_H

include/simdjson/padded_string-inl.h

@@ -187,11 +187,11 @@ inline simdjson_result<padded_string> padded_string::load(std::string_view filen
 
 } // namespace simdjson
 
-inline simdjson::padded_string operator ""_padded(const char *str, size_t len) {
+inline simdjson::padded_string operator "" _padded(const char *str, size_t len) {
   return simdjson::padded_string(str, len);
 }
 #ifdef __cpp_char8_t
-inline simdjson::padded_string operator ""_padded(const char8_t *str, size_t len) {
+inline simdjson::padded_string operator "" _padded(const char8_t *str, size_t len) {
   return simdjson::padded_string(reinterpret_cast<const char8_t *>(str), len);
 }
 #endif

include/simdjson/padded_string.h

@@ -161,9 +161,9 @@ inline std::ostream& operator<<(std::ostream& out, simdjson_result<padded_string
 } // namespace simdjson
 
 // This is deliberately outside of simdjson so that people get it without having to use the namespace
-inline simdjson::padded_string operator ""_padded(const char *str, size_t len);
+inline simdjson::padded_string operator "" _padded(const char *str, size_t len);
 #ifdef __cpp_char8_t
-inline simdjson::padded_string operator ""_padded(const char8_t *str, size_t len);
+inline simdjson::padded_string operator "" _padded(const char8_t *str, size_t len);
 #endif
 
 namespace simdjson {

include/simdjson/padded_string_view-inl.h

@@ -53,11 +53,6 @@ inline bool padded_string_view::remove_utf8_bom() noexcept {
 inline std::ostream& operator<<(std::ostream& out, simdjson_result<padded_string_view> &s) noexcept(false) { return out << s.value(); }
 #endif
 
-inline padded_string_view pad(std::string& s) noexcept {
-  const auto len = s.size();
-  s.append(SIMDJSON_PADDING, ' ');
-  return padded_string_view(s.data(), len, s.size());
-}
 } // namespace simdjson
 
 

include/simdjson/padded_string_view.h

@@ -83,15 +83,6 @@ public:
 inline std::ostream& operator<<(std::ostream& out, simdjson_result<padded_string_view> &s) noexcept(false);
 #endif
 
-/**
- * Create a padded_string_view from a string. The string will be padded with SIMDJSON_PADDING
- * space characters. The resulting padded_string_view will have a length equal to the original
- * string.
- *
- * @param s The string.
- * @return The padded string.
- */
-inline padded_string_view pad(std::string& s) noexcept;
 } // namespace simdjson
 
 #endif // SIMDJSON_PADDED_STRING_VIEW_H

include/simdjson/ppc64/implementation.h

@@ -32,6 +32,7 @@ public:
                                          size_t &dst_len) const noexcept final;
   simdjson_warn_unused bool validate_utf8(const char *buf,
                                           size_t len) const noexcept final;
+  simdjson_warn_unused size_t write_string_escaped(const std::string_view input, char *out) const noexcept final;
 };
 
 } // namespace ppc64

include/simdjson/ppc64/stringparsing_defs.h

@@ -58,6 +58,31 @@ backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) {
   };
 }
 
+
+struct escaping {
+  static constexpr uint32_t BYTES_PROCESSED = 16;
+  simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst);
+
+  simdjson_inline bool has_escape() { return escape_bits != 0; }
+  simdjson_inline int escape_index() { return trailing_zeroes(escape_bits); }
+
+  uint64_t escape_bits;
+}; // struct escaping
+
+
+
+simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) {
+  static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes");
+  simd8<uint8_t> v(src);
+  v.store(dst);
+  simd8<bool> is_quote = (v == '"');
+  simd8<bool> is_backslash = (v == '\\');
+  simd8<bool> is_control = (v < 32);
+  return {
+    (is_backslash | is_quote | is_control).to_bitmask()
+  };
+}
+
 } // unnamed namespace
 } // namespace ppc64
 } // namespace simdjson

include/simdjson/simdjson_version.h

@@ -4,7 +4,7 @@
 #define SIMDJSON_SIMDJSON_VERSION_H
 
 /** The version of simdjson being used (major.minor.revision) */
-#define SIMDJSON_VERSION "3.11.2"
+#define SIMDJSON_VERSION "3.10.1"
 
 namespace simdjson {
 enum {
@@ -15,11 +15,11 @@ enum {
   /**
    * The minor version (major.MINOR.revision) of simdjson being used.
    */
-  SIMDJSON_VERSION_MINOR = 11,
+  SIMDJSON_VERSION_MINOR = 10,
   /**
    * The revision (major.minor.REVISION) of simdjson being used.
    */
-  SIMDJSON_VERSION_REVISION = 2
+  SIMDJSON_VERSION_REVISION = 1
 };
 } // namespace simdjson
 

include/simdjson/westmere/implementation.h

@@ -24,6 +24,7 @@ public:
   ) const noexcept final;
   simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final;
   simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final;
+  simdjson_warn_unused size_t write_string_escaped(const std::string_view input, char *out) const noexcept final;
 };
 
 } // namespace westmere

include/simdjson/westmere/stringparsing_defs.h

@@ -40,6 +40,31 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin
   };
 }
 
+
+struct escaping {
+  static constexpr uint32_t BYTES_PROCESSED = 16;
+  simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst);
+
+  simdjson_inline bool has_escape() { return escape_bits != 0; }
+  simdjson_inline int escape_index() { return trailing_zeroes(escape_bits); }
+
+  uint64_t escape_bits;
+}; // struct escaping
+
+
+
+simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) {
+  static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes");
+  simd8<uint8_t> v(src);
+  v.store(dst);
+  simd8<bool> is_quote = (v == '"');
+  simd8<bool> is_backslash = (v == '\\');
+  simd8<bool> is_control = (v < 32);
+  return {
+    uint64_t((is_backslash | is_quote | is_control).to_bitmask())
+  };
+}
+
 } // unnamed namespace
 } // namespace westmere
 } // namespace simdjson

singleheader/amalgamate.py

@@ -10,12 +10,41 @@ import os
 import re
 import shutil
 import datetime
+import json
 from typing import Dict, List, Optional, Set, TextIO, Union, cast
 
+# Check for Python 3, this does not actually work.
 if sys.version_info < (3, 0):
     sys.stdout.write("Sorry, requires Python 3.x or better\n")
     sys.exit(1)
 
+rules = """
+
+We refer your to the HACKING.md file for more information on how the project is organized.
+
+To help understand the error, here are the rules for including files in simdjson:
+
+All implementation-specific files, including arm64.h, arm64/implementation.h and
+arm64/ondemand.h, must be within SIMDJSON_CONDITIONAL_INCLUDE blocks.
+
+Top-level headers must not be included in any SIMDJSON_CONDITIONAL_INCLUDE block.
+
+Generic files must be included only in amalgamator files (arm64.h,
+arm64/implementation.h, arm64/ondemand.h, generic/amalgamated.h).
+
+We fail if an implementation-specific file is included more than once in the same block.
+We fail if a generic file is included more than once per implementation in the same block.
+
+
+Tip: generally, "file" will search the including file's source directory first, then
+the search paths while <file> does it the other way around.
+We prefer to use <> in simdjson headers to avoid accidentally including a file from the
+wrong directory.
+
+The amalgamate.py script checks that all files are included.
+
+"""
+
 SCRIPTPATH = os.path.dirname(os.path.abspath(sys.argv[0]))
 PROJECTPATH = os.path.dirname(SCRIPTPATH)
 print(f"SCRIPTPATH={SCRIPTPATH} PROJECTPATH={PROJECTPATH}")
@@ -62,6 +91,22 @@ class SimdjsonFile:
     def __str__(self):
         return self.include_path
 
+    def dump(self):
+        return {
+            'root': self.root,
+            'include_path': self.include_path,
+            'includes': [include.include_path for include in self.includes],
+            'included_from': [included_from.include_path for included_from in self.included_from],
+            'editor_only_includes': [editor_only_include.include_path for editor_only_include in self.editor_only_includes],
+            'editor_only_included_from': [editor_only_included_from.include_path for editor_only_included_from in self.editor_only_included_from],
+            'processed': self.processed,
+            'dependency_file': self.dependency_file.include_path if self.dependency_file else None,
+            'is_amalgamator': self.is_amalgamator,
+            'implementation': self.implementation,
+        }
+    def json(self):
+        return json.dumps(self.dump(), indent=4, sort_keys=True, ensure_ascii=False)
+
     def __repr__(self):
         return self.include_path
 
@@ -162,20 +207,21 @@ class SimdjsonFile:
 
     def add_include(self, include: 'SimdjsonFile'):
         if self.is_conditional_include:
-            assert include.is_conditional_include, f"{self} cannot include {include} without #ifndef SIMDJSON_CONDITIONAL_INCLUDE."
+            # If I have a dependency file, I can only include something that has a dependency file.
+            assert include.is_conditional_include, f"{self} cannot include {include} without #ifndef SIMDJSON_CONDITIONAL_INCLUDE. {rules}"
             # TODO make sure we only include amalgamated files that are guaranteed to be included with us (or before us)
             # if include.amalgamator_file:
             #     assert include.amalgamator_file == self, f"{self} cannot include {include}: it should be included from {include.amalgamator_file} instead."
         else:
-            assert include.is_amalgamator or not include.is_conditional_include, f"{self} cannot include {include} because it is an amalgamated file."
+            assert include.is_amalgamator or not include.is_conditional_include, f"{self} cannot include {include} because it is an amalgamated file. {rules}"
 
         self.includes.append(include)
         include.included_from.add(self)
 
     def add_editor_only_include(self, include: 'SimdjsonFile'):
-        assert self.is_conditional_include, f"Cannot use #ifndef SIMDJSON_CONDITIONAL_INCLUDE in {self} because it is not an amalgamated file."
+        assert self.is_conditional_include, f"Cannot use #ifndef SIMDJSON_CONDITIONAL_INCLUDE in {self} because it is not an amalgamated file. {rules}"
         if not include.is_conditional_include:
-            assert self.dependency_file, f"{self} cannot include {include} without #ifndef SIMDJSON_CONDITIONAL_INCLUDE."
+            assert self.dependency_file, f"{self} cannot include {include} without #ifndef SIMDJSON_CONDITIONAL_INCLUDE. {rules}"
         # TODO make sure we only include amalgamated files that are guaranteed to be included with us (or before us)
         # elif include.amalgamator_file:
         #     assert self.is_amalgamated_before(self.amalgamator_file), f"{self} cannot include {include}: it should be included from {include.amalgamator_file} instead."
@@ -190,11 +236,11 @@ class SimdjsonFile:
                 if file.dependency_file == self:
                     for editor_only_include in file.editor_only_includes:
                         if not editor_only_include.is_conditional_include:
-                            assert editor_only_include in self.includes, f"{file} includes {editor_only_include}, but it is not included from {self}. It must be added to {self}."
+                            assert editor_only_include in self.includes, f"{file} includes {editor_only_include}, but it is not included from {self}. It must be added to {self}. {rules}"
                             if editor_only_include in extra_include_set:
                                 extra_include_set.remove(editor_only_include)
 
-            assert len(extra_include_set) == 0, f"{self} unnecessarily includes {extra_include_set}. They are not included in the corresponding amalgamated files."
+            assert len(extra_include_set) == 0, f"{self} unnecessarily includes {extra_include_set}. They are not included in the corresponding amalgamated files. {rules}"
 
 class SimdjsonRepository:
     def __init__(self, project_path: str, relative_roots: List[RelativeRoot]):
@@ -320,6 +366,7 @@ class Amalgamator:
 
         assert not self.editor_only_region
         with open(file.absolute_path, 'r') as fid2:
+            print(f"including: {file}")
             for line in fid2:
                 line = line.rstrip('\n')
 
@@ -329,9 +376,9 @@ class Amalgamator:
 
                 # Ignore lines inside #ifndef SIMDJSON_CONDITIONAL_INCLUDE
                 if re.search(r'^#ifndef\s+SIMDJSON_CONDITIONAL_INCLUDE\s*$', line):
-                    assert file.is_conditional_include, f"{file} uses #ifndef SIMDJSON_CONDITIONAL_INCLUDE but is not an amalgamated file!"
-                    assert self.in_conditional_include_block, f"{file} uses #ifndef SIMDJSON_CONDITIONAL_INCLUDE without a prior #define SIMDJSON_CONDITIONAL_INCLUDE: {self.include_stack}"
-                    assert not self.editor_only_region, f"{file} uses #ifndef SIMDJSON_CONDITIONAL_INCLUDE twice in a row"
+                    assert file.is_conditional_include, f"{file} uses #ifndef SIMDJSON_CONDITIONAL_INCLUDE but is not an amalgamated file! {rules}"
+                    assert self.in_conditional_include_block, f"{file} uses #ifndef SIMDJSON_CONDITIONAL_INCLUDE without a prior #define SIMDJSON_CONDITIONAL_INCLUDE: {self.include_stack} {rules}"
+                    assert not self.editor_only_region, f"{file} uses #ifndef SIMDJSON_CONDITIONAL_INCLUDE twice in a row {rules}"
                     self.editor_only_region = True
 
                 # Handle ignored lines (and ending ignore blocks)
@@ -349,7 +396,7 @@ class Amalgamator:
                         self.editor_only_region = False
                     continue
 
-                assert not end_ignore, f"{file} has #endif // SIMDJSON_CONDITIONAL_INCLUDE without #ifndef SIMDJSON_CONDITIONAL_INCLUDE"
+                assert not end_ignore, f"{file} has #endif // SIMDJSON_CONDITIONAL_INCLUDE without #ifndef SIMDJSON_CONDITIONAL_INCLUDE {rules}"
 
                 # Handle #include lines
                 included = re.search(r'^#include\s+["<]([^">]*)[">]', line)
@@ -376,26 +423,26 @@ class Amalgamator:
                     self.implementation = None
                 elif re.search(r'\bSIMDJSON_IMPLEMENTATION\b', line) and file.include_path != IMPLEMENTATION_DETECTION_H:
                     # copy the line, with SIMDJSON_IMPLEMENTATION replace to what it is currently defined to
-                    assert self.implementation, f"Use of SIMDJSON_IMPLEMENTATION while not defined in {file}: {line}"
+                    assert self.implementation, f"Use of SIMDJSON_IMPLEMENTATION while not defined in {file}: {line}\n{rules}"
                     line = re.sub(r'\bSIMDJSON_IMPLEMENTATION\b',self.implementation,line)
 
                 # Handle defining and undefining SIMDJSON_CONDITIONAL_INCLUDE
                 defined = re.search(r'^#define\s+SIMDJSON_CONDITIONAL_INCLUDE\s*$', line)
                 if defined:
-                    assert not file.is_conditional_include, "SIMDJSON_CONDITIONAL_INCLUDE defined in amalgamated file {file}! Not allowed."
-                    assert not self.in_conditional_include_block, f"{file} redefines SIMDJSON_CONDITIONAL_INCLUDE"
+                    assert not file.is_conditional_include, "SIMDJSON_CONDITIONAL_INCLUDE defined in amalgamated file {file}! Not allowed. {rules}"
+                    assert not self.in_conditional_include_block, f"{file} redefines SIMDJSON_CONDITIONAL_INCLUDE {rules}"
                     self.in_conditional_include_block = True
                     self.found_includes_per_conditional_block.clear()
                     self.write(f'/* defining SIMDJSON_CONDITIONAL_INCLUDE */')
                 elif re.search(r'^#undef\s+SIMDJSON_CONDITIONAL_INCLUDE\s*$', line):
-                    assert not file.is_conditional_include, "SIMDJSON_CONDITIONAL_INCLUDE undefined in amalgamated file {file}! Not allowed."
-                    assert self.in_conditional_include_block, f"{file} undefines SIMDJSON_CONDITIONAL_INCLUDE without defining it"
+                    assert not file.is_conditional_include, "SIMDJSON_CONDITIONAL_INCLUDE undefined in amalgamated file {file}! Not allowed. {rules}"
+                    assert self.in_conditional_include_block, f"{file} undefines SIMDJSON_CONDITIONAL_INCLUDE without defining it {rules}"
                     self.write(f'/* undefining SIMDJSON_CONDITIONAL_INCLUDE */')
                     self.in_conditional_include_block = False
 
                 self.write(line)
 
-            assert not self.editor_only_region, f"{file} ended without #endif // SIMDJSON_CONDITIONAL_INCLUDE"
+            assert not self.editor_only_region, f"{file} ended without #endif // SIMDJSON_CONDITIONAL_INCLUDE {rules}"
 
         self.write(f"/* end file {self.file_to_str(file)} */")
 

singleheader/simdjson.cpp

@@ -1,4 +1,4 @@
-/* auto-generated on 2024-12-09 21:36:12 -0500. Do not edit! */
+/* auto-generated on 2024-10-28 20:27:26 -0400. Do not edit! */
 /* including simdjson.cpp:  */
 /* begin file simdjson.cpp */
 #define SIMDJSON_SRC_SIMDJSON_CPP
@@ -2443,7 +2443,8 @@ enum error_code {
   SCALAR_DOCUMENT_AS_VALUE,   ///< A scalar document is treated as a value.
   OUT_OF_BOUNDS,              ///< Attempted to access location outside of document.
   TRAILING_CONTENT,           ///< Unexpected trailing content in the JSON input
-  NUM_ERROR_CODES
+  OUT_OF_CAPACITY,            ///< The capacity was exceeded, we cannot allocate enough memory.
+  NUM_ERROR_CODES             ///< Placeholder for end of error code list.
 };
 
 /**
@@ -2480,7 +2481,7 @@ struct simdjson_error : public std::exception {
    */
   simdjson_error(error_code error) noexcept : _error{error} { }
   /** The error message */
-  const char *what() const noexcept override { return error_message(error()); }
+  const char *what() const noexcept { return error_message(error()); }
   /** The error code */
   error_code error() const noexcept { return _error; }
 private:
@@ -2736,14 +2737,14 @@ namespace details {
     };                                                                         \
   };
 
-SIMDJSON_IMPL_CONCEPT(emplace_back, emplace_back)
-SIMDJSON_IMPL_CONCEPT(emplace, emplace)
-SIMDJSON_IMPL_CONCEPT(push_back, push_back)
-SIMDJSON_IMPL_CONCEPT(add, add)
-SIMDJSON_IMPL_CONCEPT(push, push)
-SIMDJSON_IMPL_CONCEPT(append, append)
-SIMDJSON_IMPL_CONCEPT(insert, insert)
-SIMDJSON_IMPL_CONCEPT(op_append, operator+=)
+SIMDJSON_IMPL_CONCEPT(emplace_back, emplace_back);
+SIMDJSON_IMPL_CONCEPT(emplace, emplace);
+SIMDJSON_IMPL_CONCEPT(push_back, push_back);
+SIMDJSON_IMPL_CONCEPT(add, add);
+SIMDJSON_IMPL_CONCEPT(push, push);
+SIMDJSON_IMPL_CONCEPT(append, append);
+SIMDJSON_IMPL_CONCEPT(insert, insert);
+SIMDJSON_IMPL_CONCEPT(op_append, operator+=);
 
 #undef SIMDJSON_IMPL_CONCEPT
 } // namespace details
@@ -7788,7 +7789,7 @@ SIMDJSON_NO_SANITIZE_UNDEFINED
 // See issue https://github.com/simdjson/simdjson/issues/1965
 SIMDJSON_NO_SANITIZE_MEMORY
 simdjson_inline int trailing_zeroes(uint64_t input_num) {
-#if SIMDJSON_REGULAR_VISUAL_STUDIO
+#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
   unsigned long ret;
   // Search the mask data from least significant bit (LSB)
   // to the most significant bit (MSB) for a set bit (1).
@@ -7812,7 +7813,7 @@ simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) {
 SIMDJSON_NO_SANITIZE_UNDEFINED
 /* result might be undefined when input_num is zero */
 simdjson_inline int leading_zeroes(uint64_t input_num) {
-#if SIMDJSON_REGULAR_VISUAL_STUDIO
+#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
   unsigned long leading_zero = 0;
   // Search the mask data from most significant bit (MSB)
   // to least significant bit (LSB) for a set bit (1).
@@ -7865,7 +7866,7 @@ simdjson_inline uint64_t zero_leading_bit(uint64_t rev_bits, int leading_zeroes)
 #endif
 
 simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t *result) {
-#if SIMDJSON_REGULAR_VISUAL_STUDIO
+#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
   *result = value1 + value2;
   return *result < value1;
 #else
@@ -8008,7 +8009,7 @@ namespace arm64 {
 namespace {
 namespace simd {
 
-#if SIMDJSON_REGULAR_VISUAL_STUDIO
+#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
 namespace {
 // Start of private section with Visual Studio workaround
 
@@ -8117,7 +8118,7 @@ namespace {
     // We return uint32_t instead of uint16_t because that seems to be more efficient for most
     // purposes (cutting it down to uint16_t costs performance in some compilers).
     simdjson_inline uint32_t to_bitmask() const {
-#if SIMDJSON_REGULAR_VISUAL_STUDIO
+#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
       const uint8x16_t bit_mask =  simdjson_make_uint8x16_t(0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80,
                                                    0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80);
 #else
@@ -8148,7 +8149,7 @@ namespace {
     // Splat constructor
     simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {}
     // Member-by-member initialization
-#if SIMDJSON_REGULAR_VISUAL_STUDIO
+#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
     simdjson_inline simd8(
       uint8_t v0,  uint8_t v1,  uint8_t v2,  uint8_t v3,  uint8_t v4,  uint8_t v5,  uint8_t v6,  uint8_t v7,
       uint8_t v8,  uint8_t v9,  uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15
@@ -8242,7 +8243,7 @@ namespace {
       uint64x2_t shufmask64 = {thintable_epi8[mask1], thintable_epi8[mask2]};
       uint8x16_t shufmask = vreinterpretq_u8_u64(shufmask64);
       // we increment by 0x08 the second half of the mask
-#if SIMDJSON_REGULAR_VISUAL_STUDIO
+#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
       uint8x16_t inc = simdjson_make_uint8x16_t(0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08);
 #else
       uint8x16_t inc = {0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08};
@@ -8272,7 +8273,7 @@ namespace {
       uint8x8_t compactmask1 = vcreate_u8(thintable_epi8[mask1]);
       uint8x8_t compactmask2 = vcreate_u8(thintable_epi8[mask2]);
       // we increment by 0x08 the second half of the mask
-#if SIMDJSON_REGULAR_VISUAL_STUDIO
+#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
       uint8x8_t inc = simdjson_make_uint8x8_t(0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08);
 #else
       uint8x8_t inc = {0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08};
@@ -8324,7 +8325,7 @@ namespace {
     // Array constructor
     simdjson_inline simd8(const int8_t* values) : simd8(load(values)) {}
     // Member-by-member initialization
-#if SIMDJSON_REGULAR_VISUAL_STUDIO
+#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
     simdjson_inline simd8(
       int8_t v0,  int8_t v1,  int8_t v2,  int8_t v3, int8_t v4,  int8_t v5,  int8_t v6,  int8_t v7,
       int8_t v8,  int8_t v9,  int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15
@@ -8445,7 +8446,7 @@ namespace {
     }
 
     simdjson_inline uint64_t to_bitmask() const {
-#if SIMDJSON_REGULAR_VISUAL_STUDIO
+#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
       const uint8x16_t bit_mask = simdjson_make_uint8x16_t(
         0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80,
         0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80
@@ -10555,7 +10556,7 @@ SIMDJSON_NO_SANITIZE_UNDEFINED
 // See issue https://github.com/simdjson/simdjson/issues/1965
 SIMDJSON_NO_SANITIZE_MEMORY
 simdjson_inline int trailing_zeroes(uint64_t input_num) {
-#if SIMDJSON_REGULAR_VISUAL_STUDIO
+#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
   unsigned long ret;
   // Search the mask data from least significant bit (LSB)
   // to the most significant bit (MSB) for a set bit (1).
@@ -10579,7 +10580,7 @@ simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) {
 SIMDJSON_NO_SANITIZE_UNDEFINED
 /* result might be undefined when input_num is zero */
 simdjson_inline int leading_zeroes(uint64_t input_num) {
-#if SIMDJSON_REGULAR_VISUAL_STUDIO
+#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
   unsigned long leading_zero = 0;
   // Search the mask data from most significant bit (MSB)
   // to least significant bit (LSB) for a set bit (1).
@@ -10632,7 +10633,7 @@ simdjson_inline uint64_t zero_leading_bit(uint64_t rev_bits, int leading_zeroes)
 #endif
 
 simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t *result) {
-#if SIMDJSON_REGULAR_VISUAL_STUDIO
+#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
   *result = value1 + value2;
   return *result < value1;
 #else
@@ -10775,7 +10776,7 @@ namespace arm64 {
 namespace {
 namespace simd {
 
-#if SIMDJSON_REGULAR_VISUAL_STUDIO
+#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
 namespace {
 // Start of private section with Visual Studio workaround
 
@@ -10884,7 +10885,7 @@ namespace {
     // We return uint32_t instead of uint16_t because that seems to be more efficient for most
     // purposes (cutting it down to uint16_t costs performance in some compilers).
     simdjson_inline uint32_t to_bitmask() const {
-#if SIMDJSON_REGULAR_VISUAL_STUDIO
+#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
       const uint8x16_t bit_mask =  simdjson_make_uint8x16_t(0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80,
                                                    0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80);
 #else
@@ -10915,7 +10916,7 @@ namespace {
     // Splat constructor
     simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {}
     // Member-by-member initialization
-#if SIMDJSON_REGULAR_VISUAL_STUDIO
+#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
     simdjson_inline simd8(
       uint8_t v0,  uint8_t v1,  uint8_t v2,  uint8_t v3,  uint8_t v4,  uint8_t v5,  uint8_t v6,  uint8_t v7,
       uint8_t v8,  uint8_t v9,  uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15
@@ -11009,7 +11010,7 @@ namespace {
       uint64x2_t shufmask64 = {thintable_epi8[mask1], thintable_epi8[mask2]};
       uint8x16_t shufmask = vreinterpretq_u8_u64(shufmask64);
       // we increment by 0x08 the second half of the mask
-#if SIMDJSON_REGULAR_VISUAL_STUDIO
+#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
       uint8x16_t inc = simdjson_make_uint8x16_t(0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08);
 #else
       uint8x16_t inc = {0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08};
@@ -11039,7 +11040,7 @@ namespace {
       uint8x8_t compactmask1 = vcreate_u8(thintable_epi8[mask1]);
       uint8x8_t compactmask2 = vcreate_u8(thintable_epi8[mask2]);
       // we increment by 0x08 the second half of the mask
-#if SIMDJSON_REGULAR_VISUAL_STUDIO
+#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
       uint8x8_t inc = simdjson_make_uint8x8_t(0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08);
 #else
       uint8x8_t inc = {0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08};
@@ -11091,7 +11092,7 @@ namespace {
     // Array constructor
     simdjson_inline simd8(const int8_t* values) : simd8(load(values)) {}
     // Member-by-member initialization
-#if SIMDJSON_REGULAR_VISUAL_STUDIO
+#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
     simdjson_inline simd8(
       int8_t v0,  int8_t v1,  int8_t v2,  int8_t v3, int8_t v4,  int8_t v5,  int8_t v6,  int8_t v7,
       int8_t v8,  int8_t v9,  int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15
@@ -11212,7 +11213,7 @@ namespace {
     }
 
     simdjson_inline uint64_t to_bitmask() const {
-#if SIMDJSON_REGULAR_VISUAL_STUDIO
+#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
       const uint8x16_t bit_mask = simdjson_make_uint8x16_t(
         0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80,
         0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80
@@ -14165,7 +14166,6 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu
   return stage2::tape_builder::parse_document<true>(*this, _doc);
 }
 
-SIMDJSON_NO_SANITIZE_MEMORY
 simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept {
   return arm64::stringparsing::parse_string(src, dst, allow_replacement);
 }
@@ -20393,7 +20393,6 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu
   return stage2::tape_builder::parse_document<true>(*this, _doc);
 }
 
-SIMDJSON_NO_SANITIZE_MEMORY
 simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool replacement_char) const noexcept {
   return haswell::stringparsing::parse_string(src, dst, replacement_char);
 }
@@ -26648,7 +26647,6 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu
   return stage2::tape_builder::parse_document<true>(*this, _doc);
 }
 
-SIMDJSON_NO_SANITIZE_MEMORY
 simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool replacement_char) const noexcept {
   return icelake::stringparsing::parse_string(src, dst, replacement_char);
 }
@@ -33073,7 +33071,6 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu
   return stage2::tape_builder::parse_document<true>(*this, _doc);
 }
 
-SIMDJSON_NO_SANITIZE_MEMORY
 simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool replacement_char) const noexcept {
   return ppc64::stringparsing::parse_string(src, dst, replacement_char);
 }
@@ -40162,7 +40159,6 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu
   return stage2::tape_builder::parse_document<true>(*this, _doc);
 }
 
-SIMDJSON_NO_SANITIZE_MEMORY
 simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool replacement_char) const noexcept {
   return westmere::stringparsing::parse_string(src, dst, replacement_char);
 }
@@ -46160,7 +46156,6 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu
   return stage2::tape_builder::parse_document<true>(*this, _doc);
 }
 
-SIMDJSON_NO_SANITIZE_MEMORY
 simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept {
   return lsx::stringparsing::parse_string(src, dst, allow_replacement);
 }
@@ -52183,7 +52178,6 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu
   return stage2::tape_builder::parse_document<true>(*this, _doc);
 }
 
-SIMDJSON_NO_SANITIZE_MEMORY
 simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept {
   return lasx::stringparsing::parse_string(src, dst, allow_replacement);
 }
@@ -56157,7 +56151,6 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu
   return stage2::tape_builder::parse_document<true>(*this, _doc);
 }
 
-SIMDJSON_NO_SANITIZE_MEMORY
 simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool replacement_char) const noexcept {
   return fallback::stringparsing::parse_string(src, dst, replacement_char);
 }

src/arm64.cpp

@@ -150,7 +150,6 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu
   return stage2::tape_builder::parse_document<true>(*this, _doc);
 }
 
-SIMDJSON_NO_SANITIZE_MEMORY
 simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept {
   return arm64::stringparsing::parse_string(src, dst, allow_replacement);
 }
@@ -165,6 +164,10 @@ simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *
   return stage2(_doc);
 }
 
+simdjson_warn_unused size_t implementation::write_string_escaped(const std::string_view input, char *out) const noexcept {
+  return arm64::stringparsing::write_string_escaped(input, out);
+}
+
 } // namespace arm64
 } // namespace simdjson
 

src/fallback.cpp

@@ -388,7 +388,6 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu
   return stage2::tape_builder::parse_document<true>(*this, _doc);
 }
 
-SIMDJSON_NO_SANITIZE_MEMORY
 simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool replacement_char) const noexcept {
   return fallback::stringparsing::parse_string(src, dst, replacement_char);
 }
@@ -403,6 +402,10 @@ simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *
   return stage2(_doc);
 }
 
+simdjson_warn_unused size_t implementation::write_string_escaped(const std::string_view input, char *out) const noexcept {
+  return fallback::stringparsing::write_string_escaped(input, out);
+}
+
 } // namespace fallback
 } // namespace simdjson
 

src/generic/stage2/stringparsing.h

@@ -236,6 +236,101 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t
   }
 }
 
+simdjson_warn_unused size_t write_string_escaped(const std::string_view input, char *out) noexcept {
+  // We are making the following assumption: most strings will either be very short or they will not
+  // need escaping.
+  size_t i = 0;
+  size_t pos = 0;
+  if(input.size() >= escaping::BYTES_PROCESSED) {
+    auto vec_processing = [input,out]() -> size_t {
+      size_t i = 0;
+      size_t pos = 0;
+      for(;input.size() - i >= escaping::BYTES_PROCESSED; i += escaping::BYTES_PROCESSED) {
+        escaping vinput = escaping::copy_and_find(reinterpret_cast<const uint8_t *>(input.data()) + i, reinterpret_cast<uint8_t *>(out) + pos);
+        if(vinput.has_escape()) {
+          return i + vinput.escape_index(); // We have a character that needs escaping
+        }
+        pos += escaping::BYTES_PROCESSED;
+      }
+      if(i == input.size()) { return input.size(); }
+      // We virtually backtrack so we can load a full vector register
+      i = input.size() - escaping::BYTES_PROCESSED;
+      pos = i;
+      escaping vinput = escaping::copy_and_find(reinterpret_cast<const uint8_t *>(input.data()) + i, reinterpret_cast<uint8_t *>(out) + pos);
+      if(vinput.has_escape()) {
+        return i + vinput.escape_index(); // We have a character that needs escaping
+      }
+      return input.size();
+    };
+    i = vec_processing();
+    pos = i;
+    if(i == input.size()) { return pos; }
+    // Here we only continue if there was a character that needed escaping.
+  }
+  static std::string_view control_chars[] = {
+    "\\x0000", "\\x0001", "\\x0002", "\\x0003", "\\x0004", "\\x0005", "\\x0006",
+    "\\x0007", "\\x0008", "\\t",     "\\n",     "\\x000b", "\\f",     "\\r",
+    "\\x000e", "\\x000f", "\\x0010", "\\x0011", "\\x0012", "\\x0013", "\\x0014",
+    "\\x0015", "\\x0016", "\\x0017", "\\x0018", "\\x0019", "\\x001a", "\\x001b",
+    "\\x001c", "\\x001d", "\\x001e", "\\x001f"};
+  static std::array<uint8_t, 256> json_quotable_character =
+    []() constexpr {
+      std::array<uint8_t, 256> result{};
+      for (int i = 0; i < 32; i++) {
+        result[i] = 1;
+      }
+      for (int i : {'"', '\\'}) {
+        result[i] = 1;
+      }
+      return result;
+    }();
+  // The rest could possibly be vectorized, but consider that we expect most strings
+  // to be short or not to require escaping.
+  for (; i < input.size(); i++) {
+    uint8_t c = static_cast<uint8_t>(input[i]);
+    if(json_quotable_character[c]) {
+      switch (c) {
+      case '"':
+        out[pos++] = '\\';
+        out[pos++] = '"';
+        break;
+      case '\\':
+        out[pos++] = '\\';
+        out[pos++] = '\\';
+        break;
+      case '\b':
+        out[pos++] = '\\';
+        out[pos++] = 'b';
+        break;
+      case '\f':
+        out[pos++] = '\\';
+        out[pos++] = 'f';
+        break;
+      case '\n':
+        out[pos++] = '\\';
+        out[pos++] = 'n';
+        break;
+      case '\r':
+        out[pos++] = '\\';
+        out[pos++] = 'r';
+        break;
+      case '\t':
+        out[pos++] = '\\';
+        out[pos++] = 't';
+        break;
+      default:
+        control_chars[c].copy(out + pos, 6);
+        pos += 6;
+      }
+    } else {
+      out[pos++] = c;
+    }
+  }
+  return pos;
+}
+
+
+
 } // namespace stringparsing
 } // unnamed namespace
 } // namespace SIMDJSON_IMPLEMENTATION

src/haswell.cpp

@@ -147,7 +147,6 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu
   return stage2::tape_builder::parse_document<true>(*this, _doc);
 }
 
-SIMDJSON_NO_SANITIZE_MEMORY
 simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool replacement_char) const noexcept {
   return haswell::stringparsing::parse_string(src, dst, replacement_char);
 }
@@ -162,6 +161,10 @@ simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *
   return stage2(_doc);
 }
 
+simdjson_warn_unused size_t implementation::write_string_escaped(const std::string_view input, char *out) const noexcept {
+  return haswell::stringparsing::write_string_escaped(input, out);
+}
+
 } // namespace SIMDJSON_IMPLEMENTATION
 } // namespace simdjson
 

src/icelake.cpp

@@ -193,7 +193,6 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu
   return stage2::tape_builder::parse_document<true>(*this, _doc);
 }
 
-SIMDJSON_NO_SANITIZE_MEMORY
 simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool replacement_char) const noexcept {
   return icelake::stringparsing::parse_string(src, dst, replacement_char);
 }
@@ -208,6 +207,10 @@ simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *
   return stage2(_doc);
 }
 
+simdjson_warn_unused size_t implementation::write_string_escaped(const std::string_view input, char *out) const noexcept {
+  return icelake::stringparsing::write_string_escaped(input, out);
+}
+
 } // namespace icelake
 } // namespace simdjson
 

src/implementation.cpp

@@ -186,6 +186,9 @@ public:
   simdjson_warn_unused bool validate_utf8(const char * buf, size_t len) const noexcept final override {
     return set_best()->validate_utf8(buf, len);
   }
+  simdjson_warn_unused size_t write_string_escaped(const std::string_view input, char *out) const noexcept final {
+    return set_best()->write_string_escaped(input, out);
+  }
   simdjson_inline detect_best_supported_implementation_on_first_use() noexcept : implementation("best_supported_detector", "Detects the best supported implementation and sets it", 0) {}
 private:
   const implementation *set_best() const noexcept;
@@ -236,6 +239,9 @@ public:
   simdjson_warn_unused error_code minify(const uint8_t *, size_t, uint8_t *, size_t &) const noexcept final override {
     return UNSUPPORTED_ARCHITECTURE;
   }
+  simdjson_warn_unused size_t write_string_escaped(const std::string_view input, char *out) const noexcept final {
+    return 0;
+  }
   simdjson_warn_unused bool validate_utf8(const char *, size_t) const noexcept final override {
     return false; // Just refuse to validate. Given that we have a fallback implementation
     // it seems unlikely that unsupported_implementation will ever be used. If it is used,
@@ -319,6 +325,9 @@ simdjson_warn_unused error_code minify(const char *buf, size_t len, char *dst, s
 simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) noexcept {
   return get_active_implementation()->validate_utf8(buf, len);
 }
+simdjson_warn_unused size_t write_string_escaped(const std::string_view input, char *out) noexcept {
+  return get_active_implementation()->write_string_escaped(input, out);
+}
 const implementation * builtin_implementation() {
   static const implementation * builtin_impl = get_available_implementations()[SIMDJSON_STRINGIFY(SIMDJSON_BUILTIN_IMPLEMENTATION)];
   assert(builtin_impl);

src/lasx.cpp

@@ -110,7 +110,6 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu
   return stage2::tape_builder::parse_document<true>(*this, _doc);
 }
 
-SIMDJSON_NO_SANITIZE_MEMORY
 simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept {
   return lasx::stringparsing::parse_string(src, dst, allow_replacement);
 }
@@ -125,6 +124,10 @@ simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *
   return stage2(_doc);
 }
 
+simdjson_warn_unused size_t implementation::write_string_escaped(const std::string_view input, char *out) const noexcept {
+  return lasx::stringparsing::write_string_escaped(input, out);
+}
+
 } // namespace lasx
 } // namespace simdjson
 

src/lsx.cpp

@@ -114,7 +114,6 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu
   return stage2::tape_builder::parse_document<true>(*this, _doc);
 }
 
-SIMDJSON_NO_SANITIZE_MEMORY
 simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept {
   return lsx::stringparsing::parse_string(src, dst, allow_replacement);
 }
@@ -129,6 +128,10 @@ simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *
   return stage2(_doc);
 }
 
+simdjson_warn_unused size_t implementation::write_string_escaped(const std::string_view input, char *out) const noexcept {
+  return lsx::stringparsing::write_string_escaped(input, out);
+}
+
 } // namespace lsx
 } // namespace simdjson
 

src/ppc64.cpp

@@ -120,7 +120,6 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu
   return stage2::tape_builder::parse_document<true>(*this, _doc);
 }
 
-SIMDJSON_NO_SANITIZE_MEMORY
 simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool replacement_char) const noexcept {
   return ppc64::stringparsing::parse_string(src, dst, replacement_char);
 }
@@ -135,6 +134,10 @@ simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *
   return stage2(_doc);
 }
 
+simdjson_warn_unused size_t implementation::write_string_escaped(const std::string_view input, char *out) const noexcept {
+  return ppc64::stringparsing::write_string_escaped(input, out);
+}
+
 } // namespace ppc64
 } // namespace simdjson
 

src/westmere.cpp

@@ -152,7 +152,6 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu
   return stage2::tape_builder::parse_document<true>(*this, _doc);
 }
 
-SIMDJSON_NO_SANITIZE_MEMORY
 simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool replacement_char) const noexcept {
   return westmere::stringparsing::parse_string(src, dst, replacement_char);
 }
@@ -167,6 +166,10 @@ simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *
   return stage2(_doc);
 }
 
+simdjson_warn_unused size_t implementation::write_string_escaped(const std::string_view input, char *out) const noexcept {
+  return westmere::stringparsing::write_string_escaped(input, out);
+}
+
 } // namespace westmere
 } // namespace simdjson
 

tests/CMakeLists.txt

@@ -26,3 +26,4 @@ endif()
 add_cpp_test(checkimplementation LABELS other per_implementation)
 
 add_subdirectory(compilation_failure_tests)
+add_subdirectory(builder)

tests/builder/CMakeLists.txt

@@ -0,0 +1,12 @@
+# All remaining tests link with simdjson proper
+link_libraries(simdjson)
+include_directories(..)
+add_cpp_test(builder_string_builder_tests              LABELS ondemand acceptance per_implementation)
+
+# Copy the simdjson dll into the tests directory
+if(MSVC AND BUILD_SHARED_LIBS)
+  add_custom_command(TARGET builder_string_builder_tests POST_BUILD        # Adds a post-build event
+    COMMAND ${CMAKE_COMMAND} -E copy_if_different  # which executes "cmake -E copy_if_different..."
+        "$<TARGET_FILE:simdjson>"      # <--this is in-file
+        "$<TARGET_FILE_DIR:builder_string_builder_tests>")                 # <--this is out-file path
+endif(MSVC AND BUILD_SHARED_LIBS)

tests/builder/builder_string_builder_tests.cpp

@@ -0,0 +1,178 @@
+#include "simdjson.h"
+#include "test_builder.h"
+#include <string_view>
+
+using namespace simdjson;
+
+namespace builder_tests {
+  using namespace std;
+
+    #if SIMDJSON_EXCEPTIONS
+    bool string_convertion_except() {
+        TEST_START();
+        simdjson::ondemand::parser p;
+        simdjson::builder::string_builder sb;
+        sb.append('a');
+        std::string r(sb);
+        ASSERT_EQUAL(r, "a");
+        TEST_SUCCEED();
+    }
+    #endif
+
+
+    bool append_char() {
+        TEST_START();
+        simdjson::builder::string_builder sb;
+        sb.append('a');
+        ASSERT_EQUAL(sb.size(), 1);
+        std::string_view p;
+        auto result = sb.view().get(p);
+        ASSERT_EQUAL(result, SUCCESS);
+        ASSERT_EQUAL(p, "a");
+        TEST_SUCCEED();
+    }
+
+
+    bool append_integer() {
+        TEST_START();
+        simdjson::builder::string_builder sb;
+        sb.append(42);
+        ASSERT_EQUAL(sb.size(), 2);
+        std::string_view p;
+        auto result = sb.view().get(p);
+        ASSERT_EQUAL(result, SUCCESS);
+        ASSERT_EQUAL(p, "42");
+        TEST_SUCCEED();
+    }
+
+    bool append_float() {
+        TEST_START();
+        simdjson::builder::string_builder sb;
+        sb.append(1.1);
+        ASSERT_EQUAL(sb.size(), 3);
+        std::string_view p;
+        auto result = sb.view().get(p);
+        ASSERT_EQUAL(result, SUCCESS);
+        ASSERT_EQUAL(p, "1.1");
+        TEST_SUCCEED();
+    }
+
+    bool append_null() {
+        TEST_START();
+        simdjson::builder::string_builder sb;
+        sb.append_null();
+        ASSERT_EQUAL(sb.size(), 4);
+        std::string_view p;
+        auto result = sb.view().get(p);
+        ASSERT_EQUAL(result, SUCCESS);
+        ASSERT_EQUAL(p, "null");
+        TEST_SUCCEED();
+    }
+
+    bool clear() {
+        TEST_START();
+        simdjson::builder::string_builder sb;
+        sb.append('a');
+        sb.clear();
+        ASSERT_EQUAL(sb.size(), 0);
+        TEST_SUCCEED();
+    }
+
+    bool escape_and_append() {
+        TEST_START();
+        simdjson::builder::string_builder sb;
+        sb.escape_and_append("Hello, \"world\"!");
+        std::string_view p;
+        auto result = sb.view().get(p);
+        ASSERT_EQUAL(result, SUCCESS);
+        ASSERT_EQUAL(p, "Hello, \"world\"!");
+        TEST_SUCCEED();
+    }
+
+    bool escape_and_append_with_quotes() {
+        TEST_START();
+        simdjson::builder::string_builder sb;
+        sb.escape_and_append_with_quotes("Hello, \"world\"!");
+        std::string_view p;
+        auto result = sb.view().get(p);
+        ASSERT_EQUAL(result, SUCCESS);
+        ASSERT_EQUAL(p, "\"Hello, \\\"world\\\"!\"");
+        TEST_SUCCEED();
+    }
+
+    bool append_raw() {
+        TEST_START();
+        simdjson::builder::string_builder sb;
+        sb.append_raw("Test");
+        std::string_view p;
+        auto result = sb.view().get(p);
+        ASSERT_EQUAL(result, SUCCESS);
+        ASSERT_EQUAL(p, "Test");
+        TEST_SUCCEED();
+    }
+
+    bool raw_with_length() {
+        TEST_START();
+        simdjson::builder::string_builder sb;
+        sb.append_raw("Test String", 4);
+        std::string_view p;
+        auto result = sb.view().get(p);
+        ASSERT_EQUAL(result, SUCCESS);
+        ASSERT_EQUAL(p, "Test");
+        TEST_SUCCEED();
+    }
+
+    bool string_convertion() {
+        TEST_START();
+        simdjson::builder::string_builder sb;
+        sb.append('a');
+        std::string_view p;
+        auto result = sb.view().get(p);
+        ASSERT_EQUAL(result, SUCCESS);
+        ASSERT_EQUAL(p, "a");
+        TEST_SUCCEED();
+    }
+
+    bool unicode_validation() {
+        TEST_START();
+        simdjson::builder::string_builder sb;
+        sb.append('a');
+        ASSERT_TRUE(sb.validate_unicode());
+        TEST_SUCCEED();
+    }
+
+    bool buffer_growth() {
+        TEST_START();
+        simdjson::builder::string_builder sb;
+        for(int i = 0; i < 3; ++i) {
+            sb.append('a');
+        }
+        ASSERT_EQUAL(sb.size(), 3);
+        TEST_SUCCEED();
+    }
+
+    bool run() {
+        return
+    #if SIMDJSON_EXCEPTIONS
+            string_convertion_except() &&
+    #endif
+            append_char() &&
+            append_integer() &&
+            append_float() &&
+            append_null() &&
+            clear() &&
+            escape_and_append() &&
+            escape_and_append_with_quotes() &&
+            append_raw() &&
+            raw_with_length() &&
+            string_convertion() &&
+            buffer_growth() &&
+            unicode_validation() &&
+            true;
+    }
+
+} // namespace twitter_tests
+
+int main(int argc, char *argv[]) {
+  return test_main(argc, argv, builder_tests::run);
+}

tests/builder/test_builder.h

@@ -0,0 +1,51 @@
+#ifndef ONDEMAND_TEST_BUILDER_H
+#define ONDEMAND_TEST_BUILDER_H
+
+#include <iostream>
+#include <unistd.h>
+#include "simdjson.h"
+#include "cast_tester.h"
+#include "test_macros.h"
+
+template<typename F>
+int test_main(int argc, char *argv[], const F& test_function) {
+  std::cout << std::unitbuf;
+  int c;
+  while ((c = getopt(argc, argv, "a:")) != -1) {
+    switch (c) {
+    case 'a': {
+      const simdjson::implementation *impl = simdjson::get_available_implementations()[optarg];
+      if (!impl) {
+        std::fprintf(stderr, "Unsupported architecture value -a %s\n", optarg);
+        return EXIT_FAILURE;
+      }
+      simdjson::get_active_implementation() = impl;
+      break;
+    }
+    default:
+      std::fprintf(stderr, "Unexpected argument %c\n", c);
+      return EXIT_FAILURE;
+    }
+  }
+
+  // this is put here deliberately to check that the documentation is correct (README),
+  // should this fail to compile, you should update the documentation:
+  if (simdjson::get_active_implementation()->name() == "unsupported") {
+    std::printf("unsupported CPU\n");
+    std::abort();
+  }
+  // We want to know what we are testing.
+  std::cout << "builtin_implementation -- " << simdjson::builtin_implementation()->name() << std::endl;
+  std::cout << "------------------------------------------------------------" << std::endl;
+
+  std::cout << "Running tests." << std::endl;
+  if (test_function()) {
+    std::cout << "Success!" << std::endl;
+    return EXIT_SUCCESS;
+  } else {
+    std::cerr << "FAILED." << std::endl;
+    return EXIT_FAILURE;
+  }
+}
+
+#endif // ONDEMAND_TEST_BUILDER_H

tests/dom/readme_examples.cpp

@@ -461,13 +461,6 @@ void parse_documentation_lowlevel() {
   (void)element;
 }
 
-void simplepad() {
-  std::string json = "[1]";
-  dom::parser parser;
-  dom::element doc;
-  auto error = parser.parse(simdjson::pad(json)).get(doc);
-  if(error) { exit(-1); }
-}
 
 void jsondollar() {
   dom::parser parser;
@@ -504,7 +497,6 @@ void jsonpath() {
 }
 
 int main() {
-  simplepad();
   jsonpath();
   jsondollar();
   basics_dom_1();

tests/ondemand/ondemand_readme_examples.cpp

@@ -8,15 +8,7 @@
 #endif
 using namespace std;
 using namespace simdjson;
-using error_code = simdjson::error_code;
-
-bool simplepad() {
-  std::string json = "[1]";
-  ondemand::parser parser;
-  ondemand::document doc;
-  auto error = parser.iterate(simdjson::pad(json)).get(doc);
-  return error == SUCCESS;
-}
+using error_code=simdjson::error_code;
 
 bool string1() {
   const char * data = "my data"; // 7 bytes
@@ -1926,7 +1918,6 @@ bool run() {
     && using_the_parsed_json_4()
     && using_the_parsed_json_5()
 #endif
-    && simplepad()
     && using_the_parsed_json_6()
     && json_pointer_simple()
     && json_pointer_unicode()