Producing a minor patch release

This fixes a bug that caused simdjsonTargets.cmake not to be included in
CPack-generated packages, which—unlike `cmake --install`—does not
pick up this mislabeled install component.

I git-grepped through the code base, after this change all components
are either `simdjson_Development` or `simdjson_Runtime`.

.github/workflows/ubuntu18-checkperf.yml

@@ -1,29 +0,0 @@
-name: Performance check on Ubuntu 18.04 CI (GCC 7)
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
-
-jobs:
-  ubuntu-build:
-    if: >-
-      ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
-      ! contains(toJSON(github.event.commits.*.message), '[skip github]')
-    runs-on: ubuntu-18.04
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/cache@v3
-        with:
-          path: dependencies/.cache
-          key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
-      - name: Use cmake
-        run: |
-          mkdir build &&
-          cd build &&
-          cmake  -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_ENABLE_DOM_CHECKPERF=ON -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination ..  &&
-          cmake --build . --target checkperf  &&
-          ctest --output-on-failure -R checkperf  ubuntu18-checkperf.yml

.github/workflows/ubuntu18-oldclang.yml

@@ -1,31 +0,0 @@
-name: Ubuntu 18.04 CI (LLVM 7)
-
-on: [push, pull_request]
-
-jobs:
-  ubuntu-build:
-    if: >-
-      ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
-      ! contains(toJSON(github.event.commits.*.message), '[skip github]')
-    runs-on: ubuntu-18.04
-
-    env:
-      CC: clang-7
-      CXX: clang++-7
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/cache@v3
-        with:
-          path: dependencies/.cache
-          key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
-      - name: install clang 7
-        run: |
-          sudo apt update
-          sudo apt install clang-7
-      - name: Use cmake
-        run: |
-          mkdir build &&
-          cd build &&
-          cmake  -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination ..  &&
-          cmake --build .   &&
-          ctest -j --output-on-failure -LE explicitonly

.github/workflows/ubuntu18-threadsani.yml

@@ -1,25 +0,0 @@
-name: Ubuntu 18.04 CI (GCC 7) with Thread Sanitizer
-
-on: [push, pull_request]
-
-
-jobs:
-  ubuntu-build:
-    if: >-
-      ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
-      ! contains(toJSON(github.event.commits.*.message), '[skip github]')
-    runs-on: ubuntu-18.04
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/cache@v3
-        with:
-          path: dependencies/.cache
-          key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
-      - name: Use cmake
-        run: |
-          mkdir build &&
-          cd build &&
-          cmake  -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_SANITIZE_THREADS=ON ..  &&
-          cmake --build . --target document_stream_tests --target ondemand_document_stream_tests --target parse_many_test  &&
-          ctest --output-on-failure  -R parse_many_test  &&
-          ctest --output-on-failure  -R document_stream_tests

.github/workflows/ubuntu18.yml

@@ -1,25 +0,0 @@
-name: Ubuntu 18.04 CI (GCC 7)
-
-on: [push, pull_request]
-
-jobs:
-  ubuntu-build:
-    if: >-
-      ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
-      ! contains(toJSON(github.event.commits.*.message), '[skip github]')
-    runs-on: ubuntu-18.04
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/cache@v3
-        with:
-          path: dependencies/.cache
-          key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
-      - name: Use cmake
-        run: |
-          mkdir build &&
-          cd build &&
-          cmake  -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination ..  &&
-          cmake --build .   &&
-          ctest -j --output-on-failure -LE explicitonly   &&
-          make install  &&
-          echo -e '#include <simdjson.h>\nint main(int argc,char**argv) {simdjson::dom::parser parser;simdjson::dom::element tweets = parser.load(argv[1]); }' > tmp.cpp && c++ -Idestination/include -Ldestination/lib -std=c++17 -Wl,-rpath,destination/lib -o linkandrun tmp.cpp -lsimdjson && ./linkandrun jsonexamples/twitter.json

CMakeLists.txt

@@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.14)
 project(
     simdjson
     # The version number is modified by tools/release.py
-    VERSION 3.1.2
+    VERSION 3.1.4
     DESCRIPTION "Parsing gigabytes of JSON per second"
     HOMEPAGE_URL "https://simdjson.org/"
     LANGUAGES CXX C
@@ -149,7 +149,7 @@ install(
     EXPORT simdjsonTargets
     NAMESPACE simdjson::
     DESTINATION "${SIMDJSON_INSTALL_CMAKEDIR}"
-    COMPONENT example_Development
+    COMPONENT simdjson_Development
 )
 
 # pkg-config
@@ -190,11 +190,14 @@ endif()
 
 if(is_top_project AND NOT SIMDJSON_DEVELOPER_MODE)
   message(STATUS "Building only the library. Advanced users and contributors may want to turn SIMDJSON_DEVELOPER_MODE to ON, e.g., via -D SIMDJSON_DEVELOPER_MODE=ON.")
-  return()
 elseif(SIMDJSON_DEVELOPER_MODE AND NOT is_top_project)
   message(AUTHOR_WARNING "Developer mode in simdjson is intended for the developers of simdjson")
 endif()
 
+if(NOT SIMDJSON_DEVELOPER_MODE)
+  return()
+endif()
+
 simdjson_apply_props(simdjson-internal-flags)
 
 set(

Doxyfile

@@ -38,7 +38,7 @@ PROJECT_NAME           = simdjson
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = "3.1.2"
+PROJECT_NUMBER         = "3.1.4"
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a

README.md

@@ -1,5 +1,4 @@
 
-![Ubuntu 18.04 CI](https://github.com/simdjson/simdjson/workflows/Ubuntu%2018.04%20CI%20(GCC%207)/badge.svg)
 [![Ubuntu 20.04 CI](https://github.com/simdjson/simdjson/workflows/Ubuntu%2020.04%20CI%20(GCC%209)/badge.svg)](https://simdjson.org/plots.html)
 ![VS16-CI](https://github.com/simdjson/simdjson/workflows/VS16-CI/badge.svg)
 ![MinGW64-CI](https://github.com/simdjson/simdjson/workflows/MinGW64-CI/badge.svg)
@@ -122,7 +121,8 @@ Real-world usage
 - [WatermelonDB](https://github.com/Nozbe/WatermelonDB)
 - [Apache Doris](https://github.com/apache/doris)
 - [Dgraph](https://github.com/dgraph-io/dgraph)
-
+- [UJRPC](https://github.com/unum-cloud/ujrpc)
+- [fastgltf](https://github.com/spnda/fastgltf)
 
 If you are planning to use simdjson in a product, please work from one of our releases.
 
@@ -191,7 +191,7 @@ License
 
 This code is made available under the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0.html).
 
-Under Windows, we build some tools using the windows/dirent_portable.h file (which is outside our library code): it under the liberal (business-friendly) MIT license.
+Under Windows, we build some tools using the windows/dirent_portable.h file (which is outside our library code): it is under the liberal (business-friendly) MIT license.
 
 For compilers that do not support [C++17](https://en.wikipedia.org/wiki/C%2B%2B17), we bundle the string-view library which is published under the [Boost license](http://www.boost.org/LICENSE_1_0.txt). Like the Apache license, the Boost license is a permissive license allowing commercial redistribution.
 

cmake/developer-options.cmake

@@ -187,6 +187,18 @@ else()
   message(STATUS "AVX-512 instructions are not allowed.")
 endif()
 
+option(
+    SIMDJSON_SKIPUTF8VALIDATION
+    "SKIP UTF8 VALIDATION."
+    OFF
+)
+if(SIMDJSON_SKIPUTF8VALIDATION)
+  add_compile_definitions(SIMDJSON_UTF8VALIDATION=0)
+  message(STATUS "SKIP UTF8 VALIDATION")
+else()
+  add_compile_definitions(SIMDJSON_UTF8VALIDATION=1)
+endif()
+
 include(CheckSymbolExists)
 check_symbol_exists(fork unistd.h HAVE_POSIX_FORK)
 check_symbol_exists(wait sys/wait.h HAVE_POSIX_WAIT)

dependencies/CMakeLists.txt

@@ -15,7 +15,7 @@ if(SIMDJSON_GOOGLE_BENCHMARKS)
   set_off(BENCHMARK_ENABLE_INSTALL)
   set_off(BENCHMARK_ENABLE_WERROR)
 
-  import_dependency(google_benchmarks google/benchmark d572f47)
+  import_dependency(google_benchmarks google/benchmark v1.7.1)
   add_dependency(google_benchmarks)
 endif()
 

doc/basics.md

@@ -12,7 +12,7 @@ An overview of what you need to know to use simdjson, with examples.
   - [The Basics: Loading and Parsing JSON Documents](#the-basics-loading-and-parsing-json-documents)
   - [Documents are Iterators](#documents-are-iterators)
     - [Parser, Document and JSON Scope](#parser-document-and-json-scope)
-  - [C++11 Support and string_view](#c11-support-and-string_view)
+  - [string_view](#string_view)
   - [Using the Parsed JSON](#using-the-parsed-json)
     - [Using the Parsed JSON: Additional examples](#using-the-parsed-json-additional-examples)
   - [Minifying JSON strings without parsing](#minifying-json-strings-without-parsing)
@@ -32,6 +32,7 @@ An overview of what you need to know to use simdjson, with examples.
   - [Thread Safety](#thread-safety)
   - [Standard Compliance](#standard-compliance)
   - [Backwards Compatibility](#backwards-compatibility)
+  - [Examples](#examples)
 
 
 Requirements
@@ -41,7 +42,7 @@ Requirements
 - Visual Studio 2017 or better under 64-bit Windows. Users should target a 64-bit build (x64 or ARM64) instead of a 32-bit build (x86). We support the LLVM clang compiler under Visual Studio (clangcl) as well as as the regular Visual Studio compiler. We also support MinGW 64-bit under Windows.
 
 
-Support for AVX-512 require a processor with AVX512-VBMI2 support (Ice Lake or better) under a 64-bit system and a recent compiler (LLVM clang 6 or better, GCC 8 or better, Visual Studio 2019 or better). You need a correspondingly recent assembler such as gas (2.30+) or nasm (2.14+): recent compilers usually come with recent assemblers. If you mix a recent compiler with an incompatible/old assembler (e.g., when using a recent compiler with an old Linux distribution), you may get errors at build time because the compiler produces instructions that the assembler does not recognize: you should update your assembler to match your compiler (e.g., upgrade binutils to version 2.30 or better under Linux) or use an older compiler matching the capabilities of your assembler.
+Support for AVX-512 require a processor with AVX512-VBMI2 support (Ice Lake or better, AMD Zen 4 or better) under a 64-bit system and a recent compiler (LLVM clang 6 or better, GCC 8 or better, Visual Studio 2019 or better). You need a correspondingly recent assembler such as gas (2.30+) or nasm (2.14+): recent compilers usually come with recent assemblers. If you mix a recent compiler with an incompatible/old assembler (e.g., when using a recent compiler with an old Linux distribution), you may get errors at build time because the compiler produces instructions that the assembler does not recognize: you should update your assembler to match your compiler (e.g., upgrade binutils to version 2.30 or better under Linux) or use an older compiler matching the capabilities of your assembler.
 
 We test the library on a big-endian system (IBM s390x with Linux) .
 
@@ -142,6 +143,12 @@ As required by the standard, your JSON document should be in a Unicode (UTF-8) s
 string, from the beginning to the end, needs to be valid: we do not attempt to tolerate bad
 inputs before or after a document.
 
+For efficiency reasons, simdjson requires a string with a few bytes (`simdjson::SIMDJSON_PADDING`)
+at the end, these bytes may be read but their content does not affect the parsing. In practice,
+it means that the JSON inputs should be stored in a memory region with `simdjson::SIMDJSON_PADDING`
+extra bytes at the end. You do not have to set these bytes to specific values though you may
+want to if you want to avoid runtime warnings with some sanitizers.
+
 The simdjson library offers a tree-like [API](https://en.wikipedia.org/wiki/API), which you can
 access by creating a `ondemand::parser` and calling the `iterate()` method. The iterate method
 quickly indexes the input string and may detect some errors. The following example illustrates
@@ -153,8 +160,7 @@ auto json = padded_string::load("twitter.json"); // load JSON file 'twitter.json
 ondemand::document doc = parser.iterate(json); // position a pointer at the beginning of the JSON data
 ```
 
-You can also create a padded string---for efficiency reasons, simdjson requires a string
-with a few bytes (`simdjson::SIMDJSON_PADDING`) at the end---and calling `iterate()`:
+You can also create a padded string---and call `iterate()`:
 
 ```c++
 ondemand::parser parser;
@@ -208,7 +214,9 @@ For best performance, a `parser` instance should be reused over several files: o
 needlessly reallocate memory, an expensive process. It is also possible to avoid entirely memory
 allocations during parsing when using simdjson. [See our performance notes for details](performance.md).
 
-C++11 Support and string_view
+If you need to have several documents active at once, you should have several parser instances.
+
+string_view
 -------------
 
 The simdjson library builds on compilers supporting the [C++11 standard](https://en.wikipedia.org/wiki/C%2B%2B11).
@@ -287,6 +295,11 @@ We invite you to keep the following rules in mind:
 2. Because On Demand is really just an iterator, you must fully consume the current object or array before accessing a sibling object or array.
 3. Values can only be consumed once, you should get the values and store them if you plan to need them multiple times. You are expected to access the keys of an object just once. You are expected to go through the values of an array just once.
 
+The simdjson library makes generous use of `std::string_view` instances. If you are unfamiliar
+with `std::string_view` in C++, make sure to [read the section on std::string_view](#string_view).
+They behave much like an immutable `std::string` but they require no memory allocation. You can
+create a `std::string` instance from an `std::string_view` when you need it.
+
 The following specific instructions indicate how to use the JSON when exceptions are enabled, but simdjson has full, idiomatic
 support for users who avoid exceptions. See [the simdjson error handling documentation](basics.md#error-handling) for more.
 
@@ -300,7 +313,13 @@ support for users who avoid exceptions. See [the simdjson error handling documen
   `get_uint64()`, `get_int64()`, `get_bool()`, `get_object()` and `get_array()`. After a cast or an explicit method,
   the number, string or boolean will be parsed, or the initial `[` or `{` will be verified. An exception is thrown if
   the cast is not possible. The `get_string()` returns a valid UTF-8 string, after
-  unescaping characters as needed: unmatched surrogate pairs are treated as an error. When calling `get_uint64()` and `get_int64()`, if the number does not fit in a corresponding 64-bit integer type, it is also considered an error.
+  unescaping characters as needed: unmatched surrogate pairs are treated as an error unless you
+  pass `true` (`get_string(true)`) as a parameter to get replacement characters where errors
+  occur. If you somehow need to access non-UTF-8 strings in a lossless manner
+  (e.g., if you strings contain unpaired surrogates), you may use the `get_wobbly_string()` function to get a string in the [WTF-8 format](https://simonsapin.github.io/wtf-8).
+  Or you may pass `true` as a parameter to the
+  When calling `get_uint64()` and `get_int64()`, if the number does not fit in a corresponding
+  64-bit integer type, it is also considered an error.
 
   > IMPORTANT NOTE: values can only be parsed once. Since documents are *iterators*, once you have
   > parsed a value (such as by casting to double), you cannot get at it again. It is an error to call
@@ -616,6 +635,41 @@ for (ondemand::object car : parser.iterate(cars_json)) {
 }
 ```
 
+
+The previous example had an array of objects, but we can use essentially the same
+approach with an object of objects.
+
+```c++
+ondemand::parser parser;
+auto cars_json = R"( {
+  "identifier1":{ "make": "Toyota", "model": "Camry",  "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
+  "identifier2":{ "make": "Kia",    "model": "Soul",   "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
+  "identifier3":{ "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
+} )"_padded;
+// Iterating through an array of objects
+ondemand::document doc = parser.iterate(cars_json);
+for (ondemand::field key_car : doc.get_object()) {
+  // If I need a string_view and/or, I can use key_car.unescaped_key() instead, but
+  // key_car.key() will be more performant otherwise.
+  cout << "identifier : " << key_car.key() << std::endl;
+  // I can now access the subobject:
+  ondemand::object car = key_car.value();
+  // Accessing a field by name
+  cout << "Make/Model: " << std::string_view(car["make"]) << "/" << std::string_view(car["model"]) << endl;
+
+  // Casting a JSON element to an integer
+  uint64_t year = car["year"];
+  cout << "- This car is " << 2020 - year << "years old." << endl;
+
+  // Iterating through an array of floats
+  double total_tire_pressure = 0;
+  for (double tire_pressure : car["tire_pressure"]) {
+    total_tire_pressure += tire_pressure;
+  }
+  cout << "- Average tire pressure: " << (total_tire_pressure / 4) << endl;
+}
+```
+
 The following example illustrates how you may also iterate through object values, effectively visiting all key-value pairs in the object.
 
 ```C++
@@ -1017,6 +1071,8 @@ having to handle exceptions.
   error = doc.get_object().get(object);
   if(error) { return false; }
   for(auto field : object) {
+    // We could replace 'field.key() with field.unescaped_key(),
+    // and ondemand::raw_json_string by std::string_view.
     ondemand::raw_json_string keyv;
     error = field.key().get(keyv);
     if(error) { return false; }
@@ -1574,3 +1630,190 @@ Backwards Compatibility
 The only header file supported by simdjson is `simdjson.h`. Older versions of simdjson published a
 number of other include files such as `document.h` or `ParsedJson.h` alongside `simdjson.h`; these headers
 may be moved or removed in future versions.
+
+Examples
+--------
+
+Some users like to have example. The following code samples illustrate how to process specific JSON inputs.
+For simplicity, we do not include full error support: this code would throw exceptions on error.
+
+
+* Example 1: ZuluBBox
+
+```C++
+struct ZuluBBox {
+  double xmin;
+  double ymin;
+  double width;
+  double height;
+
+  void print() {
+    std::cout << xmin << ", " << ymin << ", " << width << ", " << height
+              << std::endl;
+  }
+};
+
+bool example() {
+
+  auto json = R"+( {
+  "ZuluROI": {
+    "ZuluBBox": {
+      "xmin": 0,
+      "ymin": 0,
+      "width": 1,
+      "height": 1
+    },
+    "SubObjects": [
+      {
+        "ZuluDetection": {
+          "label": "car",
+          "class_id": 3,
+          "confidence": 0.7587034106254578,
+          "ZuluBBox": {
+            "xmin": 0.3843536376953125,
+            "ymin": 0.4532909393310547,
+            "width": 0.09115534275770187,
+            "height": 0.04127710685133934
+          },
+          "SubObjects": []
+        }
+      },
+      {
+        "ZuluDetection": {
+          "label": "car",
+          "class_id": 3,
+          "confidence": 0.6718865633010864,
+          "ZuluBBox": {
+            "xmin": 0.7500002980232239,
+            "ymin": 0.5212296843528748,
+            "width": 0.07592231780290604,
+            "height": 0.038947589695453644
+          },
+          "SubObjects": []
+        }
+      },
+      {
+        "ZuluDetection": {
+          "label": "car",
+          "class_id": 3,
+          "confidence": 0.5806200504302979,
+          "ZuluBBox": {
+            "xmin": 0.9025363922119141,
+            "ymin": 0.5925348401069641,
+            "width": 0.05478987470269203,
+            "height": 0.046337299048900604
+          },
+          "SubObjects": []
+        }
+      }
+    ]
+  },
+  "timestamp (ms)": 1677085594421,
+  "buffer_offset": 35673
+} )+"_padded;
+  ondemand::parser parser;
+  ondemand::document doc = parser.iterate(json);
+  ondemand::object root_object = doc.get_object();
+  ondemand::object roi_object = root_object["ZuluROI"];
+
+  ondemand::object box_roi_object = roi_object["ZuluBBox"];
+  ZuluBBox box = {
+      double(box_roi_object["xmin"]), double(box_roi_object["ymin"]),
+      double(box_roi_object["width"]), double(box_roi_object["height"])};
+  box.print();
+
+  for (ondemand::object value : roi_object["SubObjects"]) {
+    ondemand::object detect = value["ZuluDetection"];
+    std::cout << detect["label"].get_string() << std::endl;
+    std::cout << detect["class_id"].get_uint64() << std::endl;
+    std::cout << detect["confidence"].get_double() << std::endl;
+
+    ondemand::object vbox_roi_object = detect["ZuluBBox"];
+    ZuluBBox vbox = {
+        double(vbox_roi_object["xmin"]), double(vbox_roi_object["ymin"]),
+        double(vbox_roi_object["width"]), double(vbox_roi_object["height"])};
+    vbox.print();
+  }
+
+  std::cout << root_object["timestamp (ms)"].get_uint64() << std::endl;
+  std::cout << root_object["buffer_offset"].get_uint64() << std::endl;
+  return true;
+}
+```
+
+
+* Example 2: Demos
+
+```C++
+bool example() {
+  auto json = R"+( {
+    "5f08a730b280e54fd1e75a7046b93fdc": {
+        "file": "/DEMOS/0-9/10_Orbyte.sid",
+        "len": [
+            "1:17"
+        ],
+        "loud": [
+            "-22.8"
+        ],
+        "name": "10 Orbyte",
+        "author": "Michael Becker (Premium)",
+        "release": "2014 Tristar & Red Sector Inc.",
+        "bits": 20
+    },
+    "2727236ead44a62f0c6e01f6dd4dc484": {
+        "file": "/DEMOS/0-9/12345.sid",
+        "len": [
+            "0:56"
+        ],
+        "loud": [
+            "-33.3"
+        ],
+        "name": "12345",
+        "author": "Beal",
+        "release": "1988 Beal",
+        "bits": 20
+    },
+    "7ea765fce6c0f92570b18adc7bf52f54": {
+        "file": "/DEMOS/0-9/128_Byte_Blues_BASIC.sid",
+        "len": [
+            "0:18"
+        ],
+        "loud": [
+            "-27.1"
+        ],
+        "name": "128 Byte Blues",
+        "author": "Leonard J. Paul (Freaky DNA)",
+        "release": "2005 Freaky DNA",
+        "bits": 62
+    }
+} )+"_padded;
+  ondemand::parser parser;
+  ondemand::document doc = parser.iterate(json);
+  ondemand::object root_object = doc.get_object();
+  for(auto key_value : root_object) {
+    // could get std::string_view with 'unescaped_key()':
+    std::cout << "key: " << key_value.key() << std::endl;
+    ondemand::object obj = key_value.value();
+
+    std::cout << "file: " << std::string_view(obj["file"]) << std::endl;
+
+    std::cout << "len: ";
+    for(std::string_view values : obj["len"]) {
+      std::cout << values << std::endl;
+    }
+    std::cout << std::endl;
+
+    std::cout << "loud: ";
+    for(std::string_view values : obj["loud"]) {
+      std::cout << values << std::endl;
+    }
+    std::cout << std::endl;
+
+    std::cout << "name: " << std::string_view(obj["name"]) << std::endl;
+    std::cout << "author: " << std::string_view(obj["author"]) << std::endl;
+    std::cout << "release: " << std::string_view(obj["release"]) << std::endl;
+    std::cout << "bits: " << uint64_t(obj["bits"]) << std::endl;
+  }
+  return true;
+}
+```

doc/performance.md

@@ -160,8 +160,7 @@ You should not expect the simdjson library to cause *downclocking* of your recen
 - [Whenever 512-bit AVX-512 instructions are used](https://lemire.me/blog/2018/09/07/avx-512-when-and-how-to-use-these-new-instructions/).
 - Whenever heavy 256-bit or wider instructions are used. Heavy instructions are those involving floating point operations or integer multiplications (since these execute on the floating point unit).
 
-The simdjson library does not make use of heavy 256-bit instructions. We do use vectorized multiplications, but only using 128-bit registers. Thus there should be no downclocking due to simdjson on recent processors, except when AVX-512 is allowed and
-detected. However, we only allow AVX-512 on recent processors (Ice Lake/Tiger Lake or better, AMD Zen 4 or better) where [little to no frequency throttling is expected](https://travisdowns.github.io/blog/2020/08/19/icl-avx512-freq.html). If you can still concerned, you can easily disable AVX-512 with the CMake option `SIMDJSON_AVX512_ALLOWED` set to `OFF` (e.g., `cmake -D SIMDJSON_AVX512_ALLOWED=OFF -B build && cmake --build build`) or by setting
+The simdjson library does not generally make use of heavy 256-bit instructions. On AVX2 kernels, we use vectorized multiplications, but only using 128-bit registers. On recent processors (Ice Lake/Tiger Lake or better, AMD Zen 4 or better) [no frequency throttling is found](https://travisdowns.github.io/blog/2020/08/19/icl-avx512-freq.html) due to SIMD instructions: we are thus more aggressive with SIMD on these machines. If you can still concerned, you can easily disable AVX-512 with the CMake option `SIMDJSON_AVX512_ALLOWED` set to `OFF` (e.g., `cmake -D SIMDJSON_AVX512_ALLOWED=OFF -B build && cmake --build build`) or by setting
 the macro `SIMDJSON_AVX512_ALLOWED` to `0` in C++ prior to importing the headers.
 
 You may still be worried about which SIMD instruction set is used by simdjson.  Thankfully,  [you can always determine and change which architecture-specific implementation is used](implementation-selection.md) by simdjson. Thus even if your CPU supports AVX2, you do not need to use AVX2. You are in control.

include/simdjson/common_defs.h

@@ -329,4 +329,8 @@ namespace std {
 #define SIMDJSON_DEVELOPMENT_ASSERT(expr) do { } while (0)
 #endif
 
+#ifndef SIMDJSON_UTF8VALIDATION
+#define SIMDJSON_UTF8VALIDATION 1
+#endif
+
 #endif // SIMDJSON_COMMON_DEFS_H

include/simdjson/dom/parser.h

@@ -145,7 +145,9 @@ public:
    * ### REQUIRED: Buffer Padding
    *
    * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what
-   * those bytes are initialized to, as long as they are allocated.
+   * those bytes are initialized to, as long as they are allocated. These bytes will be read: if you
+   * using a sanitizer that verifies that no uninitialized byte is read, then you should initialize the
+   * SIMDJSON_PADDING bytes to avoid runtime warnings.
    *
    * If realloc_if_needed is true (the default), it is assumed that the buffer does *not* have enough padding,
    * and it is copied into an enlarged temporary buffer before parsing. Thus the following is safe:
@@ -408,7 +410,9 @@ public:
    * ### REQUIRED: Buffer Padding
    *
    * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what
-   * those bytes are initialized to, as long as they are allocated.
+   * those bytes are initialized to, as long as they are allocated. These bytes will be read: if you
+   * using a sanitizer that verifies that no uninitialized byte is read, then you should initialize the
+   * SIMDJSON_PADDING bytes to avoid runtime warnings.
    *
    * ### Threads
    *

include/simdjson/generic/dom_parser_implementation.h

@@ -35,7 +35,8 @@ public:
   simdjson_warn_unused error_code stage1(const uint8_t *buf, size_t len, stage1_mode partial) noexcept final;
   simdjson_warn_unused error_code stage2(dom::document &doc) noexcept final;
   simdjson_warn_unused error_code stage2_next(dom::document &doc) noexcept final;
-  simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst) const noexcept final;
+  simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept final;
+  simdjson_warn_unused uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept final;
   inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final;
   inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final;
 private:

include/simdjson/generic/ondemand/document-inl.h

@@ -94,8 +94,11 @@ simdjson_inline simdjson_result<double> document::get_double() noexcept {
 simdjson_inline simdjson_result<double> document::get_double_in_string() noexcept {
   return get_root_value_iterator().get_root_double_in_string(true);
 }
-simdjson_inline simdjson_result<std::string_view> document::get_string() noexcept {
-  return get_root_value_iterator().get_root_string(true);
+simdjson_inline simdjson_result<std::string_view> document::get_string(bool allow_replacement) noexcept {
+  return get_root_value_iterator().get_root_string(true, allow_replacement);
+}
+simdjson_inline simdjson_result<std::string_view> document::get_wobbly_string() noexcept {
+  return get_root_value_iterator().get_root_wobbly_string(true);
 }
 simdjson_inline simdjson_result<raw_json_string> document::get_raw_json_string() noexcept {
   return get_root_value_iterator().get_root_raw_json_string(true);
@@ -110,7 +113,7 @@ simdjson_inline simdjson_result<bool> document::is_null() noexcept {
 template<> simdjson_inline simdjson_result<array> document::get() & noexcept { return get_array(); }
 template<> simdjson_inline simdjson_result<object> document::get() & noexcept { return get_object(); }
 template<> simdjson_inline simdjson_result<raw_json_string> document::get() & noexcept { return get_raw_json_string(); }
-template<> simdjson_inline simdjson_result<std::string_view> document::get() & noexcept { return get_string(); }
+template<> simdjson_inline simdjson_result<std::string_view> document::get() & noexcept { return get_string(false); }
 template<> simdjson_inline simdjson_result<double> document::get() & noexcept { return get_double(); }
 template<> simdjson_inline simdjson_result<uint64_t> document::get() & noexcept { return get_uint64(); }
 template<> simdjson_inline simdjson_result<int64_t> document::get() & noexcept { return get_int64(); }
@@ -118,7 +121,7 @@ template<> simdjson_inline simdjson_result<bool> document::get() & noexcept { re
 template<> simdjson_inline simdjson_result<value> document::get() & noexcept { return get_value(); }
 
 template<> simdjson_inline simdjson_result<raw_json_string> document::get() && noexcept { return get_raw_json_string(); }
-template<> simdjson_inline simdjson_result<std::string_view> document::get() && noexcept { return get_string(); }
+template<> simdjson_inline simdjson_result<std::string_view> document::get() && noexcept { return get_string(false); }
 template<> simdjson_inline simdjson_result<double> document::get() && noexcept { return std::forward<document>(*this).get_double(); }
 template<> simdjson_inline simdjson_result<uint64_t> document::get() && noexcept { return std::forward<document>(*this).get_uint64(); }
 template<> simdjson_inline simdjson_result<int64_t> document::get() && noexcept { return std::forward<document>(*this).get_int64(); }
@@ -138,7 +141,7 @@ simdjson_inline document::operator object() & noexcept(false) { return get_objec
 simdjson_inline document::operator uint64_t() noexcept(false) { return get_uint64(); }
 simdjson_inline document::operator int64_t() noexcept(false) { return get_int64(); }
 simdjson_inline document::operator double() noexcept(false) { return get_double(); }
-simdjson_inline document::operator std::string_view() noexcept(false) { return get_string(); }
+simdjson_inline document::operator std::string_view() noexcept(false) { return get_string(false); }
 simdjson_inline document::operator raw_json_string() noexcept(false) { return get_raw_json_string(); }
 simdjson_inline document::operator bool() noexcept(false) { return get_bool(); }
 simdjson_inline document::operator value() noexcept(false) { return get_value(); }
@@ -359,9 +362,13 @@ simdjson_inline simdjson_result<double> simdjson_result<SIMDJSON_IMPLEMENTATION:
   if (error()) { return error(); }
   return first.get_double_in_string();
 }
-simdjson_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::get_string() noexcept {
+simdjson_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::get_string(bool allow_replacement) noexcept {
   if (error()) { return error(); }
-  return first.get_string();
+  return first.get_string(allow_replacement);
+}
+simdjson_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::get_wobbly_string() noexcept {
+  if (error()) { return error(); }
+  return first.get_wobbly_string();
 }
 simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::raw_json_string> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::get_raw_json_string() noexcept {
   if (error()) { return error(); }
@@ -538,7 +545,8 @@ simdjson_inline simdjson_result<int64_t> document_reference::get_int64() noexcep
 simdjson_inline simdjson_result<int64_t> document_reference::get_int64_in_string() noexcept { return doc->get_root_value_iterator().get_root_int64_in_string(false); }
 simdjson_inline simdjson_result<double> document_reference::get_double() noexcept { return doc->get_root_value_iterator().get_root_double(false); }
 simdjson_inline simdjson_result<double> document_reference::get_double_in_string() noexcept { return doc->get_root_value_iterator().get_root_double(false); }
-simdjson_inline simdjson_result<std::string_view> document_reference::get_string() noexcept { return doc->get_root_value_iterator().get_root_string(false); }
+simdjson_inline simdjson_result<std::string_view> document_reference::get_string(bool allow_replacement) noexcept { return doc->get_root_value_iterator().get_root_string(false, allow_replacement); }
+simdjson_inline simdjson_result<std::string_view> document_reference::get_wobbly_string() noexcept { return doc->get_root_value_iterator().get_root_wobbly_string(false); }
 simdjson_inline simdjson_result<raw_json_string> document_reference::get_raw_json_string() noexcept { return doc->get_root_value_iterator().get_root_raw_json_string(false); }
 simdjson_inline simdjson_result<bool> document_reference::get_bool() noexcept { return doc->get_root_value_iterator().get_root_bool(false); }
 simdjson_inline simdjson_result<value> document_reference::get_value() noexcept { return doc->get_value(); }
@@ -670,9 +678,13 @@ simdjson_inline simdjson_result<double> simdjson_result<SIMDJSON_IMPLEMENTATION:
   if (error()) { return error(); }
   return first.get_double_in_string();
 }
-simdjson_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::get_string() noexcept {
+simdjson_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::get_string(bool allow_replacement) noexcept {
   if (error()) { return error(); }
-  return first.get_string();
+  return first.get_string(allow_replacement);
+}
+simdjson_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::get_wobbly_string() noexcept {
+  if (error()) { return error(); }
+  return first.get_wobbly_string();
 }
 simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::raw_json_string> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::get_raw_json_string() noexcept {
   if (error()) { return error(); }

include/simdjson/generic/ondemand/document.h

@@ -96,11 +96,24 @@ public:
    *
    * Important: Calling get_string() twice on the same document is an error.
    *
+   * @param Whether to allow a replacement character for unmatched surrogate pairs.
    * @returns An UTF-8 string. The string is stored in the parser and will be invalidated the next
    *          time it parses a document or when it is destroyed.
    * @returns INCORRECT_TYPE if the JSON value is not a string.
    */
-  simdjson_inline simdjson_result<std::string_view> get_string() noexcept;
+  simdjson_inline simdjson_result<std::string_view> get_string(bool allow_replacement = false) noexcept;
+  /**
+   * Cast this JSON value to a string.
+   *
+   * The string is not guaranteed to be valid UTF-8. See https://simonsapin.github.io/wtf-8/
+   *
+   * Important: Calling get_wobbly_string() twice on the same document is an error.
+   *
+   * @returns An UTF-8 string. The string is stored in the parser and will be invalidated the next
+   *          time it parses a document or when it is destroyed.
+   * @returns INCORRECT_TYPE if the JSON value is not a string.
+   */
+  simdjson_inline simdjson_result<std::string_view> get_wobbly_string() noexcept;
   /**
    * Cast this JSON value to a raw_json_string.
    *
@@ -599,7 +612,8 @@ public:
   simdjson_inline simdjson_result<int64_t> get_int64_in_string() noexcept;
   simdjson_inline simdjson_result<double> get_double() noexcept;
   simdjson_inline simdjson_result<double> get_double_in_string() noexcept;
-  simdjson_inline simdjson_result<std::string_view> get_string() noexcept;
+  simdjson_inline simdjson_result<std::string_view> get_string(bool allow_replacement = false) noexcept;
+  simdjson_inline simdjson_result<std::string_view> get_wobbly_string() noexcept;
   simdjson_inline simdjson_result<raw_json_string> get_raw_json_string() noexcept;
   simdjson_inline simdjson_result<bool> get_bool() noexcept;
   simdjson_inline simdjson_result<value> get_value() noexcept;
@@ -667,7 +681,8 @@ public:
   simdjson_inline simdjson_result<int64_t> get_int64_in_string() noexcept;
   simdjson_inline simdjson_result<double> get_double() noexcept;
   simdjson_inline simdjson_result<double> get_double_in_string() noexcept;
-  simdjson_inline simdjson_result<std::string_view> get_string() noexcept;
+  simdjson_inline simdjson_result<std::string_view> get_string(bool allow_replacement = false) noexcept;
+  simdjson_inline simdjson_result<std::string_view> get_wobbly_string() noexcept;
   simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::raw_json_string> get_raw_json_string() noexcept;
   simdjson_inline simdjson_result<bool> get_bool() noexcept;
   simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> get_value() noexcept;
@@ -737,7 +752,8 @@ public:
   simdjson_inline simdjson_result<int64_t> get_int64_in_string() noexcept;
   simdjson_inline simdjson_result<double> get_double() noexcept;
   simdjson_inline simdjson_result<double> get_double_in_string() noexcept;
-  simdjson_inline simdjson_result<std::string_view> get_string() noexcept;
+  simdjson_inline simdjson_result<std::string_view> get_string(bool allow_replacement = false) noexcept;
+  simdjson_inline simdjson_result<std::string_view> get_wobbly_string() noexcept;
   simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::raw_json_string> get_raw_json_string() noexcept;
   simdjson_inline simdjson_result<bool> get_bool() noexcept;
   simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> get_value() noexcept;

include/simdjson/generic/ondemand/field-inl.h

@@ -21,9 +21,9 @@ simdjson_inline simdjson_result<field> field::start(const value_iterator &parent
     return field(key, parent_iter.child());
 }
 
-simdjson_inline simdjson_warn_unused simdjson_result<std::string_view> field::unescaped_key() noexcept {
+simdjson_inline simdjson_warn_unused simdjson_result<std::string_view> field::unescaped_key(bool allow_replacement) noexcept {
   SIMDJSON_ASSUME(first.buf != nullptr); // We would like to call .alive() but Visual Studio won't let us.
-  simdjson_result<std::string_view> answer = first.unescape(second.iter.json_iter());
+  simdjson_result<std::string_view> answer = first.unescape(second.iter.json_iter(), allow_replacement);
   first.consume();
   return answer;
 }
@@ -66,9 +66,9 @@ simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::raw_json_stri
   if (error()) { return error(); }
   return first.key();
 }
-simdjson_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::field>::unescaped_key() noexcept {
+simdjson_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::field>::unescaped_key(bool allow_replacement) noexcept {
   if (error()) { return error(); }
-  return first.unescaped_key();
+  return first.unescaped_key(allow_replacement);
 }
 simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::field>::value() noexcept {
   if (error()) { return error(); }

include/simdjson/generic/ondemand/field.h

@@ -28,7 +28,7 @@ public:
    * This consumes the key: once you have called unescaped_key(), you cannot
    * call it again nor can you call key().
    */
-  simdjson_inline simdjson_warn_unused simdjson_result<std::string_view> unescaped_key() noexcept;
+  simdjson_inline simdjson_warn_unused simdjson_result<std::string_view> unescaped_key(bool allow_replacement) noexcept;
   /**
    * Get the key as a raw_json_string. Can be used for direct comparison with
    * an unescaped C string: e.g., key() == "test".
@@ -64,7 +64,7 @@ public:
   simdjson_inline simdjson_result(error_code error) noexcept; ///< @private
   simdjson_inline simdjson_result() noexcept = default;
 
-  simdjson_inline simdjson_result<std::string_view> unescaped_key() noexcept;
+  simdjson_inline simdjson_result<std::string_view> unescaped_key(bool allow_replacement = false) noexcept;
   simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::raw_json_string> key() noexcept;
   simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> value() noexcept;
 };

include/simdjson/generic/ondemand/json_iterator-inl.h

@@ -316,8 +316,12 @@ simdjson_inline token_position json_iterator::position() const noexcept {
   return token.position();
 }
 
-simdjson_inline simdjson_result<std::string_view> json_iterator::unescape(raw_json_string in) noexcept {
-  return parser->unescape(in, _string_buf_loc);
+simdjson_inline simdjson_result<std::string_view> json_iterator::unescape(raw_json_string in, bool allow_replacement) noexcept {
+  return parser->unescape(in, _string_buf_loc, allow_replacement);
+}
+
+simdjson_inline simdjson_result<std::string_view> json_iterator::unescape_wobbly(raw_json_string in) noexcept {
+  return parser->unescape_wobbly(in, _string_buf_loc);
 }
 
 simdjson_inline void json_iterator::reenter_child(token_position position, depth_t child_depth) noexcept {

include/simdjson/generic/ondemand/json_iterator.h

@@ -246,7 +246,8 @@ public:
    * Each raw_json_string should be unescaped once, or else the string buffer might
    * overflow.
    */
-  simdjson_inline simdjson_result<std::string_view> unescape(raw_json_string in) noexcept;
+  simdjson_inline simdjson_result<std::string_view> unescape(raw_json_string in, bool allow_replacement) noexcept;
+  simdjson_inline simdjson_result<std::string_view> unescape_wobbly(raw_json_string in) noexcept;
   simdjson_inline void reenter_child(token_position position, depth_t child_depth) noexcept;
 
 #if SIMDJSON_DEVELOPMENT_CHECKS

include/simdjson/generic/ondemand/parser-inl.h

@@ -116,8 +116,16 @@ simdjson_inline void parser::set_max_capacity(size_t max_capacity) noexcept {
   }
 }
 
-simdjson_inline simdjson_warn_unused simdjson_result<std::string_view> parser::unescape(raw_json_string in, uint8_t *&dst) const noexcept {
-  uint8_t *end = implementation->parse_string(in.buf, dst);
+simdjson_inline simdjson_warn_unused simdjson_result<std::string_view> parser::unescape(raw_json_string in, uint8_t *&dst, bool allow_replacement) const noexcept {
+  uint8_t *end = implementation->parse_string(in.buf, dst, allow_replacement);
+  if (!end) { return STRING_ERROR; }
+  std::string_view result(reinterpret_cast<const char *>(dst), end-dst);
+  dst = end;
+  return result;
+}
+
+simdjson_inline simdjson_warn_unused simdjson_result<std::string_view> parser::unescape_wobbly(raw_json_string in, uint8_t *&dst) const noexcept {
+  uint8_t *end = implementation->parse_wobbly_string(in.buf, dst);
   if (!end) { return STRING_ERROR; }
   std::string_view result(reinterpret_cast<const char *>(dst), end-dst);
   dst = end;

include/simdjson/generic/ondemand/parser.h

@@ -78,7 +78,9 @@ public:
    * ### REQUIRED: Buffer Padding
    *
    * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what
-   * those bytes are initialized to, as long as they are allocated.
+   * those bytes are initialized to, as long as they are allocated. These bytes will be read: if you
+   * using a sanitizer that verifies that no uninitialized byte is read, then you should initialize the
+   * SIMDJSON_PADDING bytes to avoid runtime warnings.
    *
    * @param json The JSON to parse.
    * @param len The length of the JSON.
@@ -134,7 +136,9 @@ public:
    * ### REQUIRED: Buffer Padding
    *
    * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what
-   * those bytes are initialized to, as long as they are allocated.
+   * those bytes are initialized to, as long as they are allocated. These bytes will be read: if you
+   * using a sanitizer that verifies that no uninitialized byte is read, then you should initialize the
+   * SIMDJSON_PADDING bytes to avoid runtime warnings.
    *
    * @param json The JSON to parse.
    *
@@ -188,7 +192,9 @@ public:
    * ### REQUIRED: Buffer Padding
    *
    * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what
-   * those bytes are initialized to, as long as they are allocated.
+   * those bytes are initialized to, as long as they are allocated. These bytes will be read: if you
+   * using a sanitizer that verifies that no uninitialized byte is read, then you should initialize the
+   * SIMDJSON_PADDING bytes to avoid runtime warnings.
    *
    * ### Threads
    *
@@ -263,6 +269,31 @@ public:
 
   /**
    * Unescape this JSON string, replacing \\ with \, \n with newline, etc. to a user-provided buffer.
+   * The result must be valid UTF-8.
+   * The provided pointer is advanced to the end of the string by reference, and a string_view instance
+   * is returned. You can ensure that your buffer is large enough by allocating a block of memory at least
+   * as large as the input JSON plus SIMDJSON_PADDING and then unescape all strings to this one buffer.
+   *
+   * This unescape function is a low-level function. If you want a more user-friendly approach, you should
+   * avoid raw_json_string instances (e.g., by calling unescaped_key() instead of key() or get_string()
+   * instead of get_raw_json_string()).
+   *
+   * ## IMPORTANT: string_view lifetime
+   *
+   * The string_view is only valid as long as the bytes in dst.
+   *
+   * @param raw_json_string input
+   * @param dst A pointer to a buffer at least large enough to write this string as well as
+   *            an additional SIMDJSON_PADDING bytes.
+   * @param allow_replacement Whether we allow a replacement if the input string contains unmatched surrogate pairs.
+   * @return A string_view pointing at the unescaped string in dst
+   * @error STRING_ERROR if escapes are incorrect.
+   */
+  simdjson_inline simdjson_result<std::string_view> unescape(raw_json_string in, uint8_t *&dst, bool allow_replacement = false) const noexcept;
+
+  /**
+   * Unescape this JSON string, replacing \\ with \, \n with newline, etc. to a user-provided buffer.
+   * The result may not be valid UTF-8. See https://simonsapin.github.io/wtf-8/
    * The provided pointer is advanced to the end of the string by reference, and a string_view instance
    * is returned. You can ensure that your buffer is large enough by allocating a block of memory at least
    * as large as the input JSON plus SIMDJSON_PADDING and then unescape all strings to this one buffer.
@@ -281,7 +312,8 @@ public:
    * @return A string_view pointing at the unescaped string in dst
    * @error STRING_ERROR if escapes are incorrect.
    */
-  simdjson_inline simdjson_result<std::string_view> unescape(raw_json_string in, uint8_t *&dst) const noexcept;
+  simdjson_inline simdjson_result<std::string_view> unescape_wobbly(raw_json_string in, uint8_t *&dst) const noexcept;
+
 private:
   /** @private [for benchmarking access] The implementation to use */
   std::unique_ptr<internal::dom_parser_implementation> implementation{};

include/simdjson/generic/ondemand/raw_json_string-inl.h

@@ -143,10 +143,13 @@ simdjson_unused simdjson_inline bool operator!=(std::string_view c, const raw_js
 }
 
 
-simdjson_inline simdjson_warn_unused simdjson_result<std::string_view> raw_json_string::unescape(json_iterator &iter) const noexcept {
-  return iter.unescape(*this);
+simdjson_inline simdjson_warn_unused simdjson_result<std::string_view> raw_json_string::unescape(json_iterator &iter, bool allow_replacement) const noexcept {
+  return iter.unescape(*this, allow_replacement);
 }
 
+simdjson_inline simdjson_warn_unused simdjson_result<std::string_view> raw_json_string::unescape_wobbly(json_iterator &iter) const noexcept {
+  return iter.unescape_wobbly(*this);
+}
 
 simdjson_unused simdjson_inline std::ostream &operator<<(std::ostream &out, const raw_json_string &str) noexcept {
   bool in_escape = false;
@@ -177,9 +180,12 @@ simdjson_inline simdjson_result<const char *> simdjson_result<SIMDJSON_IMPLEMENT
   if (error()) { return error(); }
   return first.raw();
 }
-simdjson_inline simdjson_warn_unused simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::raw_json_string>::unescape(SIMDJSON_IMPLEMENTATION::ondemand::json_iterator &iter) const noexcept {
+simdjson_inline simdjson_warn_unused simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::raw_json_string>::unescape(SIMDJSON_IMPLEMENTATION::ondemand::json_iterator &iter, bool allow_replacement) const noexcept {
   if (error()) { return error(); }
-  return first.unescape(iter);
+  return first.unescape(iter, allow_replacement);
+}
+simdjson_inline simdjson_warn_unused simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::raw_json_string>::unescape_wobbly(SIMDJSON_IMPLEMENTATION::ondemand::json_iterator &iter) const noexcept {
+  if (error()) { return error(); }
+  return first.unescape_wobbly(iter);
 }
-
 } // namespace simdjson

include/simdjson/generic/ondemand/raw_json_string.h

@@ -139,6 +139,20 @@ private:
 
   /**
    * Unescape this JSON string, replacing \\ with \, \n with newline, etc.
+   * The result will be a valid UTF-8.
+   *
+   * ## IMPORTANT: string_view lifetime
+   *
+   * The string_view is only valid until the next parse() call on the parser.
+   *
+   * @param iter A json_iterator, which contains a buffer where the string will be written.
+   * @param allow_replacement Whether we allow replacement of invalid surrogate pairs.
+   */
+  simdjson_inline simdjson_warn_unused simdjson_result<std::string_view> unescape(json_iterator &iter, bool allow_replacement) const noexcept;
+
+  /**
+   * Unescape this JSON string, replacing \\ with \, \n with newline, etc.
+   * The result may not be a valid UTF-8. https://simonsapin.github.io/wtf-8/
    *
    * ## IMPORTANT: string_view lifetime
    *
@@ -146,8 +160,7 @@ private:
    *
    * @param iter A json_iterator, which contains a buffer where the string will be written.
    */
-  simdjson_inline simdjson_warn_unused simdjson_result<std::string_view> unescape(json_iterator &iter) const noexcept;
-
+  simdjson_inline simdjson_warn_unused simdjson_result<std::string_view> unescape_wobbly(json_iterator &iter) const noexcept;
   const uint8_t * buf{};
   friend class object;
   friend class field;
@@ -182,7 +195,8 @@ public:
   simdjson_inline ~simdjson_result() noexcept = default; ///< @private
 
   simdjson_inline simdjson_result<const char *> raw() const noexcept;
-  simdjson_inline simdjson_warn_unused simdjson_result<std::string_view> unescape(SIMDJSON_IMPLEMENTATION::ondemand::json_iterator &iter) const noexcept;
+  simdjson_inline simdjson_warn_unused simdjson_result<std::string_view> unescape(SIMDJSON_IMPLEMENTATION::ondemand::json_iterator &iter, bool allow_replacement) const noexcept;
+  simdjson_inline simdjson_warn_unused simdjson_result<std::string_view> unescape_wobbly(SIMDJSON_IMPLEMENTATION::ondemand::json_iterator &iter) const noexcept;
 };
 
 } // namespace simdjson

include/simdjson/generic/ondemand/value-inl.h

@@ -30,8 +30,11 @@ simdjson_inline simdjson_result<object> value::start_or_resume_object() noexcept
 simdjson_inline simdjson_result<raw_json_string> value::get_raw_json_string() noexcept {
   return iter.get_raw_json_string();
 }
-simdjson_inline simdjson_result<std::string_view> value::get_string() noexcept {
-  return iter.get_string();
+simdjson_inline simdjson_result<std::string_view> value::get_string(bool allow_replacement) noexcept {
+  return iter.get_string(allow_replacement);
+}
+simdjson_inline simdjson_result<std::string_view> value::get_wobbly_string() noexcept {
+  return iter.get_wobbly_string();
 }
 simdjson_inline simdjson_result<double> value::get_double() noexcept {
   return iter.get_double();
@@ -60,7 +63,7 @@ simdjson_inline simdjson_result<bool> value::is_null() noexcept {
 template<> simdjson_inline simdjson_result<array> value::get() noexcept { return get_array(); }
 template<> simdjson_inline simdjson_result<object> value::get() noexcept { return get_object(); }
 template<> simdjson_inline simdjson_result<raw_json_string> value::get() noexcept { return get_raw_json_string(); }
-template<> simdjson_inline simdjson_result<std::string_view> value::get() noexcept { return get_string(); }
+template<> simdjson_inline simdjson_result<std::string_view> value::get() noexcept { return get_string(false); }
 template<> simdjson_inline simdjson_result<number> value::get() noexcept { return get_number(); }
 template<> simdjson_inline simdjson_result<double> value::get() noexcept { return get_double(); }
 template<> simdjson_inline simdjson_result<uint64_t> value::get() noexcept { return get_uint64(); }
@@ -88,7 +91,7 @@ simdjson_inline value::operator double() noexcept(false) {
   return get_double();
 }
 simdjson_inline value::operator std::string_view() noexcept(false) {
-  return get_string();
+  return get_string(false);
 }
 simdjson_inline value::operator raw_json_string() noexcept(false) {
   return get_raw_json_string();
@@ -298,9 +301,13 @@ simdjson_inline simdjson_result<double> simdjson_result<SIMDJSON_IMPLEMENTATION:
   if (error()) { return error(); }
   return first.get_double_in_string();
 }
-simdjson_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::get_string() noexcept {
+simdjson_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::get_string(bool allow_replacement) noexcept {
   if (error()) { return error(); }
-  return first.get_string();
+  return first.get_string(allow_replacement);
+}
+simdjson_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::get_wobbly_string() noexcept {
+  if (error()) { return error(); }
+  return first.get_wobbly_string();
 }
 simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::raw_json_string> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::get_raw_json_string() noexcept {
   if (error()) { return error(); }

include/simdjson/generic/ondemand/value.h

@@ -129,8 +129,23 @@ public:
    *          time it parses a document or when it is destroyed.
    * @returns INCORRECT_TYPE if the JSON value is not a string.
    */
-  simdjson_inline simdjson_result<std::string_view> get_string() noexcept;
+  simdjson_inline simdjson_result<std::string_view> get_string(bool allow_replacement = false) noexcept;
 
+
+  /**
+   * Cast this JSON value to a "wobbly" string.
+   *
+   * The string is may not be a valid UTF-8 string.
+   * See https://simonsapin.github.io/wtf-8/
+   *
+   * Important: a value should be consumed once. Calling get_wobbly_string() twice on the same value
+   * is an error.
+   *
+   * @returns An UTF-8 string. The string is stored in the parser and will be invalidated the next
+   *          time it parses a document or when it is destroyed.
+   * @returns INCORRECT_TYPE if the JSON value is not a string.
+   */
+  simdjson_inline simdjson_result<std::string_view> get_wobbly_string() noexcept;
   /**
    * Cast this JSON value to a raw_json_string.
    *
@@ -585,7 +600,8 @@ public:
   simdjson_inline simdjson_result<int64_t> get_int64_in_string() noexcept;
   simdjson_inline simdjson_result<double> get_double() noexcept;
   simdjson_inline simdjson_result<double> get_double_in_string() noexcept;
-  simdjson_inline simdjson_result<std::string_view> get_string() noexcept;
+  simdjson_inline simdjson_result<std::string_view> get_string(bool allow_replacement = false) noexcept;
+  simdjson_inline simdjson_result<std::string_view> get_wobbly_string() noexcept;
   simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::raw_json_string> get_raw_json_string() noexcept;
   simdjson_inline simdjson_result<bool> get_bool() noexcept;
   simdjson_inline simdjson_result<bool> is_null() noexcept;

include/simdjson/generic/ondemand/value_iterator-inl.h

@@ -475,8 +475,11 @@ simdjson_warn_unused simdjson_inline simdjson_result<bool> value_iterator::parse
   return is_null_string;
 }
 
-simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> value_iterator::get_string() noexcept {
-  return get_raw_json_string().unescape(json_iter());
+simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> value_iterator::get_string(bool allow_replacement) noexcept {
+  return get_raw_json_string().unescape(json_iter(), allow_replacement);
+}
+simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> value_iterator::get_wobbly_string() noexcept {
+  return get_raw_json_string().unescape_wobbly(json_iter());
 }
 simdjson_warn_unused simdjson_inline simdjson_result<raw_json_string> value_iterator::get_raw_json_string() noexcept {
   auto json = peek_scalar("string");
@@ -592,8 +595,11 @@ simdjson_inline simdjson_result<number> value_iterator::get_root_number(bool che
   advance_root_scalar("number");
   return num;
 }
-simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> value_iterator::get_root_string(bool check_trailing) noexcept {
-  return get_root_raw_json_string(check_trailing).unescape(json_iter());
+simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> value_iterator::get_root_string(bool check_trailing, bool allow_replacement) noexcept {
+  return get_root_raw_json_string(check_trailing).unescape(json_iter(), allow_replacement);
+}
+simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> value_iterator::get_root_wobbly_string(bool check_trailing) noexcept {
+  return get_root_raw_json_string(check_trailing).unescape_wobbly(json_iter());
 }
 simdjson_warn_unused simdjson_inline simdjson_result<raw_json_string> value_iterator::get_root_raw_json_string(bool check_trailing) noexcept {
   auto json = peek_scalar("string");

include/simdjson/generic/ondemand/value_iterator.h

@@ -280,7 +280,8 @@ public:
    * @{
    */
 
-  simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> get_string() noexcept;
+  simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> get_string(bool allow_replacement) noexcept;
+  simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> get_wobbly_string() noexcept;
   simdjson_warn_unused simdjson_inline simdjson_result<raw_json_string> get_raw_json_string() noexcept;
   simdjson_warn_unused simdjson_inline simdjson_result<uint64_t> get_uint64() noexcept;
   simdjson_warn_unused simdjson_inline simdjson_result<uint64_t> get_uint64_in_string() noexcept;
@@ -295,7 +296,8 @@ public:
   simdjson_warn_unused simdjson_inline simdjson_result<number_type> get_number_type() noexcept;
   simdjson_warn_unused simdjson_inline simdjson_result<number> get_number() noexcept;
 
-  simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> get_root_string(bool check_trailing) noexcept;
+  simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> get_root_string(bool check_trailing, bool allow_replacement) noexcept;
+    simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> get_root_wobbly_string(bool check_trailing) noexcept;
   simdjson_warn_unused simdjson_inline simdjson_result<raw_json_string> get_root_raw_json_string(bool check_trailing) noexcept;
   simdjson_warn_unused simdjson_inline simdjson_result<uint64_t> get_root_uint64(bool check_trailing) noexcept;
   simdjson_warn_unused simdjson_inline simdjson_result<uint64_t> get_root_uint64_in_string(bool check_trailing) noexcept;

include/simdjson/internal/dom_parser_implementation.h

@@ -114,9 +114,26 @@ public:
    *
    * @param str pointer to the beginning of a valid UTF-8 JSON string, must end with an unescaped quote.
    * @param dst pointer to a destination buffer, it must point a region in memory of sufficient size.
+   * @param allow_replacement whether we allow a replacement character when the UTF-8 contains unmatched surrogate pairs.
    * @return end of the of the written region (exclusive) or nullptr in case of error.
    */
-  simdjson_warn_unused virtual uint8_t *parse_string(const uint8_t *src, uint8_t *dst) const noexcept = 0;
+  simdjson_warn_unused virtual uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept = 0;
+
+  /**
+   * Unescape a NON-valid UTF-8 string from src to dst, stopping at a final unescaped quote. There
+   * must be an unescaped quote terminating the string. It returns the final output
+   * position as pointer. In case of error (e.g., the string has bad escaped codes),
+   * then null_nullptrptr is returned. It is assumed that the output buffer is large
+   * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes +
+   * SIMDJSON_PADDING bytes.
+   *
+   * Overridden by each implementation.
+   *
+   * @param str pointer to the beginning of a possibly invalid UTF-8 JSON string, must end with an unescaped quote.
+   * @param dst pointer to a destination buffer, it must point a region in memory of sufficient size.
+   * @return end of the of the written region (exclusive) or nullptr in case of error.
+   */
+  simdjson_warn_unused virtual uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept = 0;
 
   /**
    * Change the capacity of this parser.

include/simdjson/simdjson_version.h

@@ -4,7 +4,7 @@
 #define SIMDJSON_SIMDJSON_VERSION_H
 
 /** The version of simdjson being used (major.minor.revision) */
-#define SIMDJSON_VERSION "3.1.2"
+#define SIMDJSON_VERSION "3.1.4"
 
 namespace simdjson {
 enum {
@@ -19,7 +19,7 @@ enum {
   /**
    * The revision (major.minor.REVISION) of simdjson being used.
    */
-  SIMDJSON_VERSION_REVISION = 2
+  SIMDJSON_VERSION_REVISION = 4
 };
 } // namespace simdjson
 

singleheader/simdjson.h

@@ -1,4 +1,4 @@
-/* auto-generated on 2023-02-22 20:34:51 -0500. Do not edit! */
+/* auto-generated on 2023-03-09 10:42:23 -0500. Do not edit! */
 /* begin file include/simdjson.h */
 #ifndef SIMDJSON_H
 #define SIMDJSON_H
@@ -43,7 +43,7 @@
 #define SIMDJSON_SIMDJSON_VERSION_H
 
 /** The version of simdjson being used (major.minor.revision) */
-#define SIMDJSON_VERSION "3.1.2"
+#define SIMDJSON_VERSION "3.1.4"
 
 namespace simdjson {
 enum {
@@ -58,7 +58,7 @@ enum {
   /**
    * The revision (major.minor.REVISION) of simdjson being used.
    */
-  SIMDJSON_VERSION_REVISION = 2
+  SIMDJSON_VERSION_REVISION = 4
 };
 } // namespace simdjson
 
@@ -2263,6 +2263,10 @@ namespace std {
 #define SIMDJSON_DEVELOPMENT_ASSERT(expr) do { } while (0)
 #endif
 
+#ifndef SIMDJSON_UTF8VALIDATION
+#define SIMDJSON_UTF8VALIDATION 1
+#endif
+
 #endif // SIMDJSON_COMMON_DEFS_H
 /* end file include/simdjson/common_defs.h */
 
@@ -2973,9 +2977,26 @@ public:
    *
    * @param str pointer to the beginning of a valid UTF-8 JSON string, must end with an unescaped quote.
    * @param dst pointer to a destination buffer, it must point a region in memory of sufficient size.
+   * @param allow_replacement whether we allow a replacement character when the UTF-8 contains unmatched surrogate pairs.
    * @return end of the of the written region (exclusive) or nullptr in case of error.
    */
-  simdjson_warn_unused virtual uint8_t *parse_string(const uint8_t *src, uint8_t *dst) const noexcept = 0;
+  simdjson_warn_unused virtual uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept = 0;
+
+  /**
+   * Unescape a NON-valid UTF-8 string from src to dst, stopping at a final unescaped quote. There
+   * must be an unescaped quote terminating the string. It returns the final output
+   * position as pointer. In case of error (e.g., the string has bad escaped codes),
+   * then null_nullptrptr is returned. It is assumed that the output buffer is large
+   * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes +
+   * SIMDJSON_PADDING bytes.
+   *
+   * Overridden by each implementation.
+   *
+   * @param str pointer to the beginning of a possibly invalid UTF-8 JSON string, must end with an unescaped quote.
+   * @param dst pointer to a destination buffer, it must point a region in memory of sufficient size.
+   * @return end of the of the written region (exclusive) or nullptr in case of error.
+   */
+  simdjson_warn_unused virtual uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept = 0;
 
   /**
    * Change the capacity of this parser.
@@ -4480,7 +4501,9 @@ public:
    * ### REQUIRED: Buffer Padding
    *
    * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what
-   * those bytes are initialized to, as long as they are allocated.
+   * those bytes are initialized to, as long as they are allocated. These bytes will be read: if you
+   * using a sanitizer that verifies that no uninitialized byte is read, then you should initialize the
+   * SIMDJSON_PADDING bytes to avoid runtime warnings.
    *
    * If realloc_if_needed is true (the default), it is assumed that the buffer does *not* have enough padding,
    * and it is copied into an enlarged temporary buffer before parsing. Thus the following is safe:
@@ -4743,7 +4766,9 @@ public:
    * ### REQUIRED: Buffer Padding
    *
    * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what
-   * those bytes are initialized to, as long as they are allocated.
+   * those bytes are initialized to, as long as they are allocated. These bytes will be read: if you
+   * using a sanitizer that verifies that no uninitialized byte is read, then you should initialize the
+   * SIMDJSON_PADDING bytes to avoid runtime warnings.
    *
    * ### Threads
    *
@@ -9750,7 +9775,8 @@ public:
   simdjson_warn_unused error_code stage1(const uint8_t *buf, size_t len, stage1_mode partial) noexcept final;
   simdjson_warn_unused error_code stage2(dom::document &doc) noexcept final;
   simdjson_warn_unused error_code stage2_next(dom::document &doc) noexcept final;
-  simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst) const noexcept final;
+  simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept final;
+  simdjson_warn_unused uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept final;
   inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final;
   inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final;
 private:
@@ -12097,7 +12123,8 @@ public:
   simdjson_warn_unused error_code stage1(const uint8_t *buf, size_t len, stage1_mode partial) noexcept final;
   simdjson_warn_unused error_code stage2(dom::document &doc) noexcept final;
   simdjson_warn_unused error_code stage2_next(dom::document &doc) noexcept final;
-  simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst) const noexcept final;
+  simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept final;
+  simdjson_warn_unused uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept final;
   inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final;
   inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final;
 private:
@@ -13871,7 +13898,8 @@ public:
   simdjson_warn_unused error_code stage1(const uint8_t *buf, size_t len, stage1_mode partial) noexcept final;
   simdjson_warn_unused error_code stage2(dom::document &doc) noexcept final;
   simdjson_warn_unused error_code stage2_next(dom::document &doc) noexcept final;
-  simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst) const noexcept final;
+  simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept final;
+  simdjson_warn_unused uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept final;
   inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final;
   inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final;
 private:
@@ -16060,7 +16088,8 @@ public:
   simdjson_warn_unused error_code stage1(const uint8_t *buf, size_t len, stage1_mode partial) noexcept final;
   simdjson_warn_unused error_code stage2(dom::document &doc) noexcept final;
   simdjson_warn_unused error_code stage2_next(dom::document &doc) noexcept final;
-  simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst) const noexcept final;
+  simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept final;
+  simdjson_warn_unused uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept final;
   inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final;
   inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final;
 private:
@@ -18183,7 +18212,8 @@ public:
   simdjson_warn_unused error_code stage1(const uint8_t *buf, size_t len, stage1_mode partial) noexcept final;
   simdjson_warn_unused error_code stage2(dom::document &doc) noexcept final;
   simdjson_warn_unused error_code stage2_next(dom::document &doc) noexcept final;
-  simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst) const noexcept final;
+  simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept final;
+  simdjson_warn_unused uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept final;
   inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final;
   inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final;
 private:
@@ -20521,7 +20551,8 @@ public:
   simdjson_warn_unused error_code stage1(const uint8_t *buf, size_t len, stage1_mode partial) noexcept final;
   simdjson_warn_unused error_code stage2(dom::document &doc) noexcept final;
   simdjson_warn_unused error_code stage2_next(dom::document &doc) noexcept final;
-  simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst) const noexcept final;
+  simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept final;
+  simdjson_warn_unused uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept final;
   inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final;
   inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final;
 private:
@@ -23057,6 +23088,20 @@ private:
 
   /**
    * Unescape this JSON string, replacing \\ with \, \n with newline, etc.
+   * The result will be a valid UTF-8.
+   *
+   * ## IMPORTANT: string_view lifetime
+   *
+   * The string_view is only valid until the next parse() call on the parser.
+   *
+   * @param iter A json_iterator, which contains a buffer where the string will be written.
+   * @param allow_replacement Whether we allow replacement of invalid surrogate pairs.
+   */
+  simdjson_inline simdjson_warn_unused simdjson_result<std::string_view> unescape(json_iterator &iter, bool allow_replacement) const noexcept;
+
+  /**
+   * Unescape this JSON string, replacing \\ with \, \n with newline, etc.
+   * The result may not be a valid UTF-8. https://simonsapin.github.io/wtf-8/
    *
    * ## IMPORTANT: string_view lifetime
    *
@@ -23064,8 +23109,7 @@ private:
    *
    * @param iter A json_iterator, which contains a buffer where the string will be written.
    */
-  simdjson_inline simdjson_warn_unused simdjson_result<std::string_view> unescape(json_iterator &iter) const noexcept;
-
+  simdjson_inline simdjson_warn_unused simdjson_result<std::string_view> unescape_wobbly(json_iterator &iter) const noexcept;
   const uint8_t * buf{};
   friend class object;
   friend class field;
@@ -23100,7 +23144,8 @@ public:
   simdjson_inline ~simdjson_result() noexcept = default; ///< @private
 
   simdjson_inline simdjson_result<const char *> raw() const noexcept;
-  simdjson_inline simdjson_warn_unused simdjson_result<std::string_view> unescape(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::json_iterator &iter) const noexcept;
+  simdjson_inline simdjson_warn_unused simdjson_result<std::string_view> unescape(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::json_iterator &iter, bool allow_replacement) const noexcept;
+  simdjson_inline simdjson_warn_unused simdjson_result<std::string_view> unescape_wobbly(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::json_iterator &iter) const noexcept;
 };
 
 } // namespace simdjson
@@ -23494,7 +23539,8 @@ public:
    * Each raw_json_string should be unescaped once, or else the string buffer might
    * overflow.
    */
-  simdjson_inline simdjson_result<std::string_view> unescape(raw_json_string in) noexcept;
+  simdjson_inline simdjson_result<std::string_view> unescape(raw_json_string in, bool allow_replacement) noexcept;
+  simdjson_inline simdjson_result<std::string_view> unescape_wobbly(raw_json_string in) noexcept;
   simdjson_inline void reenter_child(token_position position, depth_t child_depth) noexcept;
 
 #if SIMDJSON_DEVELOPMENT_CHECKS
@@ -23843,7 +23889,8 @@ public:
    * @{
    */
 
-  simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> get_string() noexcept;
+  simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> get_string(bool allow_replacement) noexcept;
+  simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> get_wobbly_string() noexcept;
   simdjson_warn_unused simdjson_inline simdjson_result<raw_json_string> get_raw_json_string() noexcept;
   simdjson_warn_unused simdjson_inline simdjson_result<uint64_t> get_uint64() noexcept;
   simdjson_warn_unused simdjson_inline simdjson_result<uint64_t> get_uint64_in_string() noexcept;
@@ -23858,7 +23905,8 @@ public:
   simdjson_warn_unused simdjson_inline simdjson_result<number_type> get_number_type() noexcept;
   simdjson_warn_unused simdjson_inline simdjson_result<number> get_number() noexcept;
 
-  simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> get_root_string(bool check_trailing) noexcept;
+  simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> get_root_string(bool check_trailing, bool allow_replacement) noexcept;
+    simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> get_root_wobbly_string(bool check_trailing) noexcept;
   simdjson_warn_unused simdjson_inline simdjson_result<raw_json_string> get_root_raw_json_string(bool check_trailing) noexcept;
   simdjson_warn_unused simdjson_inline simdjson_result<uint64_t> get_root_uint64(bool check_trailing) noexcept;
   simdjson_warn_unused simdjson_inline simdjson_result<uint64_t> get_root_uint64_in_string(bool check_trailing) noexcept;
@@ -24484,11 +24532,24 @@ public:
    *
    * Important: Calling get_string() twice on the same document is an error.
    *
+   * @param Whether to allow a replacement character for unmatched surrogate pairs.
    * @returns An UTF-8 string. The string is stored in the parser and will be invalidated the next
    *          time it parses a document or when it is destroyed.
    * @returns INCORRECT_TYPE if the JSON value is not a string.
    */
-  simdjson_inline simdjson_result<std::string_view> get_string() noexcept;
+  simdjson_inline simdjson_result<std::string_view> get_string(bool allow_replacement = false) noexcept;
+  /**
+   * Cast this JSON value to a string.
+   *
+   * The string is not guaranteed to be valid UTF-8. See https://simonsapin.github.io/wtf-8/
+   *
+   * Important: Calling get_wobbly_string() twice on the same document is an error.
+   *
+   * @returns An UTF-8 string. The string is stored in the parser and will be invalidated the next
+   *          time it parses a document or when it is destroyed.
+   * @returns INCORRECT_TYPE if the JSON value is not a string.
+   */
+  simdjson_inline simdjson_result<std::string_view> get_wobbly_string() noexcept;
   /**
    * Cast this JSON value to a raw_json_string.
    *
@@ -24987,7 +25048,8 @@ public:
   simdjson_inline simdjson_result<int64_t> get_int64_in_string() noexcept;
   simdjson_inline simdjson_result<double> get_double() noexcept;
   simdjson_inline simdjson_result<double> get_double_in_string() noexcept;
-  simdjson_inline simdjson_result<std::string_view> get_string() noexcept;
+  simdjson_inline simdjson_result<std::string_view> get_string(bool allow_replacement = false) noexcept;
+  simdjson_inline simdjson_result<std::string_view> get_wobbly_string() noexcept;
   simdjson_inline simdjson_result<raw_json_string> get_raw_json_string() noexcept;
   simdjson_inline simdjson_result<bool> get_bool() noexcept;
   simdjson_inline simdjson_result<value> get_value() noexcept;
@@ -25055,7 +25117,8 @@ public:
   simdjson_inline simdjson_result<int64_t> get_int64_in_string() noexcept;
   simdjson_inline simdjson_result<double> get_double() noexcept;
   simdjson_inline simdjson_result<double> get_double_in_string() noexcept;
-  simdjson_inline simdjson_result<std::string_view> get_string() noexcept;
+  simdjson_inline simdjson_result<std::string_view> get_string(bool allow_replacement = false) noexcept;
+  simdjson_inline simdjson_result<std::string_view> get_wobbly_string() noexcept;
   simdjson_inline simdjson_result<SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::raw_json_string> get_raw_json_string() noexcept;
   simdjson_inline simdjson_result<bool> get_bool() noexcept;
   simdjson_inline simdjson_result<SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::value> get_value() noexcept;
@@ -25125,7 +25188,8 @@ public:
   simdjson_inline simdjson_result<int64_t> get_int64_in_string() noexcept;
   simdjson_inline simdjson_result<double> get_double() noexcept;
   simdjson_inline simdjson_result<double> get_double_in_string() noexcept;
-  simdjson_inline simdjson_result<std::string_view> get_string() noexcept;
+  simdjson_inline simdjson_result<std::string_view> get_string(bool allow_replacement = false) noexcept;
+  simdjson_inline simdjson_result<std::string_view> get_wobbly_string() noexcept;
   simdjson_inline simdjson_result<SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::raw_json_string> get_raw_json_string() noexcept;
   simdjson_inline simdjson_result<bool> get_bool() noexcept;
   simdjson_inline simdjson_result<SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::value> get_value() noexcept;
@@ -25301,8 +25365,23 @@ public:
    *          time it parses a document or when it is destroyed.
    * @returns INCORRECT_TYPE if the JSON value is not a string.
    */
-  simdjson_inline simdjson_result<std::string_view> get_string() noexcept;
+  simdjson_inline simdjson_result<std::string_view> get_string(bool allow_replacement = false) noexcept;
 
+
+  /**
+   * Cast this JSON value to a "wobbly" string.
+   *
+   * The string is may not be a valid UTF-8 string.
+   * See https://simonsapin.github.io/wtf-8/
+   *
+   * Important: a value should be consumed once. Calling get_wobbly_string() twice on the same value
+   * is an error.
+   *
+   * @returns An UTF-8 string. The string is stored in the parser and will be invalidated the next
+   *          time it parses a document or when it is destroyed.
+   * @returns INCORRECT_TYPE if the JSON value is not a string.
+   */
+  simdjson_inline simdjson_result<std::string_view> get_wobbly_string() noexcept;
   /**
    * Cast this JSON value to a raw_json_string.
    *
@@ -25757,7 +25836,8 @@ public:
   simdjson_inline simdjson_result<int64_t> get_int64_in_string() noexcept;
   simdjson_inline simdjson_result<double> get_double() noexcept;
   simdjson_inline simdjson_result<double> get_double_in_string() noexcept;
-  simdjson_inline simdjson_result<std::string_view> get_string() noexcept;
+  simdjson_inline simdjson_result<std::string_view> get_string(bool allow_replacement = false) noexcept;
+  simdjson_inline simdjson_result<std::string_view> get_wobbly_string() noexcept;
   simdjson_inline simdjson_result<SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::raw_json_string> get_raw_json_string() noexcept;
   simdjson_inline simdjson_result<bool> get_bool() noexcept;
   simdjson_inline simdjson_result<bool> is_null() noexcept;
@@ -25889,7 +25969,7 @@ public:
    * This consumes the key: once you have called unescaped_key(), you cannot
    * call it again nor can you call key().
    */
-  simdjson_inline simdjson_warn_unused simdjson_result<std::string_view> unescaped_key() noexcept;
+  simdjson_inline simdjson_warn_unused simdjson_result<std::string_view> unescaped_key(bool allow_replacement) noexcept;
   /**
    * Get the key as a raw_json_string. Can be used for direct comparison with
    * an unescaped C string: e.g., key() == "test".
@@ -25925,7 +26005,7 @@ public:
   simdjson_inline simdjson_result(error_code error) noexcept; ///< @private
   simdjson_inline simdjson_result() noexcept = default;
 
-  simdjson_inline simdjson_result<std::string_view> unescaped_key() noexcept;
+  simdjson_inline simdjson_result<std::string_view> unescaped_key(bool allow_replacement = false) noexcept;
   simdjson_inline simdjson_result<SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::raw_json_string> key() noexcept;
   simdjson_inline simdjson_result<SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::value> value() noexcept;
 };
@@ -26242,7 +26322,9 @@ public:
    * ### REQUIRED: Buffer Padding
    *
    * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what
-   * those bytes are initialized to, as long as they are allocated.
+   * those bytes are initialized to, as long as they are allocated. These bytes will be read: if you
+   * using a sanitizer that verifies that no uninitialized byte is read, then you should initialize the
+   * SIMDJSON_PADDING bytes to avoid runtime warnings.
    *
    * @param json The JSON to parse.
    * @param len The length of the JSON.
@@ -26298,7 +26380,9 @@ public:
    * ### REQUIRED: Buffer Padding
    *
    * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what
-   * those bytes are initialized to, as long as they are allocated.
+   * those bytes are initialized to, as long as they are allocated. These bytes will be read: if you
+   * using a sanitizer that verifies that no uninitialized byte is read, then you should initialize the
+   * SIMDJSON_PADDING bytes to avoid runtime warnings.
    *
    * @param json The JSON to parse.
    *
@@ -26352,7 +26436,9 @@ public:
    * ### REQUIRED: Buffer Padding
    *
    * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what
-   * those bytes are initialized to, as long as they are allocated.
+   * those bytes are initialized to, as long as they are allocated. These bytes will be read: if you
+   * using a sanitizer that verifies that no uninitialized byte is read, then you should initialize the
+   * SIMDJSON_PADDING bytes to avoid runtime warnings.
    *
    * ### Threads
    *
@@ -26427,6 +26513,31 @@ public:
 
   /**
    * Unescape this JSON string, replacing \\ with \, \n with newline, etc. to a user-provided buffer.
+   * The result must be valid UTF-8.
+   * The provided pointer is advanced to the end of the string by reference, and a string_view instance
+   * is returned. You can ensure that your buffer is large enough by allocating a block of memory at least
+   * as large as the input JSON plus SIMDJSON_PADDING and then unescape all strings to this one buffer.
+   *
+   * This unescape function is a low-level function. If you want a more user-friendly approach, you should
+   * avoid raw_json_string instances (e.g., by calling unescaped_key() instead of key() or get_string()
+   * instead of get_raw_json_string()).
+   *
+   * ## IMPORTANT: string_view lifetime
+   *
+   * The string_view is only valid as long as the bytes in dst.
+   *
+   * @param raw_json_string input
+   * @param dst A pointer to a buffer at least large enough to write this string as well as
+   *            an additional SIMDJSON_PADDING bytes.
+   * @param allow_replacement Whether we allow a replacement if the input string contains unmatched surrogate pairs.
+   * @return A string_view pointing at the unescaped string in dst
+   * @error STRING_ERROR if escapes are incorrect.
+   */
+  simdjson_inline simdjson_result<std::string_view> unescape(raw_json_string in, uint8_t *&dst, bool allow_replacement = false) const noexcept;
+
+  /**
+   * Unescape this JSON string, replacing \\ with \, \n with newline, etc. to a user-provided buffer.
+   * The result may not be valid UTF-8. See https://simonsapin.github.io/wtf-8/
    * The provided pointer is advanced to the end of the string by reference, and a string_view instance
    * is returned. You can ensure that your buffer is large enough by allocating a block of memory at least
    * as large as the input JSON plus SIMDJSON_PADDING and then unescape all strings to this one buffer.
@@ -26445,7 +26556,8 @@ public:
    * @return A string_view pointing at the unescaped string in dst
    * @error STRING_ERROR if escapes are incorrect.
    */
-  simdjson_inline simdjson_result<std::string_view> unescape(raw_json_string in, uint8_t *&dst) const noexcept;
+  simdjson_inline simdjson_result<std::string_view> unescape_wobbly(raw_json_string in, uint8_t *&dst) const noexcept;
+
 private:
   /** @private [for benchmarking access] The implementation to use */
   std::unique_ptr<internal::dom_parser_implementation> implementation{};
@@ -27428,10 +27540,13 @@ simdjson_unused simdjson_inline bool operator!=(std::string_view c, const raw_js
 }
 
 
-simdjson_inline simdjson_warn_unused simdjson_result<std::string_view> raw_json_string::unescape(json_iterator &iter) const noexcept {
-  return iter.unescape(*this);
+simdjson_inline simdjson_warn_unused simdjson_result<std::string_view> raw_json_string::unescape(json_iterator &iter, bool allow_replacement) const noexcept {
+  return iter.unescape(*this, allow_replacement);
 }
 
+simdjson_inline simdjson_warn_unused simdjson_result<std::string_view> raw_json_string::unescape_wobbly(json_iterator &iter) const noexcept {
+  return iter.unescape_wobbly(*this);
+}
 
 simdjson_unused simdjson_inline std::ostream &operator<<(std::ostream &out, const raw_json_string &str) noexcept {
   bool in_escape = false;
@@ -27462,11 +27577,14 @@ simdjson_inline simdjson_result<const char *> simdjson_result<SIMDJSON_BUILTIN_I
   if (error()) { return error(); }
   return first.raw();
 }
-simdjson_inline simdjson_warn_unused simdjson_result<std::string_view> simdjson_result<SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::raw_json_string>::unescape(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::json_iterator &iter) const noexcept {
+simdjson_inline simdjson_warn_unused simdjson_result<std::string_view> simdjson_result<SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::raw_json_string>::unescape(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::json_iterator &iter, bool allow_replacement) const noexcept {
   if (error()) { return error(); }
-  return first.unescape(iter);
+  return first.unescape(iter, allow_replacement);
+}
+simdjson_inline simdjson_warn_unused simdjson_result<std::string_view> simdjson_result<SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::raw_json_string>::unescape_wobbly(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::json_iterator &iter) const noexcept {
+  if (error()) { return error(); }
+  return first.unescape_wobbly(iter);
 }
-
 } // namespace simdjson
 /* end file include/simdjson/generic/ondemand/raw_json_string-inl.h */
 /* begin file include/simdjson/generic/ondemand/token_iterator-inl.h */
@@ -27868,8 +27986,12 @@ simdjson_inline token_position json_iterator::position() const noexcept {
   return token.position();
 }
 
-simdjson_inline simdjson_result<std::string_view> json_iterator::unescape(raw_json_string in) noexcept {
-  return parser->unescape(in, _string_buf_loc);
+simdjson_inline simdjson_result<std::string_view> json_iterator::unescape(raw_json_string in, bool allow_replacement) noexcept {
+  return parser->unescape(in, _string_buf_loc, allow_replacement);
+}
+
+simdjson_inline simdjson_result<std::string_view> json_iterator::unescape_wobbly(raw_json_string in) noexcept {
+  return parser->unescape_wobbly(in, _string_buf_loc);
 }
 
 simdjson_inline void json_iterator::reenter_child(token_position position, depth_t child_depth) noexcept {
@@ -28413,8 +28535,11 @@ simdjson_warn_unused simdjson_inline simdjson_result<bool> value_iterator::parse
   return is_null_string;
 }
 
-simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> value_iterator::get_string() noexcept {
-  return get_raw_json_string().unescape(json_iter());
+simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> value_iterator::get_string(bool allow_replacement) noexcept {
+  return get_raw_json_string().unescape(json_iter(), allow_replacement);
+}
+simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> value_iterator::get_wobbly_string() noexcept {
+  return get_raw_json_string().unescape_wobbly(json_iter());
 }
 simdjson_warn_unused simdjson_inline simdjson_result<raw_json_string> value_iterator::get_raw_json_string() noexcept {
   auto json = peek_scalar("string");
@@ -28530,8 +28655,11 @@ simdjson_inline simdjson_result<number> value_iterator::get_root_number(bool che
   advance_root_scalar("number");
   return num;
 }
-simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> value_iterator::get_root_string(bool check_trailing) noexcept {
-  return get_root_raw_json_string(check_trailing).unescape(json_iter());
+simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> value_iterator::get_root_string(bool check_trailing, bool allow_replacement) noexcept {
+  return get_root_raw_json_string(check_trailing).unescape(json_iter(), allow_replacement);
+}
+simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> value_iterator::get_root_wobbly_string(bool check_trailing) noexcept {
+  return get_root_raw_json_string(check_trailing).unescape_wobbly(json_iter());
 }
 simdjson_warn_unused simdjson_inline simdjson_result<raw_json_string> value_iterator::get_root_raw_json_string(bool check_trailing) noexcept {
   auto json = peek_scalar("string");
@@ -29446,8 +29574,11 @@ simdjson_inline simdjson_result<double> document::get_double() noexcept {
 simdjson_inline simdjson_result<double> document::get_double_in_string() noexcept {
   return get_root_value_iterator().get_root_double_in_string(true);
 }
-simdjson_inline simdjson_result<std::string_view> document::get_string() noexcept {
-  return get_root_value_iterator().get_root_string(true);
+simdjson_inline simdjson_result<std::string_view> document::get_string(bool allow_replacement) noexcept {
+  return get_root_value_iterator().get_root_string(true, allow_replacement);
+}
+simdjson_inline simdjson_result<std::string_view> document::get_wobbly_string() noexcept {
+  return get_root_value_iterator().get_root_wobbly_string(true);
 }
 simdjson_inline simdjson_result<raw_json_string> document::get_raw_json_string() noexcept {
   return get_root_value_iterator().get_root_raw_json_string(true);
@@ -29462,7 +29593,7 @@ simdjson_inline simdjson_result<bool> document::is_null() noexcept {
 template<> simdjson_inline simdjson_result<array> document::get() & noexcept { return get_array(); }
 template<> simdjson_inline simdjson_result<object> document::get() & noexcept { return get_object(); }
 template<> simdjson_inline simdjson_result<raw_json_string> document::get() & noexcept { return get_raw_json_string(); }
-template<> simdjson_inline simdjson_result<std::string_view> document::get() & noexcept { return get_string(); }
+template<> simdjson_inline simdjson_result<std::string_view> document::get() & noexcept { return get_string(false); }
 template<> simdjson_inline simdjson_result<double> document::get() & noexcept { return get_double(); }
 template<> simdjson_inline simdjson_result<uint64_t> document::get() & noexcept { return get_uint64(); }
 template<> simdjson_inline simdjson_result<int64_t> document::get() & noexcept { return get_int64(); }
@@ -29470,7 +29601,7 @@ template<> simdjson_inline simdjson_result<bool> document::get() & noexcept { re
 template<> simdjson_inline simdjson_result<value> document::get() & noexcept { return get_value(); }
 
 template<> simdjson_inline simdjson_result<raw_json_string> document::get() && noexcept { return get_raw_json_string(); }
-template<> simdjson_inline simdjson_result<std::string_view> document::get() && noexcept { return get_string(); }
+template<> simdjson_inline simdjson_result<std::string_view> document::get() && noexcept { return get_string(false); }
 template<> simdjson_inline simdjson_result<double> document::get() && noexcept { return std::forward<document>(*this).get_double(); }
 template<> simdjson_inline simdjson_result<uint64_t> document::get() && noexcept { return std::forward<document>(*this).get_uint64(); }
 template<> simdjson_inline simdjson_result<int64_t> document::get() && noexcept { return std::forward<document>(*this).get_int64(); }
@@ -29490,7 +29621,7 @@ simdjson_inline document::operator object() & noexcept(false) { return get_objec
 simdjson_inline document::operator uint64_t() noexcept(false) { return get_uint64(); }
 simdjson_inline document::operator int64_t() noexcept(false) { return get_int64(); }
 simdjson_inline document::operator double() noexcept(false) { return get_double(); }
-simdjson_inline document::operator std::string_view() noexcept(false) { return get_string(); }
+simdjson_inline document::operator std::string_view() noexcept(false) { return get_string(false); }
 simdjson_inline document::operator raw_json_string() noexcept(false) { return get_raw_json_string(); }
 simdjson_inline document::operator bool() noexcept(false) { return get_bool(); }
 simdjson_inline document::operator value() noexcept(false) { return get_value(); }
@@ -29711,9 +29842,13 @@ simdjson_inline simdjson_result<double> simdjson_result<SIMDJSON_BUILTIN_IMPLEME
   if (error()) { return error(); }
   return first.get_double_in_string();
 }
-simdjson_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::document>::get_string() noexcept {
+simdjson_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::document>::get_string(bool allow_replacement) noexcept {
   if (error()) { return error(); }
-  return first.get_string();
+  return first.get_string(allow_replacement);
+}
+simdjson_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::document>::get_wobbly_string() noexcept {
+  if (error()) { return error(); }
+  return first.get_wobbly_string();
 }
 simdjson_inline simdjson_result<SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::raw_json_string> simdjson_result<SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::document>::get_raw_json_string() noexcept {
   if (error()) { return error(); }
@@ -29890,7 +30025,8 @@ simdjson_inline simdjson_result<int64_t> document_reference::get_int64() noexcep
 simdjson_inline simdjson_result<int64_t> document_reference::get_int64_in_string() noexcept { return doc->get_root_value_iterator().get_root_int64_in_string(false); }
 simdjson_inline simdjson_result<double> document_reference::get_double() noexcept { return doc->get_root_value_iterator().get_root_double(false); }
 simdjson_inline simdjson_result<double> document_reference::get_double_in_string() noexcept { return doc->get_root_value_iterator().get_root_double(false); }
-simdjson_inline simdjson_result<std::string_view> document_reference::get_string() noexcept { return doc->get_root_value_iterator().get_root_string(false); }
+simdjson_inline simdjson_result<std::string_view> document_reference::get_string(bool allow_replacement) noexcept { return doc->get_root_value_iterator().get_root_string(false, allow_replacement); }
+simdjson_inline simdjson_result<std::string_view> document_reference::get_wobbly_string() noexcept { return doc->get_root_value_iterator().get_root_wobbly_string(false); }
 simdjson_inline simdjson_result<raw_json_string> document_reference::get_raw_json_string() noexcept { return doc->get_root_value_iterator().get_root_raw_json_string(false); }
 simdjson_inline simdjson_result<bool> document_reference::get_bool() noexcept { return doc->get_root_value_iterator().get_root_bool(false); }
 simdjson_inline simdjson_result<value> document_reference::get_value() noexcept { return doc->get_value(); }
@@ -30022,9 +30158,13 @@ simdjson_inline simdjson_result<double> simdjson_result<SIMDJSON_BUILTIN_IMPLEME
   if (error()) { return error(); }
   return first.get_double_in_string();
 }
-simdjson_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::document_reference>::get_string() noexcept {
+simdjson_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::document_reference>::get_string(bool allow_replacement) noexcept {
   if (error()) { return error(); }
-  return first.get_string();
+  return first.get_string(allow_replacement);
+}
+simdjson_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::document_reference>::get_wobbly_string() noexcept {
+  if (error()) { return error(); }
+  return first.get_wobbly_string();
 }
 simdjson_inline simdjson_result<SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::raw_json_string> simdjson_result<SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::document_reference>::get_raw_json_string() noexcept {
   if (error()) { return error(); }
@@ -30156,8 +30296,11 @@ simdjson_inline simdjson_result<object> value::start_or_resume_object() noexcept
 simdjson_inline simdjson_result<raw_json_string> value::get_raw_json_string() noexcept {
   return iter.get_raw_json_string();
 }
-simdjson_inline simdjson_result<std::string_view> value::get_string() noexcept {
-  return iter.get_string();
+simdjson_inline simdjson_result<std::string_view> value::get_string(bool allow_replacement) noexcept {
+  return iter.get_string(allow_replacement);
+}
+simdjson_inline simdjson_result<std::string_view> value::get_wobbly_string() noexcept {
+  return iter.get_wobbly_string();
 }
 simdjson_inline simdjson_result<double> value::get_double() noexcept {
   return iter.get_double();
@@ -30186,7 +30329,7 @@ simdjson_inline simdjson_result<bool> value::is_null() noexcept {
 template<> simdjson_inline simdjson_result<array> value::get() noexcept { return get_array(); }
 template<> simdjson_inline simdjson_result<object> value::get() noexcept { return get_object(); }
 template<> simdjson_inline simdjson_result<raw_json_string> value::get() noexcept { return get_raw_json_string(); }
-template<> simdjson_inline simdjson_result<std::string_view> value::get() noexcept { return get_string(); }
+template<> simdjson_inline simdjson_result<std::string_view> value::get() noexcept { return get_string(false); }
 template<> simdjson_inline simdjson_result<number> value::get() noexcept { return get_number(); }
 template<> simdjson_inline simdjson_result<double> value::get() noexcept { return get_double(); }
 template<> simdjson_inline simdjson_result<uint64_t> value::get() noexcept { return get_uint64(); }
@@ -30214,7 +30357,7 @@ simdjson_inline value::operator double() noexcept(false) {
   return get_double();
 }
 simdjson_inline value::operator std::string_view() noexcept(false) {
-  return get_string();
+  return get_string(false);
 }
 simdjson_inline value::operator raw_json_string() noexcept(false) {
   return get_raw_json_string();
@@ -30424,9 +30567,13 @@ simdjson_inline simdjson_result<double> simdjson_result<SIMDJSON_BUILTIN_IMPLEME
   if (error()) { return error(); }
   return first.get_double_in_string();
 }
-simdjson_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::value>::get_string() noexcept {
+simdjson_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::value>::get_string(bool allow_replacement) noexcept {
   if (error()) { return error(); }
-  return first.get_string();
+  return first.get_string(allow_replacement);
+}
+simdjson_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::value>::get_wobbly_string() noexcept {
+  if (error()) { return error(); }
+  return first.get_wobbly_string();
 }
 simdjson_inline simdjson_result<SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::raw_json_string> simdjson_result<SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::value>::get_raw_json_string() noexcept {
   if (error()) { return error(); }
@@ -30565,9 +30712,9 @@ simdjson_inline simdjson_result<field> field::start(const value_iterator &parent
     return field(key, parent_iter.child());
 }
 
-simdjson_inline simdjson_warn_unused simdjson_result<std::string_view> field::unescaped_key() noexcept {
+simdjson_inline simdjson_warn_unused simdjson_result<std::string_view> field::unescaped_key(bool allow_replacement) noexcept {
   SIMDJSON_ASSUME(first.buf != nullptr); // We would like to call .alive() but Visual Studio won't let us.
-  simdjson_result<std::string_view> answer = first.unescape(second.iter.json_iter());
+  simdjson_result<std::string_view> answer = first.unescape(second.iter.json_iter(), allow_replacement);
   first.consume();
   return answer;
 }
@@ -30610,9 +30757,9 @@ simdjson_inline simdjson_result<SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::raw_j
   if (error()) { return error(); }
   return first.key();
 }
-simdjson_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::field>::unescaped_key() noexcept {
+simdjson_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::field>::unescaped_key(bool allow_replacement) noexcept {
   if (error()) { return error(); }
-  return first.unescaped_key();
+  return first.unescaped_key(allow_replacement);
 }
 simdjson_inline simdjson_result<SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::field>::value() noexcept {
   if (error()) { return error(); }
@@ -30971,8 +31118,16 @@ simdjson_inline void parser::set_max_capacity(size_t max_capacity) noexcept {
   }
 }
 
-simdjson_inline simdjson_warn_unused simdjson_result<std::string_view> parser::unescape(raw_json_string in, uint8_t *&dst) const noexcept {
-  uint8_t *end = implementation->parse_string(in.buf, dst);
+simdjson_inline simdjson_warn_unused simdjson_result<std::string_view> parser::unescape(raw_json_string in, uint8_t *&dst, bool allow_replacement) const noexcept {
+  uint8_t *end = implementation->parse_string(in.buf, dst, allow_replacement);
+  if (!end) { return STRING_ERROR; }
+  std::string_view result(reinterpret_cast<const char *>(dst), end-dst);
+  dst = end;
+  return result;
+}
+
+simdjson_inline simdjson_warn_unused simdjson_result<std::string_view> parser::unescape_wobbly(raw_json_string in, uint8_t *&dst) const noexcept {
+  uint8_t *end = implementation->parse_wobbly_string(in.buf, dst);
   if (!end) { return STRING_ERROR; }
   std::string_view result(reinterpret_cast<const char *>(dst), end-dst);
   dst = end;

src/arm64/dom_parser_implementation.cpp

@@ -152,8 +152,12 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu
   return stage2::tape_builder::parse_document<true>(*this, _doc);
 }
 
-simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst) const noexcept {
-  return arm64::stringparsing::parse_string(src, dst);
+simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept {
+  return arm64::stringparsing::parse_string(src, dst, allow_replacement);
+}
+
+simdjson_warn_unused uint8_t *dom_parser_implementation::parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept {
+  return arm64::stringparsing::parse_wobbly_string(src, dst);
 }
 
 simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept {

src/fallback/dom_parser_implementation.cpp

@@ -360,8 +360,12 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu
   return stage2::tape_builder::parse_document<true>(*this, _doc);
 }
 
-simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst) const noexcept {
-  return fallback::stringparsing::parse_string(src, dst);
+simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool replacement_char) const noexcept {
+  return fallback::stringparsing::parse_string(src, dst, replacement_char);
+}
+
+simdjson_warn_unused uint8_t *dom_parser_implementation::parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept {
+  return fallback::stringparsing::parse_wobbly_string(src, dst);
 }
 
 simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept {

src/generic/stage1/json_structural_indexer.h

@@ -241,7 +241,9 @@ simdjson_inline void json_structural_indexer::step<64>(const uint8_t *block, buf
 
 simdjson_inline void json_structural_indexer::next(const simd::simd8x64<uint8_t>& in, const json_block& block, size_t idx) {
   uint64_t unescaped = in.lteq(0x1F);
+#if SIMDJSON_UTF8VALIDATION
   checker.check_next_input(in);
+#endif
   indexer.write(uint32_t(idx-64), prev_structurals); // Output *last* iteration's structurals to the parser
   prev_structurals = block.structural_start();
   unescaped_chars_error |= block.non_quote_inside_string(unescaped);

src/generic/stage2/stringparsing.h

@@ -41,7 +41,9 @@ static const uint8_t escape_map[256] = {
 // We work in little-endian then swap at write time
 simdjson_warn_unused
 simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr,
-                                            uint8_t **dst_ptr) {
+                                            uint8_t **dst_ptr, bool allow_replacement) {
+  // Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD)
+  constexpr uint32_t substitution_code_point = 0xfffd;
   // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the
   // conversion isn't valid; we defer the check for this to inside the
   // multilingual plane check
@@ -56,33 +58,79 @@ simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr,
     const uint8_t *src_data = *src_ptr;
     /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */
     if (((src_data[0] << 8) | src_data[1]) != ((static_cast<uint8_t> ('\\') << 8) | static_cast<uint8_t> ('u'))) {
-      return false;
-    }
-    uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2);
+      if(!allow_replacement) { return false; }
+      code_point = substitution_code_point;
+    } else {
+      uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2);
 
-    // We have already checked that the high surrogate is valid and
-    // (code_point - 0xd800) < 1024.
-    //
-    // Check that code_point_2 is in the range 0xdc00..0xdfff
-    // and that code_point_2 was parsed from valid hex.
-    uint32_t low_bit = code_point_2 - 0xdc00;
-    if (low_bit >> 10) {
-      return false;
-    }
+      // We have already checked that the high surrogate is valid and
+      // (code_point - 0xd800) < 1024.
+      //
+      // Check that code_point_2 is in the range 0xdc00..0xdfff
+      // and that code_point_2 was parsed from valid hex.
+      uint32_t low_bit = code_point_2 - 0xdc00;
+      if (low_bit >> 10) {
+        if(!allow_replacement) { return false; }
+        code_point = substitution_code_point;
+      } else {
+        code_point =  (((code_point - 0xd800) << 10) | low_bit) + 0x10000;
+        *src_ptr += 6;
+      }
 
-    code_point =
-        (((code_point - 0xd800) << 10) | low_bit) + 0x10000;
-    *src_ptr += 6;
+    }
   } else if (code_point >= 0xdc00 && code_point <= 0xdfff) {
       // If we encounter a low surrogate (not preceded by a high surrogate)
       // then we have an error.
-      return false;
+      if(!allow_replacement) { return false; }
+      code_point = substitution_code_point;
   }
   size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr);
   *dst_ptr += offset;
   return offset > 0;
 }
 
+
+// handle a unicode codepoint using the wobbly convention
+// https://simonsapin.github.io/wtf-8/
+// write appropriate values into dest
+// src will advance 6 bytes or 12 bytes
+// dest will advance a variable amount (return via pointer)
+// return true if the unicode codepoint was valid
+// We work in little-endian then swap at write time
+simdjson_warn_unused
+simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t **src_ptr,
+                                            uint8_t **dst_ptr) {
+  // It is not ideal that this function is nearly identical to handle_unicode_codepoint.
+  //
+  // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the
+  // conversion isn't valid; we defer the check for this to inside the
+  // multilingual plane check
+  uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2);
+  *src_ptr += 6;
+  // If we found a high surrogate, we must
+  // check for low surrogate for characters
+  // outside the Basic
+  // Multilingual Plane.
+  if (code_point >= 0xd800 && code_point < 0xdc00) {
+    const uint8_t *src_data = *src_ptr;
+    /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */
+    if (((src_data[0] << 8) | src_data[1]) == ((static_cast<uint8_t> ('\\') << 8) | static_cast<uint8_t> ('u'))) {
+      uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2);
+      uint32_t low_bit = code_point_2 - 0xdc00;
+      if ((low_bit >> 10) ==  0) {
+        code_point =
+          (((code_point - 0xd800) << 10) | low_bit) + 0x10000;
+        *src_ptr += 6;
+      }
+    }
+  }
+
+  size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr);
+  *dst_ptr += offset;
+  return offset > 0;
+}
+
+
 /**
  * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There
  * must be an unescaped quote terminating the string. It returns the final output
@@ -91,7 +139,7 @@ simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr,
  * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes +
  * SIMDJSON_PADDING bytes.
  */
-simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst) {
+simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) {
   while (1) {
     // Copy the next n bytes, and find the backslash and quote in them.
     auto bs_quote = backslash_and_quote::copy_and_find(src, dst);
@@ -110,7 +158,54 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, u
            within the unicode codepoint handling code. */
         src += bs_dist;
         dst += bs_dist;
-        if (!handle_unicode_codepoint(&src, &dst)) {
+        if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) {
+          return nullptr;
+        }
+      } else {
+        /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and
+         * write bs_dist+1 characters to output
+         * note this may reach beyond the part of the buffer we've actually
+         * seen. I think this is ok */
+        uint8_t escape_result = escape_map[escape_char];
+        if (escape_result == 0u) {
+          return nullptr; /* bogus escape value is an error */
+        }
+        dst[bs_dist] = escape_result;
+        src += bs_dist + 2;
+        dst += bs_dist + 1;
+      }
+    } else {
+      /* they are the same. Since they can't co-occur, it means we
+       * encountered neither. */
+      src += backslash_and_quote::BYTES_PROCESSED;
+      dst += backslash_and_quote::BYTES_PROCESSED;
+    }
+  }
+  /* can't be reached */
+  return nullptr;
+}
+
+simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) {
+  // It is not ideal that this function is nearly identical to parse_string.
+  while (1) {
+    // Copy the next n bytes, and find the backslash and quote in them.
+    auto bs_quote = backslash_and_quote::copy_and_find(src, dst);
+    // If the next thing is the end quote, copy and return
+    if (bs_quote.has_quote_first()) {
+      // we encountered quotes first. Move dst to point to quotes and exit
+      return dst + bs_quote.quote_index();
+    }
+    if (bs_quote.has_backslash()) {
+      /* find out where the backspace is */
+      auto bs_dist = bs_quote.backslash_index();
+      uint8_t escape_char = src[bs_dist + 1];
+      /* we encountered backslash first. Handle backslash */
+      if (escape_char == 'u') {
+        /* move src/dst up to the start; they will be further adjusted
+           within the unicode codepoint handling code. */
+        src += bs_dist;
+        dst += bs_dist;
+        if (!handle_unicode_codepoint_wobbly(&src, &dst)) {
           return nullptr;
         }
       } else {

src/generic/stage2/tape_builder.h

@@ -145,7 +145,7 @@ simdjson_inline tape_builder::tape_builder(dom::document &doc) noexcept : tape{d
 simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept {
   iter.log_value(key ? "key" : "string");
   uint8_t *dst = on_start_string(iter);
-  dst = stringparsing::parse_string(value+1, dst);
+  dst = stringparsing::parse_string(value+1, dst, false); // We do not allow replacement when the escape characters are invalid.
   if (dst == nullptr) {
     iter.log_error("Invalid escape in string");
     return STRING_ERROR;

src/haswell/dom_parser_implementation.cpp

@@ -153,8 +153,12 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu
   return stage2::tape_builder::parse_document<true>(*this, _doc);
 }
 
-simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst) const noexcept {
-  return haswell::stringparsing::parse_string(src, dst);
+simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool replacement_char) const noexcept {
+  return haswell::stringparsing::parse_string(src, dst, replacement_char);
+}
+
+simdjson_warn_unused uint8_t *dom_parser_implementation::parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept {
+  return haswell::stringparsing::parse_wobbly_string(src, dst);
 }
 
 simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept {

src/icelake/dom_parser_implementation.cpp

@@ -199,8 +199,12 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu
   return stage2::tape_builder::parse_document<true>(*this, _doc);
 }
 
-simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst) const noexcept {
-  return icelake::stringparsing::parse_string(src, dst);
+simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool replacement_char) const noexcept {
+  return icelake::stringparsing::parse_string(src, dst, replacement_char);
+}
+
+simdjson_warn_unused uint8_t *dom_parser_implementation::parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept {
+  return icelake::stringparsing::parse_wobbly_string(src, dst);
 }
 
 simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept {

src/ppc64/dom_parser_implementation.cpp

@@ -122,8 +122,12 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu
   return stage2::tape_builder::parse_document<true>(*this, _doc);
 }
 
-simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst) const noexcept {
-  return ppc64::stringparsing::parse_string(src, dst);
+simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool replacement_char) const noexcept {
+  return ppc64::stringparsing::parse_string(src, dst, replacement_char);
+}
+
+simdjson_warn_unused uint8_t *dom_parser_implementation::parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept {
+  return ppc64::stringparsing::parse_wobbly_string(src, dst);
 }
 
 simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept {

src/westmere/dom_parser_implementation.cpp

@@ -152,8 +152,12 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu
   return stage2::tape_builder::parse_document<true>(*this, _doc);
 }
 
-simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst) const noexcept {
-  return westmere::stringparsing::parse_string(src, dst);
+simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool replacement_char) const noexcept {
+  return westmere::stringparsing::parse_string(src, dst, replacement_char);
+}
+
+simdjson_warn_unused uint8_t *dom_parser_implementation::parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept {
+  return westmere::stringparsing::parse_wobbly_string(src, dst);
 }
 
 simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept {

tests/dom/basictests.cpp

@@ -551,7 +551,7 @@ namespace parse_api_tests {
     ASSERT_SUCCESS( parser.parse_many(empty_batches_ndjson, BATCH_SIZE*16).get(stream) );
     for (auto doc : stream) {
       count++;
-      uint64_t val;
+      uint64_t val{};
       ASSERT_SUCCESS( doc.get(val) );
       ASSERT_EQUAL( val, count );
     }
@@ -1039,7 +1039,7 @@ namespace dom_api_tests {
     for (auto tweet : tweets) {
       object user;
       ASSERT_SUCCESS( tweet["user"].get(user) );
-      bool default_profile;
+      bool default_profile{};
       ASSERT_SUCCESS( user["default_profile"].get(default_profile) );
       if (default_profile) {
         std::string_view screen_name;

tests/ondemand/ondemand_misc_tests.cpp

@@ -5,6 +5,56 @@ using namespace simdjson;
 
 namespace misc_tests {
   using namespace std;
+  bool replacement_char() {
+    auto fun_phrase = R"( ["I \u2665 Unicode. Even broken \ud800 Unicode." ])"_padded;
+    std::string_view expected_fun = "I \xe2\x99\xa5 Unicode. Even broken \xef\xbf\xbd Unicode.";
+
+    TEST_START();
+    ondemand::parser parser;
+    ondemand::document doc;
+    ASSERT_SUCCESS(parser.iterate(fun_phrase).get(doc));
+    ondemand::array arr;
+    ASSERT_SUCCESS( doc.get_array().get(arr));
+    std::string_view view;
+    ASSERT_SUCCESS( arr.at(0).get_string(true).get(view));
+    ASSERT_EQUAL(view, expected_fun);
+    TEST_SUCCEED();
+  }
+
+  bool wobbly_tests() {
+    auto lone_surrogate = R"( "\ud800" )"_padded;
+    std::string_view expected_lone = "\xed\xa0\x80";
+
+
+    auto fun_phrase = R"( ["I \u2665 Unicode. Even broken \ud800 Unicode." ])"_padded;
+    std::string_view expected_fun = "I \xe2\x99\xa5 Unicode. Even broken \xed\xa0\x80 Unicode.";
+
+    auto insane_url = R"({"input": "http://example.com/\uDC00\uD834\uDF06\uDC00"} )"_padded;
+    std::string_view expected_insane = "http://example.com/\xED\xB0\x80\xF0\x9D\x8C\x86\xED\xB0\x80";
+
+    TEST_START();
+    ondemand::parser parser;
+    ondemand::document doc;
+    ASSERT_SUCCESS(parser.iterate(lone_surrogate).get(doc));
+    std::string_view view;
+    ASSERT_SUCCESS( doc.get_wobbly_string().get(view));
+    ASSERT_EQUAL(view, expected_lone);
+
+    ASSERT_SUCCESS(parser.iterate(fun_phrase).get(doc));
+    ondemand::array arr;
+    ASSERT_SUCCESS( doc.get_array().get(arr));
+    ASSERT_SUCCESS( arr.at(0).get_wobbly_string().get(view));
+    ASSERT_EQUAL(view, expected_fun);
+
+    ASSERT_SUCCESS(parser.iterate(insane_url).get(doc));
+    ondemand::object obj;
+    ASSERT_SUCCESS( doc.get_object().get(obj));
+    ASSERT_SUCCESS( obj["input"].get_wobbly_string().get(view));
+
+    ASSERT_EQUAL(view, expected_insane);
+    TEST_SUCCEED();
+  }
+
   bool test_get_value() {
     TEST_START();
     ondemand::parser parser;
@@ -520,6 +570,8 @@ namespace misc_tests {
 
   bool run() {
     return
+           replacement_char() &&
+           wobbly_tests() &&
            issue_uffff() &&
            issue_backslash() &&
            issue1870() &&

tests/ondemand/ondemand_readme_examples.cpp

@@ -416,6 +416,40 @@ bool using_the_parsed_json_3() {
   TEST_SUCCEED();
 }
 
+bool using_the_parsed_json_3b() {
+  TEST_START();
+  ondemand::parser parser;
+  auto cars_json = R"( {
+    "identifier1":{ "make": "Toyota", "model": "Camry",  "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
+    "identifier2":{ "make": "Kia",    "model": "Soul",   "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
+    "identifier3":{ "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
+  } )"_padded;
+
+  // Iterating through an array of objects
+  ondemand::document doc = parser.iterate(cars_json);
+  for (ondemand::field key_car : doc.get_object()) {
+    // If I need a string_view and/or, I can use key_car.unescaped_key() instead, but
+    // key_car.key() will be more performant otherwise.
+    cout << "identifier : " << key_car.key() << std::endl;
+    // I can now access the subobject:
+    ondemand::object car = key_car.value();
+    // Accessing a field by name
+    cout << "Make/Model: " << std::string_view(car["make"]) << "/" << std::string_view(car["model"]) << endl;
+
+    // Casting a JSON element to an integer
+    uint64_t year = car["year"];
+    cout << "- This car is " << 2020 - year << "years old." << endl;
+
+    // Iterating through an array of floats
+    double total_tire_pressure = 0;
+    for (double tire_pressure : car["tire_pressure"]) {
+      total_tire_pressure += tire_pressure;
+    }
+    cout << "- Average tire pressure: " << (total_tire_pressure / 4) << endl;
+  }
+
+  TEST_SUCCEED();
+}
 
 bool using_the_parsed_json_rewind() {
   TEST_START();
@@ -993,9 +1027,185 @@ bool current_location_no_error() {
   TEST_SUCCEED();
 }
 
+
+
+struct ZuluBBox {
+  double xmin;
+  double ymin;
+  double width;
+  double height;
+
+  void print() {
+    std::cout << xmin << ", " << ymin << ", " << width << ", " << height
+              << std::endl;
+  }
+};
+#if SIMDJSON_EXCEPTIONS
+
+bool example1956() {
+
+  auto json = R"+( {
+  "ZuluROI": {
+    "ZuluBBox": {
+      "xmin": 0,
+      "ymin": 0,
+      "width": 1,
+      "height": 1
+    },
+    "SubObjects": [
+      {
+        "ZuluDetection": {
+          "label": "car",
+          "class_id": 3,
+          "confidence": 0.7587034106254578,
+          "ZuluBBox": {
+            "xmin": 0.3843536376953125,
+            "ymin": 0.4532909393310547,
+            "width": 0.09115534275770187,
+            "height": 0.04127710685133934
+          },
+          "SubObjects": []
+        }
+      },
+      {
+        "ZuluDetection": {
+          "label": "car",
+          "class_id": 3,
+          "confidence": 0.6718865633010864,
+          "ZuluBBox": {
+            "xmin": 0.7500002980232239,
+            "ymin": 0.5212296843528748,
+            "width": 0.07592231780290604,
+            "height": 0.038947589695453644
+          },
+          "SubObjects": []
+        }
+      },
+      {
+        "ZuluDetection": {
+          "label": "car",
+          "class_id": 3,
+          "confidence": 0.5806200504302979,
+          "ZuluBBox": {
+            "xmin": 0.9025363922119141,
+            "ymin": 0.5925348401069641,
+            "width": 0.05478987470269203,
+            "height": 0.046337299048900604
+          },
+          "SubObjects": []
+        }
+      }
+    ]
+  },
+  "timestamp (ms)": 1677085594421,
+  "buffer_offset": 35673
+} )+"_padded;
+  ondemand::parser parser;
+  ondemand::document doc = parser.iterate(json);
+  ondemand::object root_object = doc.get_object();
+  ondemand::object roi_object = root_object["ZuluROI"];
+
+  ondemand::object box_roi_object = roi_object["ZuluBBox"];
+  ZuluBBox box = {
+      double(box_roi_object["xmin"]), double(box_roi_object["ymin"]),
+      double(box_roi_object["width"]), double(box_roi_object["height"])};
+  box.print();
+
+  for (ondemand::object value : roi_object["SubObjects"]) {
+    ondemand::object detect = value["ZuluDetection"];
+    std::cout << detect["label"].get_string() << std::endl;
+    std::cout << detect["class_id"].get_uint64() << std::endl;
+    std::cout << detect["confidence"].get_double() << std::endl;
+
+    ondemand::object vbox_roi_object = detect["ZuluBBox"];
+    ZuluBBox vbox = {
+        double(vbox_roi_object["xmin"]), double(vbox_roi_object["ymin"]),
+        double(vbox_roi_object["width"]), double(vbox_roi_object["height"])};
+    vbox.print();
+  }
+
+  std::cout << root_object["timestamp (ms)"].get_uint64() << std::endl;
+  std::cout << root_object["buffer_offset"].get_uint64() << std::endl;
+  return true;
+}
+
+
+bool example1958() {
+  auto json = R"+( {
+    "5f08a730b280e54fd1e75a7046b93fdc": {
+        "file": "/DEMOS/0-9/10_Orbyte.sid",
+        "len": [
+            "1:17"
+        ],
+        "loud": [
+            "-22.8"
+        ],
+        "name": "10 Orbyte",
+        "author": "Michael Becker (Premium)",
+        "release": "2014 Tristar & Red Sector Inc.",
+        "bits": 20
+    },
+    "2727236ead44a62f0c6e01f6dd4dc484": {
+        "file": "/DEMOS/0-9/12345.sid",
+        "len": [
+            "0:56"
+        ],
+        "loud": [
+            "-33.3"
+        ],
+        "name": "12345",
+        "author": "Beal",
+        "release": "1988 Beal",
+        "bits": 20
+    },
+    "7ea765fce6c0f92570b18adc7bf52f54": {
+        "file": "/DEMOS/0-9/128_Byte_Blues_BASIC.sid",
+        "len": [
+            "0:18"
+        ],
+        "loud": [
+            "-27.1"
+        ],
+        "name": "128 Byte Blues",
+        "author": "Leonard J. Paul (Freaky DNA)",
+        "release": "2005 Freaky DNA",
+        "bits": 62
+    }
+} )+"_padded;
+  ondemand::parser parser;
+  ondemand::document doc = parser.iterate(json);
+  ondemand::object root_object = doc.get_object();
+  for(auto key_value : root_object) {
+    // could get std::string_view with 'unescaped_key()':
+    std::cout << "key: " << key_value.key() << std::endl;
+    ondemand::object obj = key_value.value();
+
+    std::cout << "file: " << std::string_view(obj["file"]) << std::endl;
+
+    std::cout << "len: ";
+    for(std::string_view values : obj["len"]) {
+      std::cout << values << std::endl;
+    }
+    std::cout << std::endl;
+
+    std::cout << "loud: ";
+    for(std::string_view values : obj["loud"]) {
+      std::cout << values << std::endl;
+    }
+    std::cout << std::endl;
+
+    std::cout << "name: " << std::string_view(obj["name"]) << std::endl;
+    std::cout << "author: " << std::string_view(obj["author"]) << std::endl;
+    std::cout << "release: " << std::string_view(obj["release"]) << std::endl;
+    std::cout << "bits: " << uint64_t(obj["bits"]) << std::endl;
+  }
+  return true;
+}
+#endif
 bool run() {
   return true
 #if SIMDJSON_EXCEPTIONS
+    && example1956() && example1958()
 //    && basics_1() // Fails because twitter.json isn't in current directory. Compile test only.
     &&  basics_treewalk()
     &&  basics_treewalk_breakline()
@@ -1012,6 +1222,7 @@ bool run() {
     && big_integer()
     && big_integer_in_string()
     && using_the_parsed_json_3()
+    && using_the_parsed_json_3b()
     && using_the_parsed_json_4()
     && using_the_parsed_json_5()
 #endif