Updating single header, prior to release.

* Patch for possible AVX-512 overflow.

* Updating the test for new padding.

* Preparing new version.

* replace binary integer literals with hex literals for C++11 compatibility (#1855)

Binary integer literals are a C++14 feature, so those are not supported
in C++11 and should be replaced by hexadecimal literals instead.

Fixes #1854.

Co-authored-by: Dirk Stolle <striezel-dev@web.de>

.vscode/settings.json

@@ -86,6 +86,16 @@
         "vector": "cpp",
         "*.ipp": "cpp",
         "__functional_base_03": "cpp",
-        "filesystem": "cpp"
+        "filesystem": "cpp",
+        "*.inc": "cpp",
+        "compare": "cpp",
+        "concepts": "cpp",
+        "variant": "cpp",
+        "__bits": "cpp",
+        "csignal": "cpp",
+        "future": "cpp",
+        "queue": "cpp",
+        "shared_mutex": "cpp",
+        "ranges": "cpp"
     }
 }

CMakeLists.txt

@@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.14)
 project(
     simdjson
     # The version number is modified by tools/release.py
-    VERSION 2.0.4
+    VERSION 2.2.0
     DESCRIPTION "Parsing gigabytes of JSON per second"
     HOMEPAGE_URL "https://simdjson.org/"
     LANGUAGES CXX C
@@ -20,8 +20,8 @@ string(
 # ---- Options, variables ----
 
 # These version numbers are modified by tools/release.py
-set(SIMDJSON_LIB_VERSION "11.0.0" CACHE STRING "simdjson library version")
-set(SIMDJSON_LIB_SOVERSION "11" CACHE STRING "simdjson library soversion")
+set(SIMDJSON_LIB_VERSION "13.0.0" CACHE STRING "simdjson library version")
+set(SIMDJSON_LIB_SOVERSION "13" CACHE STRING "simdjson library soversion")
 
 option(SIMDJSON_ENABLE_THREADS "Link with thread support" ON)
 

Doxyfile

@@ -38,7 +38,7 @@ PROJECT_NAME           = simdjson
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = "2.0.4"
+PROJECT_NUMBER         = "2.2.0"
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a

README.md

@@ -111,10 +111,12 @@ For NDJSON files, we can exceed 3 GB/s with [our  multithreaded parsing function
 Real-world usage
 ----------------
 
-- [Microsoft FishStore](https://github.com/microsoft/FishStore)
-- [Yandex ClickHouse](https://github.com/yandex/ClickHouse)
+- [ClickHouse](https://github.com/ClickHouse/ClickHouse)
 - [Clang Build Analyzer](https://github.com/aras-p/ClangBuildAnalyzer)
 - [Shopify HeapProfiler](https://github.com/Shopify/heap-profiler)
+- [StarRocks](https://github.com/StarRocks/starrocks)
+- [Microsoft FishStore](https://github.com/microsoft/FishStore)
+
 
 If you are planning to use simdjson in a product, please work from one of our releases.
 

benchmark/bench_ondemand.cpp

@@ -25,6 +25,11 @@ SIMDJSON_PUSH_DISABLE_ALL_WARNINGS
 #include <benchmark/benchmark.h>
 
 SIMDJSON_POP_DISABLE_WARNINGS
+#include "json2msgpack/simdjson_ondemand.h"
+#include "json2msgpack/rapidjson.h"
+#include "json2msgpack/yyjson.h"
+#include "json2msgpack/sajson.h"
+#include "json2msgpack/nlohmann_json.h"
 
 #include "amazon_cellphones/simdjson_dom.h"
 #include "amazon_cellphones/simdjson_ondemand.h"

benchmark/distinct_user_id/rapidjson.h

@@ -48,7 +48,7 @@ BENCHMARK_TEMPLATE(distinct_user_id, rapidjson)->UseManualTime();
 
 struct rapidjson_insitu : rapidjson_base {
   bool run(simdjson::padded_string &json, std::vector<uint64_t> &result) {
-    return rapidjson_base::run(doc.ParseInsitu<kParseValidateEncodingFlag>(json.data()), result);
+    return rapidjson_base::run(doc.ParseInsitu<kParseValidateEncodingFlag|kParseInsituFlag>(json.data()), result);
   }
 };
 BENCHMARK_TEMPLATE(distinct_user_id, rapidjson_insitu)->UseManualTime();

benchmark/distinct_user_id/sajson.h

@@ -9,6 +9,8 @@ namespace distinct_user_id {
 struct sajson {
   size_t ast_buffer_size{0};
   size_t *ast_buffer{nullptr};
+  ~sajson() { free(ast_buffer); }
+
   simdjson_really_inline std::string_view get_string_view(const ::sajson::value &obj, std::string_view key) {
     auto val = obj.get_value_of_key({key.data(), key.length()});
     if (val.get_type() != ::sajson::TYPE_STRING) { throw "field is not a string"; }

benchmark/find_tweet/rapidjson.h

@@ -42,7 +42,7 @@ BENCHMARK_TEMPLATE(find_tweet, rapidjson)->UseManualTime();
 
 struct rapidjson_insitu : rapidjson_base {
   bool run(simdjson::padded_string &json, uint64_t find_id, std::string_view &result) {
-    return rapidjson_base::run(doc.ParseInsitu<kParseValidateEncodingFlag>(json.data()), find_id, result);
+    return rapidjson_base::run(doc.ParseInsitu<kParseValidateEncodingFlag|kParseInsituFlag>(json.data()), find_id, result);
   }
 };
 BENCHMARK_TEMPLATE(find_tweet, rapidjson_insitu)->UseManualTime();

benchmark/find_tweet/sajson.h

@@ -11,6 +11,8 @@ struct sajson {
 
   size_t ast_buffer_size{0};
   size_t *ast_buffer{nullptr};
+  ~sajson() { free(ast_buffer); }
+
   simdjson_really_inline std::string_view get_string_view(const ::sajson::value &obj, std::string_view key) {
     auto val = obj.get_value_of_key({key.data(), key.length()});
     if (val.get_type() != ::sajson::TYPE_STRING) { throw "field is not a string"; }

benchmark/json2msgpack/json2msgpack.h

@@ -0,0 +1,48 @@
+#pragma once
+
+#include "json_benchmark/file_runner.h"
+
+namespace json2msgpack {
+
+using namespace json_benchmark;
+
+template <typename I> struct runner : public file_runner<I> {
+  std::string_view result;
+  std::unique_ptr<char[]> buffer;
+
+  bool setup(benchmark::State &state) {
+    bool isok = this->load_json(state, TWITTER_JSON);
+    if (isok) {
+      // Let us allocate a sizeable buffer.
+      buffer = std::unique_ptr<char[]>(new char[this->json.size() * 4 + 1024]);
+    }
+    return isok;
+  }
+
+  bool before_run(benchmark::State &state) {
+    if (!file_runner<I>::before_run(state)) {
+      return false;
+    }
+    // Clear the buffer.
+    ::memset(buffer.get(), 0, this->json.size() * 4 + 1024);
+    return true;
+  }
+
+  bool run(benchmark::State &) {
+    return this->implementation.run(this->json, buffer.get(), result);
+  }
+
+  template <typename R>
+  bool diff(benchmark::State &state, runner<R> &reference) {
+    return diff_results(state, result.size(), reference.result.size(), diff_flags::NONE);
+  }
+};
+
+struct simdjson_ondemand;
+
+template <typename I>
+simdjson_really_inline static void json2msgpack(benchmark::State &state) {
+  run_json_benchmark<runner<I>, runner<simdjson_ondemand>>(state);
+}
+
+} // namespace json2msgpack

benchmark/json2msgpack/nlohmann_json.h

@@ -0,0 +1,117 @@
+#pragma once
+#ifdef SIMDJSON_COMPETITION_NLOHMANN_JSON
+
+#include "json2msgpack.h"
+
+namespace json2msgpack {
+
+using namespace nlohmann;
+
+struct nlohmann_json2msgpack {
+  inline std::string_view to_msgpack(const simdjson::padded_string &json,
+                                     uint8_t *buf);
+
+private:
+  inline void write_double(const double d) noexcept;
+  inline void write_byte(const uint8_t b) noexcept;
+  inline void write_uint32(const uint32_t w) noexcept;
+  inline void write_string(const std::string& str);
+  inline void recursive_processor(basic_json<> element);
+
+  uint8_t *buff{};
+};
+
+std::string_view nlohmann_json2msgpack::to_msgpack(const simdjson::padded_string &json,
+                             uint8_t *buf) {
+  buff = buf;
+  auto val = nlohmann::json::parse(json.data(), json.data() + json.size());
+  recursive_processor(val);
+  return std::string_view(reinterpret_cast<char *>(buf), size_t(buff - buf));
+}
+
+void nlohmann_json2msgpack::write_double(const double d) noexcept {
+  *buff++ = 0xcb;
+  ::memcpy(buff, &d, sizeof(d));
+  buff += sizeof(d);
+}
+
+void nlohmann_json2msgpack::write_byte(const uint8_t b) noexcept {
+  *buff = b;
+  buff++;
+}
+
+void nlohmann_json2msgpack::write_uint32(const uint32_t w) noexcept {
+  ::memcpy(buff, &w, sizeof(w));
+  buff += sizeof(w);
+}
+
+void nlohmann_json2msgpack::write_string(const std::string & str) {
+  write_byte(0xdb);
+  write_uint32(uint32_t(str.size()));
+  ::memcpy(buff, str.data(), str.size());
+  buff += str.size();
+}
+
+void nlohmann_json2msgpack::recursive_processor(json element) {
+  switch (element.type()) {
+  case nlohmann::detail::value_t::array: {
+    uint32_t counter = 0;
+    write_byte(0xdd);
+    std::vector<json> array = element.get<std::vector<json>>();
+    write_uint32(uint32_t(array.size()));
+    for (auto child : array) {
+       recursive_processor(child);
+    }
+  } break;
+  case nlohmann::detail::value_t::object: {
+    write_byte(0xdf);
+    std::map<std::string,json> object = element.get<std::map<std::string,json>>();
+    write_uint32(uint32_t(object.size()));
+    for (auto field : object) {
+      write_string(field.first);
+      recursive_processor(field.second);
+    }
+  } break;
+
+  case nlohmann::detail::value_t::number_integer:
+  case nlohmann::detail::value_t::number_unsigned:
+  case nlohmann::detail::value_t::number_float:
+    write_double(double(element));
+    break;
+  case nlohmann::detail::value_t::string:
+    write_string(std::string(element));
+    break;
+  case nlohmann::detail::value_t::boolean:
+    write_byte(0xc2 + bool(element));
+    break;
+  case nlohmann::detail::value_t::null:
+    write_byte(0xc0);
+    break;
+  case nlohmann::detail::value_t::discarded:
+  case nlohmann::detail::value_t::binary:
+  default:
+    printf("unexpected\n");
+    break;
+  }
+}
+
+struct nlohmann_json {
+  using StringType = std::string_view;
+
+  nlohmann_json2msgpack parser{};
+
+  bool run(simdjson::padded_string &json, char *buffer,
+           std::string_view &result) {
+    result = parser.to_msgpack(json, reinterpret_cast<uint8_t *>(buffer));
+    return true;
+  }
+};
+
+BENCHMARK_TEMPLATE(json2msgpack, nlohmann_json)->UseManualTime();
+
+} // namespace json2msgpack
+
+#endif // SIMDJSON_COMPETITION_NLOHMANN_JSON
+
+
+

benchmark/json2msgpack/rapidjson.h

@@ -0,0 +1,140 @@
+
+
+#pragma once
+#ifdef SIMDJSON_COMPETITION_RAPIDJSON
+
+#include "json2msgpack.h"
+
+namespace json2msgpack {
+
+using namespace rapidjson;
+
+template <int parseflag>
+struct rapidjson2msgpack {
+  inline std::string_view to_msgpack(char *json, uint8_t *buf);
+
+private:
+  inline void write_double(const double d) noexcept;
+  inline void write_byte(const uint8_t b) noexcept;
+  inline void write_uint32(const uint32_t w) noexcept;
+  inline void write_uint32_at(const uint32_t w, uint8_t *p) noexcept;
+  void write_string(const char * s, size_t length) noexcept;
+  inline void recursive_processor(Value &v);
+
+  uint8_t *buff{};
+};
+
+template <int parseflag>
+std::string_view rapidjson2msgpack<parseflag>::to_msgpack(char *json, uint8_t *buf) {
+  buff = buf;
+  Document doc{};
+  if(parseflag & kParseInsituFlag) {
+    doc.ParseInsitu<parseflag>(json);
+  } else {
+    doc.Parse<parseflag>(json);
+  }
+  recursive_processor(doc);
+  return std::string_view(reinterpret_cast<char *>(buf), size_t(buff - buf));
+}
+
+template <int parseflag>
+void rapidjson2msgpack<parseflag>::write_double(const double d) noexcept {
+  *buff++ = 0xcb;
+  ::memcpy(buff, &d, sizeof(d));
+  buff += sizeof(d);
+}
+
+template <int parseflag>
+void rapidjson2msgpack<parseflag>::write_byte(const uint8_t b) noexcept {
+  *buff = b;
+  buff++;
+}
+
+template <int parseflag>
+void rapidjson2msgpack<parseflag>::write_string(const char * c, size_t len) noexcept {
+  write_byte(0xdb);
+  write_uint32(uint32_t(len));
+  ::memcpy(buff, c, len);
+  buff += len;
+}
+
+template <int parseflag>
+void rapidjson2msgpack<parseflag>::write_uint32(const uint32_t w) noexcept {
+  ::memcpy(buff, &w, sizeof(w));
+  buff += sizeof(w);
+}
+
+template <int parseflag>
+void rapidjson2msgpack<parseflag>::write_uint32_at(const uint32_t w, uint8_t *p) noexcept {
+  ::memcpy(p, &w, sizeof(w));
+}
+
+template <int parseflag>
+void rapidjson2msgpack<parseflag>::recursive_processor(Value &v) {
+  switch (v.GetType()) {
+  case kArrayType:
+    write_byte(0xdd);
+    write_uint32(v.Size());
+    for (Value::ValueIterator i = v.Begin(); i != v.End(); ++i) {
+      recursive_processor(*i);
+    }
+    break;
+  case kObjectType:
+    write_byte(0xdf);
+    write_uint32(uint32_t(v.MemberEnd()-v.MemberBegin()));
+    for (Value::MemberIterator m = v.MemberBegin(); m != v.MemberEnd();
+         ++m) {
+      write_string(m->name.GetString(), m->name.GetStringLength());
+      recursive_processor(m->value);
+    }
+    break;
+  case kStringType:
+    write_string(v.GetString(), v.GetStringLength());
+    break;
+  case kNumberType:
+    write_double(v.GetDouble());
+    break;
+  case kFalseType:
+    write_byte(0xc2);
+    break;
+  case kTrueType:
+    write_byte(0xc3);
+    break;
+  case kNullType:
+    write_byte(0xc0);
+    break;
+  }
+}
+
+template <int parseflag>
+struct rapidjson_base {
+  using StringType = std::string_view;
+
+  rapidjson2msgpack<parseflag> parser{};
+
+  bool run(simdjson::padded_string &json, char *buffer,
+           std::string_view &result) {
+    result =
+        parser.to_msgpack(json.data(), reinterpret_cast<uint8_t *>(buffer));
+
+    return true;
+  }
+};
+
+
+using rapidjson_lossless = rapidjson_base<kParseValidateEncodingFlag|kParseFullPrecisionFlag>;
+
+BENCHMARK_TEMPLATE(json2msgpack, rapidjson_lossless)->UseManualTime();
+
+
+using rapidjson = rapidjson_base<kParseValidateEncodingFlag>;
+
+BENCHMARK_TEMPLATE(json2msgpack, rapidjson)->UseManualTime();
+
+using rapidjson_insitu = rapidjson_base<kParseValidateEncodingFlag|kParseInsituFlag>;
+
+BENCHMARK_TEMPLATE(json2msgpack, rapidjson_insitu)->UseManualTime();
+
+} // namespace json2msgpack
+
+#endif // SIMDJSON_COMPETITION_RAPIDJSON

benchmark/json2msgpack/sajson.h

@@ -0,0 +1,131 @@
+#pragma once
+#ifdef SIMDJSON_COMPETITION_SAJSON
+
+#include "json2msgpack.h"
+
+namespace json2msgpack {
+
+using namespace sajson;
+
+
+struct sajson2msgpack {
+  inline std::string_view to_msgpack(char *json, size_t size, uint8_t *buf);
+  virtual ~sajson2msgpack() { free(ast_buffer); }
+
+private:
+  inline void write_double(const double d) noexcept;
+  inline void write_byte(const uint8_t b) noexcept;
+  inline void write_uint32(const uint32_t w) noexcept;
+  inline void write_string(const char * s, size_t length) noexcept;
+  inline void recursive_processor(const sajson::value &v);
+
+  uint8_t *buff{};
+  size_t ast_buffer_size{0};
+  size_t *ast_buffer{nullptr};
+};
+
+
+std::string_view sajson2msgpack::to_msgpack(char *json, size_t size, uint8_t *buf) {
+  buff = buf;
+
+  if (!ast_buffer) {
+    ast_buffer_size = size;
+    ast_buffer = (size_t *)std::malloc(ast_buffer_size * sizeof(size_t));
+  }
+  auto doc = parse(
+      bounded_allocation(ast_buffer, ast_buffer_size),
+      mutable_string_view(size, json)
+  );
+
+  auto root = doc.get_root();
+  recursive_processor(root);
+  return std::string_view(reinterpret_cast<char *>(buf), size_t(buff - buf));
+}
+
+void sajson2msgpack::write_string(const char * c, size_t len) noexcept {
+  write_byte(0xdb);
+  write_uint32(uint32_t(len));
+  ::memcpy(buff, c, len);
+  buff += len;
+}
+
+void sajson2msgpack::write_double(const double d) noexcept {
+  *buff++ = 0xcb;
+  ::memcpy(buff, &d, sizeof(d));
+  buff += sizeof(d);
+}
+
+void sajson2msgpack::write_byte(const uint8_t b) noexcept {
+  *buff = b;
+  buff++;
+}
+
+void sajson2msgpack::write_uint32(const uint32_t w) noexcept {
+  ::memcpy(buff, &w, sizeof(w));
+  buff += sizeof(w);
+}
+
+void sajson2msgpack::recursive_processor(const sajson::value &node) {
+  using namespace sajson;
+  switch (node.get_type()) {
+  case TYPE_NULL:
+    write_byte(0xc0);
+    break;
+  case TYPE_FALSE:
+    write_byte(0xc2);
+    break;
+  case TYPE_TRUE:
+    write_byte(0xc3);
+    break;
+  case TYPE_ARRAY: {
+    auto length = node.get_length();
+    write_byte(0xdf);
+    write_uint32(uint32_t(length));
+    for (size_t i = 0; i < length; ++i) {
+      recursive_processor(node.get_array_element(i));
+    }
+    break;
+  }
+  case TYPE_OBJECT: {
+    auto length = node.get_length();
+    write_byte(0xdd);
+    write_uint32(uint32_t(length));
+    for (auto i = 0u; i < length; ++i) {
+      auto s = node.get_object_key(i);
+      write_string(s.data(), s.length());
+      recursive_processor(node.get_object_value(i));
+    }
+    break;
+  }
+  case TYPE_STRING:
+    write_string(node.as_cstring(), node.get_string_length());
+    break;
+  case TYPE_DOUBLE:
+  case TYPE_INTEGER:
+    write_double(node.get_number_value());
+    break;
+  default:
+    assert(false && "unknown node type");
+  }
+}
+
+
+struct sajson {
+  using StringType = std::string_view;
+
+  sajson2msgpack parser{};
+
+  bool run(simdjson::padded_string &json, char *buffer,
+           std::string_view &result) {
+    result =
+        parser.to_msgpack(json.data(), json.size(), reinterpret_cast<uint8_t *>(buffer));
+
+    return true;
+  }
+};
+
+BENCHMARK_TEMPLATE(json2msgpack, sajson)->UseManualTime();
+
+} // namespace json2msgpack
+
+#endif // SIMDJSON_COMPETITION_SAJSON

benchmark/json2msgpack/simdjson_ondemand.h

@@ -0,0 +1,183 @@
+#pragma once
+#if SIMDJSON_EXCEPTIONS
+
+#include "json2msgpack.h"
+
+namespace json2msgpack {
+
+using namespace simdjson;
+
+/**
+ * @brief The simdjson2msgpack struct is used to quickly convert
+ * JSON strings to msgpack views. You must provide a pointer to
+ * a large memory region where the msgpack gets written. The
+ * buffer should be large enough to store the msgpack output (which
+ * can never be 3x larger than the input JSON) with an additional
+ * simdjson::SIMDJSON_PADDING bytes.
+ *
+ * Recommended usage:
+ *
+ * simdjson2msgpack parser{};
+ * simdjson::padded_string json = "[1,2]"_padded; // some JSON
+ * uint8_t * buffer = new uint8_t[3*json.size() + simdjson::SIMDJSON_PADDING]; // large buffer
+ *
+ * std::string_view msgpack = parser.to_msgpack(json, buffer);
+ *
+ * The result (msgpack) is a string view to a msgpack serialization of the input JSON,
+ * it points inside the buffer you provided.
+ *
+ * You may reuse the simdjson2msgpack instance though you should use
+ * one per thread.
+ */
+struct simdjson2msgpack {
+  /**
+   * @brief Converts the provided JSON into msgpack.
+   *
+   * @param json JSON input
+   * @param buf temporary buffer (must be large enough, with simdjson::SIMDJSON_PADDING bytes
+   * of padding)
+   * @return std::string_view msgpack output, writting to the temporary buffer
+   */
+  inline std::string_view to_msgpack(const simdjson::padded_string &json,
+                                     uint8_t *buf);
+
+private:
+  simdjson_really_inline void write_double(const double d) noexcept;
+  simdjson_really_inline void write_byte(const uint8_t b) noexcept;
+  simdjson_really_inline void write_uint32(const uint32_t w) noexcept;
+  simdjson_really_inline uint8_t *skip_uint32() noexcept;
+  simdjson_really_inline void write_uint32_at(const uint32_t w,
+                                              uint8_t *p) noexcept;
+  simdjson_really_inline void
+  write_raw_string(simdjson::ondemand::raw_json_string rjs);
+  inline void recursive_processor(simdjson::ondemand::value element);
+
+  simdjson::ondemand::parser parser;
+  simdjson::ondemand::document doc;
+  uint8_t *buff{};
+};
+
+std::string_view
+simdjson2msgpack::to_msgpack(const simdjson::padded_string &json,
+                             uint8_t *buf) {
+  buff = buf;
+  ondemand::document doc = parser.iterate(json);
+  if (doc.is_scalar()) {
+    // we have a special case where the JSON document is a single document...
+    switch (doc.type()) {
+    case simdjson::ondemand::json_type::number:
+      write_double(doc.get_double());
+      break;
+    case simdjson::ondemand::json_type::string:
+      write_raw_string(doc.get_raw_json_string());
+      break;
+    case simdjson::ondemand::json_type::boolean:
+      write_byte(0xc2 + doc.get_bool());
+      break;
+    case simdjson::ondemand::json_type::null:
+      write_byte(0xc0);
+      break;
+    case simdjson::ondemand::json_type::array:
+    case simdjson::ondemand::json_type::object:
+    default:
+      // impossible
+      break;
+    }
+  } else {
+    simdjson::ondemand::value val = doc;
+    recursive_processor(val);
+  }
+  return std::string_view(reinterpret_cast<char *>(buf), size_t(buff - buf));
+}
+
+void simdjson2msgpack::write_double(const double d) noexcept {
+  *buff++ = 0xcb;
+  ::memcpy(buff, &d, sizeof(d));
+  buff += sizeof(d);
+}
+
+void simdjson2msgpack::write_byte(const uint8_t b) noexcept {
+  *buff = b;
+  buff++;
+}
+
+void simdjson2msgpack::write_uint32(const uint32_t w) noexcept {
+  ::memcpy(buff, &w, sizeof(w));
+  buff += sizeof(w);
+}
+
+uint8_t *simdjson2msgpack::skip_uint32() noexcept {
+  uint8_t *ret = buff;
+  buff += sizeof(uint32_t);
+  return ret;
+}
+
+void simdjson2msgpack::write_uint32_at(const uint32_t w, uint8_t *p) noexcept {
+  ::memcpy(p, &w, sizeof(w));
+}
+
+void simdjson2msgpack::write_raw_string(
+    simdjson::ondemand::raw_json_string in) {
+  write_byte(0xdb);
+  uint8_t *location = skip_uint32();
+  std::string_view v = parser.unescape(in, buff);
+  write_uint32_at(uint32_t(v.size()), location);
+}
+
+void simdjson2msgpack::recursive_processor(simdjson::ondemand::value element) {
+  switch (element.type()) {
+  case simdjson::ondemand::json_type::array: {
+    uint32_t counter = 0;
+    write_byte(0xdd);
+    uint8_t *location = skip_uint32();
+    for (auto child : element.get_array()) {
+      counter++;
+      recursive_processor(child.value());
+    }
+    write_uint32_at(counter, location);
+  } break;
+  case simdjson::ondemand::json_type::object: {
+    uint32_t counter = 0;
+    write_byte(0xdf);
+    uint8_t *location = skip_uint32();
+    for (auto field : element.get_object()) {
+      counter++;
+      write_raw_string(field.key());
+      recursive_processor(field.value());
+    }
+    write_uint32_at(counter, location);
+  } break;
+  case simdjson::ondemand::json_type::number:
+    write_double(element.get_double());
+    break;
+  case simdjson::ondemand::json_type::string:
+    write_raw_string(element.get_raw_json_string());
+    break;
+  case simdjson::ondemand::json_type::boolean:
+    write_byte(0xc2 + element.get_bool());
+    break;
+  case simdjson::ondemand::json_type::null:
+    write_byte(0xc0);
+    break;
+  default:
+    SIMDJSON_UNREACHABLE();
+  }
+}
+
+struct simdjson_ondemand {
+  using StringType = std::string_view;
+
+  simdjson2msgpack parser{};
+
+  bool run(simdjson::padded_string &json, char *buffer,
+           std::string_view &result) {
+    result = parser.to_msgpack(json, reinterpret_cast<uint8_t *>(buffer));
+    return true;
+  }
+};
+
+BENCHMARK_TEMPLATE(json2msgpack, simdjson_ondemand)->UseManualTime();
+
+} // namespace json2msgpack
+
+#endif // SIMDJSON_EXCEPTIONS

benchmark/json2msgpack/yyjson.h

@@ -0,0 +1,122 @@
+#pragma once
+#ifdef SIMDJSON_COMPETITION_YYJSON
+
+#include "json2msgpack.h"
+
+namespace json2msgpack {
+
+struct yyjson2msgpack {
+  inline std::string_view to_msgpack(yyjson_doc *doc, uint8_t *buf);
+
+private:
+  inline void write_double(const double d) noexcept;
+  inline void write_byte(const uint8_t b) noexcept;
+  inline void write_uint32(const uint32_t w) noexcept;
+  inline void write_string(const char *s, size_t length) noexcept;
+  inline void recursive_processor(yyjson_val *obj);
+
+  uint8_t *buff{};
+};
+
+std::string_view yyjson2msgpack::to_msgpack(yyjson_doc *doc, uint8_t *buf) {
+  buff = buf;
+  yyjson_val *root = yyjson_doc_get_root(doc);
+  recursive_processor(root);
+  return std::string_view(reinterpret_cast<char *>(buf), size_t(buff - buf));
+}
+
+void yyjson2msgpack::write_string(const char *c, size_t len) noexcept {
+  write_byte(0xdb);
+  write_uint32(uint32_t(len));
+  ::memcpy(buff, c, len);
+  buff += len;
+}
+
+void yyjson2msgpack::write_double(const double d) noexcept {
+  *buff++ = 0xcb;
+  ::memcpy(buff, &d, sizeof(d));
+  buff += sizeof(d);
+}
+
+void yyjson2msgpack::write_byte(const uint8_t b) noexcept {
+  *buff = b;
+  buff++;
+}
+
+void yyjson2msgpack::write_uint32(const uint32_t w) noexcept {
+  ::memcpy(buff, &w, sizeof(w));
+  buff += sizeof(w);
+}
+
+void yyjson2msgpack::recursive_processor(yyjson_val *obj) {
+  size_t idx, max;
+  yyjson_val *val;
+  yyjson_val *key;
+  switch (yyjson_get_type(obj)) {
+  case YYJSON_TYPE_STR:
+    write_string(yyjson_get_str(obj), yyjson_get_len(obj));
+    break;
+  case YYJSON_TYPE_ARR:
+    write_byte(0xdf);
+    write_uint32(uint32_t(yyjson_arr_size(obj)));
+    yyjson_arr_foreach(obj, idx, max, val) { recursive_processor(val); }
+   break;
+  case YYJSON_TYPE_OBJ:
+    write_byte(0xdd);
+    write_uint32(uint32_t(yyjson_obj_size(obj)));
+    yyjson_obj_foreach(obj, idx, max, key, val) {
+      write_string(yyjson_get_str(key), yyjson_get_len(key));
+      recursive_processor(val);
+    }
+    break;
+  case YYJSON_TYPE_BOOL:
+    write_byte(0xc2 + yyjson_get_bool(obj));
+    break;
+  case YYJSON_TYPE_NULL:
+    write_byte(0xc0);
+    break;
+  case YYJSON_TYPE_NUM:
+    switch (yyjson_get_subtype(obj)) {
+    case YYJSON_SUBTYPE_UINT:
+      write_double(double(yyjson_get_uint(obj)));
+      break;
+    case YYJSON_SUBTYPE_SINT:
+      write_double(double(yyjson_get_sint(obj)));
+      break;
+    case YYJSON_SUBTYPE_REAL:
+      write_double(yyjson_get_real(obj));
+      break;
+    default:
+      SIMDJSON_UNREACHABLE();
+    }
+    break;
+  default:
+    SIMDJSON_UNREACHABLE();
+  }
+}
+
+struct yyjson : yyjson2msgpack {
+  bool run(simdjson::padded_string &json, char *buffer,
+           std::string_view &result) {
+    yyjson_doc *doc = yyjson_read(json.data(), json.size(), 0);
+    result = to_msgpack(doc, reinterpret_cast<uint8_t*>(buffer));
+    return true;
+  }
+};
+
+BENCHMARK_TEMPLATE(json2msgpack, yyjson)->UseManualTime();
+
+struct yyjson_insitu : yyjson2msgpack {
+  bool run(simdjson::padded_string &json, char *buffer,
+           std::string_view &result) {
+    yyjson_doc *doc =
+        yyjson_read_opts(json.data(), json.size(), YYJSON_READ_INSITU, 0, 0);
+    result = to_msgpack(doc, reinterpret_cast<uint8_t*>(buffer));
+    return true;
+  }
+};
+BENCHMARK_TEMPLATE(json2msgpack, yyjson_insitu)->UseManualTime();
+
+} // namespace json2msgpack
+
+#endif // SIMDJSON_COMPETITION_YYJSON

benchmark/kostya/nlohmann_json_sax.h

@@ -42,7 +42,7 @@ struct nlohmann_json_sax {
             return true;
         }
         bool number_unsigned(number_unsigned_t val) override {  // Need this event because coordinate value can be equal to 1
-            buffer[k] = val;
+            buffer[k] = double(val);
             if (k == 2) {
                 result.emplace_back(json_benchmark::point{buffer[0],buffer[1],buffer[2]});
                 k = 0;

benchmark/kostya/rapidjson.h

@@ -51,7 +51,7 @@ BENCHMARK_TEMPLATE(kostya, rapidjson_lossless)->UseManualTime();
 
 struct rapidjson_insitu : rapidjson_base {
   bool run(simdjson::padded_string &json, std::vector<point> &result) {
-    return rapidjson_base::run(doc.ParseInsitu<kParseValidateEncodingFlag>(json.data()), result);
+    return rapidjson_base::run(doc.ParseInsitu<kParseValidateEncodingFlag|kParseInsituFlag>(json.data()), result);
   }
 };
 BENCHMARK_TEMPLATE(kostya, rapidjson_insitu)->UseManualTime();

benchmark/kostya/sajson.h

@@ -11,6 +11,7 @@ struct sajson {
 
   size_t ast_buffer_size{0};
   size_t *ast_buffer{nullptr};
+  ~sajson() { free(ast_buffer); }
 
   simdjson_really_inline double get_double(const ::sajson::value &obj, std::string_view key) {
     using namespace sajson;

benchmark/large_random/nlohmann_json_sax.h

@@ -34,7 +34,7 @@ struct nlohmann_json_sax {
             return true;
         }
         bool number_unsigned(number_unsigned_t val) override {
-            buffer[k] = val;
+            buffer[k] = double(val);
             if (k == 2) {
                 result.emplace_back(json_benchmark::point{buffer[0],buffer[1],buffer[2]});
                 k = 0;

benchmark/large_random/rapidjson.h

@@ -48,7 +48,7 @@ BENCHMARK_TEMPLATE(large_random, rapidjson_lossless)->UseManualTime();
 
 struct rapidjson_insitu : rapidjson_base {
   bool run(simdjson::padded_string &json, std::vector<point> &result) {
-    return rapidjson_base::run(doc.ParseInsitu<kParseValidateEncodingFlag>(json.data()), result);
+    return rapidjson_base::run(doc.ParseInsitu<kParseValidateEncodingFlag|kParseInsituFlag>(json.data()), result);
   }
 };
 BENCHMARK_TEMPLATE(large_random, rapidjson_insitu)->UseManualTime();

benchmark/large_random/sajson.h

@@ -11,6 +11,7 @@ struct sajson {
 
   size_t ast_buffer_size{0};
   size_t *ast_buffer{nullptr};
+  ~sajson() { free(ast_buffer); }
 
   simdjson_really_inline double get_double(const ::sajson::value &obj, std::string_view key) {
     using namespace sajson;

benchmark/partial_tweets/rapidjson.h

@@ -70,7 +70,7 @@ BENCHMARK_TEMPLATE(partial_tweets, rapidjson)->UseManualTime();
 
 struct rapidjson_insitu : rapidjson_base {
   bool run(simdjson::padded_string &json, std::vector<tweet<std::string_view>> &result) {
-    return rapidjson_base::run(doc.ParseInsitu<kParseValidateEncodingFlag>(json.data()), result);
+    return rapidjson_base::run(doc.ParseInsitu<kParseValidateEncodingFlag|kParseInsituFlag>(json.data()), result);
   }
 };
 BENCHMARK_TEMPLATE(partial_tweets, rapidjson_insitu)->UseManualTime();

benchmark/partial_tweets/sajson.h

@@ -11,6 +11,8 @@ struct sajson {
 
   size_t ast_buffer_size{0};
   size_t *ast_buffer{nullptr};
+  ~sajson() { free(ast_buffer); }
+
   simdjson_really_inline std::string_view get_string_view(const ::sajson::value &obj, std::string_view key) {
     auto val = obj.get_value_of_key({key.data(), key.length()});
     if (val.get_type() != ::sajson::TYPE_STRING) { throw "field is not a string"; }

benchmark/top_tweet/nlohmann_json_sax.h

@@ -48,7 +48,7 @@ struct nlohmann_json_sax {
         }
         bool number_unsigned(number_unsigned_t val) override {
             if (values & key_rt && !(values & found_rt)) {   // retweet_count
-                rt = val;
+                rt = int(val);
                 values &= ~(key_rt);
                 values |= (found_rt);
                 if (rt <= max_rt && rt >= result.retweet_count) {    // Check if current tweet has more retweet than previous top tweet

benchmark/top_tweet/rapidjson.h

@@ -59,7 +59,7 @@ BENCHMARK_TEMPLATE(top_tweet, rapidjson)->UseManualTime();
 
 struct rapidjson_insitu : rapidjson_base {
   bool run(simdjson::padded_string &json, int64_t max_retweet_count, top_tweet_result<StringType> &result) {
-    return rapidjson_base::run(doc.ParseInsitu<kParseValidateEncodingFlag>(json.data()), max_retweet_count, result);
+    return rapidjson_base::run(doc.ParseInsitu<kParseValidateEncodingFlag|kParseInsituFlag>(json.data()), max_retweet_count, result);
   }
 };
 BENCHMARK_TEMPLATE(top_tweet, rapidjson_insitu)->UseManualTime();

benchmark/top_tweet/sajson.h

@@ -11,6 +11,7 @@ struct sajson {
 
   size_t ast_buffer_size{0};
   size_t *ast_buffer{nullptr};
+  ~sajson() { free(ast_buffer); }
 
   bool run(simdjson::padded_string &json, int32_t max_retweet_count, top_tweet_result<StringType> &result) {
     if (!ast_buffer) {

doc/basics.md

@@ -3,30 +3,35 @@ The Basics
 
 An overview of what you need to know to use simdjson, with examples.
 
-* [Requirements](#requirements)
-* [Including simdjson](#including-simdjson)
-* [Using simdjson with package managers](#using-simdjson-with-package-managers)
-* [Using simdjson as a CMake dependency](#using-simdjson-as-a-cmake-dependency)
-* [Versions](#versions)
-* [The Basics: Loading and Parsing JSON Documents](#the-basics-loading-and-parsing-json-documents)
-* [Documents are Iterators](#documents-are-iterators)
-* [C++11 Support and string_view](#c11-support-and-string_view)
-* [Using the Parsed JSON](#using-the-parsed-json)
-* [Minifying JSON strings without parsing](#minifying-json-strings-without-parsing)
-* [UTF-8 validation (alone)](#utf-8-validation-alone)
-* [JSON Pointer](#json-pointer)
-* [Error Handling](#error-handling)
-  * [Error Handling Example without Exceptions](#error-handling-examples-without-exceptions)
-  * [Disabling Exceptions](#disabling-exceptions)
-  * [Exceptions](#exceptions)
-  * [Current location in document](#current-location-in-document)
-* [Rewinding](#rewinding)
-* [Direct Access to the Raw String](#direct-access-to-the-raw-string)
-* [Newline-Delimited JSON (ndjson) and JSON lines](#newline-delimited-json-ndjson-and-json-lines)
-* [Parsing Numbers Inside Strings](#parsing-numbers-inside-strings)
-* [Dynamic Number Types](#dynamic-number-types)
-* [Thread Safety](#thread-safety)
-* [Standard Compliance](#standard-compliance)
+- [The Basics](#the-basics)
+  - [Requirements](#requirements)
+  - [Including simdjson](#including-simdjson)
+  - [Using simdjson with package managers](#using-simdjson-with-package-managers)
+  - [Using simdjson as a CMake dependency](#using-simdjson-as-a-cmake-dependency)
+  - [Versions](#versions)
+  - [The Basics: Loading and Parsing JSON Documents](#the-basics-loading-and-parsing-json-documents)
+  - [Documents are Iterators](#documents-are-iterators)
+    - [Parser, Document and JSON Scope](#parser-document-and-json-scope)
+  - [C++11 Support and string_view](#c11-support-and-string_view)
+  - [Using the Parsed JSON](#using-the-parsed-json)
+    - [Using the Parsed JSON: Additional examples](#using-the-parsed-json-additional-examples)
+  - [Minifying JSON strings without parsing](#minifying-json-strings-without-parsing)
+  - [UTF-8 validation (alone)](#utf-8-validation-alone)
+  - [JSON Pointer](#json-pointer)
+  - [Error Handling](#error-handling)
+    - [Error Handling Examples without Exceptions](#error-handling-examples-without-exceptions)
+    - [Disabling Exceptions](#disabling-exceptions)
+    - [Exceptions](#exceptions)
+    - [Current location in document](#current-location-in-document)
+  - [Rewinding](#rewinding)
+  - [Direct Access to the Raw String](#direct-access-to-the-raw-string)
+  - [Newline-Delimited JSON (ndjson) and JSON lines](#newline-delimited-json-ndjson-and-json-lines)
+  - [Parsing Numbers Inside Strings](#parsing-numbers-inside-strings)
+  - [Dynamic Number Types](#dynamic-number-types)
+  - [Raw Strings](#raw-strings)
+  - [Thread Safety](#thread-safety)
+  - [Standard Compliance](#standard-compliance)
+  - [Backwards Compatibility](#backwards-compatibility)
 
 
 Requirements
@@ -467,7 +472,7 @@ support for users who avoid exceptions. See [the simdjson error handling documen
   ```
 * **Tree Walking and JSON Element Types:** Sometimes you don't necessarily have a document
   with a known type, and are trying to generically inspect or walk over JSON elements. To do that, you can use iterators and the `type()` method. You can also represent arbitrary JSON values with
-  `ondemand::value` instances: it can represent anything except a scalar document (lone number, string, null or Boolean). You can check for scalar documents with the method `scalar()`.
+  `ondemand::value` instances: it can represent anything except a scalar document (lone number, string, null or Boolean). You can check for scalar documents with the method `scalar()`. You may also access [raw strings](#raw-strings).
   For example, the following is a quick and dirty recursive function that verbosely prints the JSON document as JSON. This example also illustrates lifecycle requirements: the `document` instance holds the iterator. The document must remain in scope while you are accessing instances of `value`, `object` and `array`.
   ```c++
   void recursive_print_json(ondemand::value element) {
@@ -1445,6 +1450,42 @@ It will output:
 9999999999999999999 negative: 0 is_integer: 1 large 64-bit integer: 9999999999999999999 large 64-bit integer: 9999999999999999999
 ```
 
+Raw Strings
+-----------
+
+It is sometimes useful to have access to a raw (unescaped) string: we make available a
+minimalist `raw_json_string` data type which contains a pointer inside the string in the
+original document, right after the quote. It is accessible via `get_raw_json_string()` on a
+string instance and returned by the `key()` method on an object's field instance. It is always
+optional: replacing `get_raw_json_string()` with `get_string()` and `key()` by
+`unescaped_key()` returns an `string_view` instance of the unescaped string.
+
+You can quickly compare a `raw_json_string` instance with a target string. You may also
+unescape  the `raw_json_string` on your own string buffer: `parser.unescape(mystr, ptr)`
+advances the provided pointer `ptr` and returns a string_view instance on the newly serialized
+string upon success, otherwise it returns an error. When unescaping to your own string buffer,
+you should ensure that you have sufficient memory space: the total size of the strings plus
+`simdjson::SIMDJSON_PADDING` bytes. The following example illustrates how we can unescape
+JSON string to a user-provided buffer:
+
+```C++
+    auto json = R"( {"name": "Jack The Ripper \u0033"} )"_padded;
+    // We create a buffer large enough to store all strings we need:
+    std::unique_ptr<uint8_t[]> buffer(new uint8_t[json.size() + simdjson::SIMDJSON_PADDING]);
+    uint8_t * ptr = buffer.get();
+    ondemand::parser parser;
+    ondemand::document doc = parser.iterate(json);
+    // We store our strings as 'string_view' instances in a vector:
+    std::vector<std::string_view> mystrings;
+    for (auto key_value : doc.get_object()) {
+      std::string_view keysv = parser.unescape(key_value.key(), ptr);// writes 'name'
+      mystrings.push_back(keysv);
+      std::string_view valuesv = parser.unescape(key_value.value().get_raw_json_string(), ptr);
+      // writes 'Jack The Ripper 3', escaping the \u0033
+      mystrings.push_back(valuesv);
+    }
+```
+
 Thread Safety
 -------------
 

doc/implementation-selection.md

@@ -17,6 +17,7 @@ different version of the JSON parser for different CPU architectures, often with
 algorithms to take better advantage of a given CPU!
 
 The current implementations are:
+* icelake: AVX-512F, AVX-512VBMI, etc.
 * haswell: AVX2 (2013 Intel Haswell or later)
 * westmere: SSE4.2 (2010 Westmere or later).
 * arm64: 64-bit ARMv8-A NEON
@@ -28,7 +29,7 @@ compiles *all* the implementations into the executable. On Intel, it will includ
 (haswell, westmere and fallback), on ARM it will include 2 (arm64 and fallback), and on PPC it will include 2 (ppc64 and fallback).
 
 If you know more about where you're going to run and want to save the space, you can disable any of
-these implementations at compile time with `-DSIMDJSON_IMPLEMENTATION_X=0` (where X is HASWELL,
+these implementations at compile time with `-DSIMDJSON_IMPLEMENTATION_X=0` (where X is ICELAKE, HASWELL,
 WESTMERE, ARM64, PPC64 and FALLBACK).
 
 The simdjson library automatically sets header flags for each implementation as it compiles; there

include/simdjson/arm64/intrinsics.h

@@ -5,4 +5,6 @@
 // you use visual studio or other compilers.
 #include <arm_neon.h>
 
+static_assert(sizeof(uint8x16_t) <= simdjson::SIMDJSON_PADDING, "insufficient padding for arm64");
+
 #endif //  SIMDJSON_ARM64_INTRINSICS_H

include/simdjson/common_defs.h

@@ -37,12 +37,12 @@ constexpr size_t SIMDJSON_MAXSIZE_BYTES = 0xFFFFFFFF;
 /**
  * The amount of padding needed in a buffer to parse JSON.
  *
- * the input buf should be readable up to buf + SIMDJSON_PADDING
+ * The input buf should be readable up to buf + SIMDJSON_PADDING
  * this is a stopgap; there should be a better description of the
  * main loop and its behavior that abstracts over this
  * See https://github.com/simdjson/simdjson/issues/174
  */
-constexpr size_t SIMDJSON_PADDING = 32;
+constexpr size_t SIMDJSON_PADDING = 64;
 
 /**
  * By default, simdjson supports this many nested objects and arrays.

include/simdjson/error.h

@@ -143,13 +143,6 @@ struct simdjson_result_base : protected std::pair<T, error_code> {
    */
   simdjson_really_inline error_code get(T &value) && noexcept;
 
-  /**
-   * Move the value to the provided variable.
-   *
-   * @param value The variable to assign the value to. May not be set if there is an error.
-   */
-  simdjson_really_inline const T &value(error_code &error) const & noexcept;
-
   /**
    * The error.
    */

include/simdjson/generic/dom_parser_implementation.h

@@ -35,6 +35,7 @@ public:
   simdjson_warn_unused error_code stage1(const uint8_t *buf, size_t len, stage1_mode partial) noexcept final;
   simdjson_warn_unused error_code stage2(dom::document &doc) noexcept final;
   simdjson_warn_unused error_code stage2_next(dom::document &doc) noexcept final;
+  simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst) const noexcept final;
   inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final;
   inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final;
 private:

include/simdjson/generic/ondemand/array-inl.h

@@ -89,6 +89,7 @@ simdjson_really_inline simdjson_result<std::string_view> array::raw_json() noexc
   return std::string_view(reinterpret_cast<const char*>(starting_point), size_t(final_point - starting_point));
 }
 
+SIMDJSON_PUSH_DISABLE_WARNINGS
 SIMDJSON_DISABLE_STRICT_OVERFLOW_WARNING
 simdjson_really_inline simdjson_result<size_t> array::count_elements() & noexcept {
   size_t count{0};
@@ -101,6 +102,7 @@ simdjson_really_inline simdjson_result<size_t> array::count_elements() & noexcep
   iter.reset_array();
   return count;
 }
+SIMDJSON_POP_DISABLE_WARNINGS
 
 simdjson_really_inline simdjson_result<bool> array::is_empty() & noexcept {
   bool is_not_empty;

include/simdjson/generic/ondemand/field-inl.h

@@ -23,7 +23,7 @@ simdjson_really_inline simdjson_result<field> field::start(const value_iterator
 
 simdjson_really_inline simdjson_warn_unused simdjson_result<std::string_view> field::unescaped_key() noexcept {
   SIMDJSON_ASSUME(first.buf != nullptr); // We would like to call .alive() but Visual Studio won't let us.
-  simdjson_result<std::string_view> answer = first.unescape(second.iter.string_buf_loc());
+  simdjson_result<std::string_view> answer = first.unescape(second.iter.json_iter());
   first.consume();
   return answer;
 }

include/simdjson/generic/ondemand/json_iterator-inl.h

@@ -312,6 +312,10 @@ simdjson_really_inline token_position json_iterator::position() const noexcept {
   return token.position();
 }
 
+simdjson_really_inline simdjson_result<std::string_view> json_iterator::unescape(raw_json_string in) noexcept {
+  return parser->unescape(in, _string_buf_loc);
+}
+
 simdjson_really_inline void json_iterator::reenter_child(token_position position, depth_t child_depth) noexcept {
   SIMDJSON_ASSUME(child_depth >= 1 && child_depth < INT32_MAX);
   SIMDJSON_ASSUME(_depth == child_depth - 1);

include/simdjson/generic/ondemand/json_iterator.h

@@ -233,7 +233,14 @@ public:
   template<int N> simdjson_warn_unused simdjson_really_inline bool copy_to_buffer(const uint8_t *json, uint32_t max_len, uint8_t (&tmpbuf)[N]) noexcept;
 
   simdjson_really_inline token_position position() const noexcept;
+  /**
+   * Write the raw_json_string to the string buffer and return a string_view.
+   * Each raw_json_string should be unescaped once, or else the string buffer might
+   * overflow.
+   */
+  simdjson_really_inline simdjson_result<std::string_view> unescape(raw_json_string in) noexcept;
   simdjson_really_inline void reenter_child(token_position position, depth_t child_depth) noexcept;
+
 #ifdef SIMDJSON_DEVELOPMENT_CHECKS
   simdjson_really_inline token_position start_position(depth_t depth) const noexcept;
   simdjson_really_inline void set_start_position(depth_t depth, token_position position) noexcept;

include/simdjson/generic/ondemand/object_iterator-inl.h

@@ -26,6 +26,8 @@ simdjson_really_inline bool object_iterator::operator!=(const object_iterator &)
   return iter.is_open();
 }
 
+SIMDJSON_PUSH_DISABLE_WARNINGS
+SIMDJSON_DISABLE_STRICT_OVERFLOW_WARNING
 simdjson_really_inline object_iterator &object_iterator::operator++() noexcept {
   // TODO this is a safety rail ... users should exit loops as soon as they receive an error.
   // Nonetheless, let's see if performance is OK with this if statement--the compiler may give it to us for free.
@@ -38,6 +40,7 @@ simdjson_really_inline object_iterator &object_iterator::operator++() noexcept {
   if ((error = iter.has_next_field().get(has_value) )) { return *this; };
   return *this;
 }
+SIMDJSON_POP_DISABLE_WARNINGS
 
 //
 // ### Live States

include/simdjson/generic/ondemand/parser-inl.h

@@ -117,6 +117,14 @@ simdjson_really_inline void parser::set_max_capacity(size_t max_capacity) noexce
   }
 }
 
+simdjson_really_inline simdjson_warn_unused simdjson_result<std::string_view> parser::unescape(raw_json_string in, uint8_t *&dst) const noexcept {
+  uint8_t *end = implementation->parse_string(in.buf, dst);
+  if (!end) { return STRING_ERROR; }
+  std::string_view result(reinterpret_cast<const char *>(dst), end-dst);
+  dst = end;
+  return result;
+}
+
 } // namespace ondemand
 } // namespace SIMDJSON_IMPLEMENTATION
 } // namespace simdjson

include/simdjson/generic/ondemand/parser.h

@@ -252,6 +252,27 @@ public:
   bool threaded{true};
   #endif
 
+  /**
+   * Unescape this JSON string, replacing \\ with \, \n with newline, etc. to a user-provided buffer.
+   * The provided pointer is advanced to the end of the string by reference, and a string_view instance
+   * is returned. You can ensure that your buffer is large enough by allocating a block of memory at least
+   * as large as the input JSON plus SIMDJSON_PADDING and then unescape all strings to this one buffer.
+   *
+   * This unescape function is a low-level function. If you want a more user-friendly approach, you should
+   * avoid raw_json_string instances (e.g., by calling unescaped_key() instead of key() or get_string()
+   * instead of get_raw_json_string()).
+   *
+   * ## IMPORTANT: string_view lifetime
+   *
+   * The string_view is only valid as long as the bytes in dst.
+   *
+   * @param raw_json_string input
+   * @param dst A pointer to a buffer at least large enough to write this string as well as
+   *            an additional SIMDJSON_PADDING bytes.
+   * @return A string_view pointing at the unescaped string in dst
+   * @error STRING_ERROR if escapes are incorrect.
+   */
+  simdjson_really_inline simdjson_result<std::string_view> unescape(raw_json_string in, uint8_t *&dst) const noexcept;
 private:
   /** @private [for benchmarking access] The implementation to use */
   std::unique_ptr<internal::dom_parser_implementation> implementation{};

include/simdjson/generic/ondemand/raw_json_string-inl.h

@@ -6,13 +6,7 @@ namespace ondemand {
 simdjson_really_inline raw_json_string::raw_json_string(const uint8_t * _buf) noexcept : buf{_buf} {}
 
 simdjson_really_inline const char * raw_json_string::raw() const noexcept { return reinterpret_cast<const char *>(buf); }
-simdjson_really_inline simdjson_warn_unused simdjson_result<std::string_view> raw_json_string::unescape(uint8_t *&dst) const noexcept {
-  uint8_t *end = stringparsing::parse_string(buf, dst);
-  if (!end) { return STRING_ERROR; }
-  std::string_view result(reinterpret_cast<const char *>(dst), end-dst);
-  dst = end;
-  return result;
-}
+
 
 simdjson_really_inline bool raw_json_string::is_free_from_unescaped_quote(std::string_view target) noexcept {
   size_t pos{0};
@@ -150,7 +144,7 @@ simdjson_unused simdjson_really_inline bool operator!=(std::string_view c, const
 
 
 simdjson_really_inline simdjson_warn_unused simdjson_result<std::string_view> raw_json_string::unescape(json_iterator &iter) const noexcept {
-  return unescape(iter.string_buf_loc());
+  return iter.unescape(*this);
 }
 
 
@@ -183,10 +177,6 @@ simdjson_really_inline simdjson_result<const char *> simdjson_result<SIMDJSON_IM
   if (error()) { return error(); }
   return first.raw();
 }
-simdjson_really_inline simdjson_warn_unused simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::raw_json_string>::unescape(uint8_t *&dst) const noexcept {
-  if (error()) { return error(); }
-  return first.unescape(dst);
-}
 simdjson_really_inline simdjson_warn_unused simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::raw_json_string>::unescape(SIMDJSON_IMPLEMENTATION::ondemand::json_iterator &iter) const noexcept {
   if (error()) { return error(); }
   return first.unescape(iter);

include/simdjson/generic/ondemand/raw_json_string.h

@@ -17,12 +17,19 @@ class json_iterator;
  *
  * This class is deliberately simplistic and has little functionality. You can
  * compare a raw_json_string instance with an unescaped C string, but
- * that is pretty much all you can do.
+ * that is nearly all you can do.
+ *
+ * The raw_json_string is unescaped. If you wish to write an unescaped version of it to your own
+ * buffer, you may do so using the parser.unescape(string, buff) method, using an ondemand::parser
+ * instance. Doing so requires you to have a sufficiently large buffer.
+ *
+ * The raw_json_string instances originate typically from field instance which in turn represent
+ * key-value pairs from object instances. From a field instance, you get the raw_json_string
+ * instance by calling key(). You can, if you want a more usable string_view instance, call
+ * the unescaped_key() method on the field instance. You may also create a raw_json_string from
+ * any other string value, with the value.get_raw_json_string() method. Again, you can get
+ * a more usable string_view instance by calling get_string().
  *
- * They originate typically from field instance which in turn represent key-value pairs from
- * object instances. From a field instance, you get the raw_json_string instance by calling key().
- * You can, if you want a more usable string_view instance, call the unescaped_key() method
- * on the field instance.
  */
 class raw_json_string {
 public:
@@ -130,20 +137,6 @@ private:
    */
   simdjson_really_inline simdjson_warn_unused bool alive() const noexcept { return buf != nullptr; }
 
-  /**
-   * Unescape this JSON string, replacing \\ with \, \n with newline, etc.
-   *
-   * ## IMPORTANT: string_view lifetime
-   *
-   * The string_view is only valid as long as the bytes in dst.
-   *
-   * @param dst A pointer to a buffer at least large enough to write this string as well as a \0.
-   *            dst will be updated to the next unused location (just after the \0 written out at
-   *            the end of this string).
-   * @return A string_view pointing at the unescaped string in dst
-   * @error STRING_ERROR if escapes are incorrect.
-   */
-  simdjson_really_inline simdjson_warn_unused simdjson_result<std::string_view> unescape(uint8_t *&dst) const noexcept;
   /**
    * Unescape this JSON string, replacing \\ with \, \n with newline, etc.
    *
@@ -158,6 +151,7 @@ private:
   const uint8_t * buf{};
   friend class object;
   friend class field;
+  friend class parser;
   friend struct simdjson_result<raw_json_string>;
 };
 
@@ -188,7 +182,6 @@ public:
   simdjson_really_inline ~simdjson_result() noexcept = default; ///< @private
 
   simdjson_really_inline simdjson_result<const char *> raw() const noexcept;
-  simdjson_really_inline simdjson_warn_unused simdjson_result<std::string_view> unescape(uint8_t *&dst) const noexcept;
   simdjson_really_inline simdjson_warn_unused simdjson_result<std::string_view> unescape(SIMDJSON_IMPLEMENTATION::ondemand::json_iterator &iter) const noexcept;
 };
 

include/simdjson/generic/ondemand/value_iterator-inl.h

@@ -180,6 +180,8 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator
   return false;
 }
 
+SIMDJSON_PUSH_DISABLE_WARNINGS
+SIMDJSON_DISABLE_STRICT_OVERFLOW_WARNING
 simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator::find_field_unordered_raw(const std::string_view key) noexcept {
   /**
    * When find_field_unordered_raw is called, we can either be pointing at the
@@ -367,6 +369,7 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator
   // never reach this point.
   return false;
 }
+SIMDJSON_POP_DISABLE_WARNINGS
 
 simdjson_warn_unused simdjson_really_inline simdjson_result<raw_json_string> value_iterator::field_key() noexcept {
   assert_at_next();
@@ -470,7 +473,7 @@ simdjson_really_inline bool value_iterator::parse_null(const uint8_t *json) cons
 }
 
 simdjson_warn_unused simdjson_really_inline simdjson_result<std::string_view> value_iterator::get_string() noexcept {
-  return get_raw_json_string().unescape(_json_iter->string_buf_loc());
+  return get_raw_json_string().unescape(json_iter());
 }
 simdjson_warn_unused simdjson_really_inline simdjson_result<raw_json_string> value_iterator::get_raw_json_string() noexcept {
   auto json = peek_scalar("string");

include/simdjson/generic/stringparsing.h

@@ -1,138 +0,0 @@
-// This file contains the common code every implementation uses
-// It is intended to be included multiple times and compiled multiple times
-
-namespace simdjson {
-namespace SIMDJSON_IMPLEMENTATION {
-namespace {
-/// @private
-namespace stringparsing {
-
-// begin copypasta
-// These chars yield themselves: " \ /
-// b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab
-// u not handled in this table as it's complex
-static const uint8_t escape_map[256] = {
-    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0, // 0x0.
-    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0,
-    0, 0, 0x22, 0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0x2f,
-    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0,
-
-    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0, // 0x4.
-    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0x5c, 0, 0,    0, // 0x5.
-    0, 0, 0x08, 0, 0,    0, 0x0c, 0, 0, 0, 0, 0, 0,    0, 0x0a, 0, // 0x6.
-    0, 0, 0x0d, 0, 0x09, 0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0, // 0x7.
-
-    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0,
-    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0,
-    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0,
-    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0,
-
-    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0,
-    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0,
-    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0,
-    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0,
-};
-
-// handle a unicode codepoint
-// write appropriate values into dest
-// src will advance 6 bytes or 12 bytes
-// dest will advance a variable amount (return via pointer)
-// return true if the unicode codepoint was valid
-// We work in little-endian then swap at write time
-simdjson_warn_unused
-simdjson_really_inline bool handle_unicode_codepoint(const uint8_t **src_ptr,
-                                            uint8_t **dst_ptr) {
-  // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the
-  // conversion isn't valid; we defer the check for this to inside the
-  // multilingual plane check
-  uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2);
-  *src_ptr += 6;
-  // check for low surrogate for characters outside the Basic
-  // Multilingual Plane.
-  if (code_point >= 0xd800 && code_point < 0xdc00) {
-    if (((*src_ptr)[0] != '\\') || (*src_ptr)[1] != 'u') {
-      return false;
-    }
-    uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2);
-
-    // if the first code point is invalid we will get here, as we will go past
-    // the check for being outside the Basic Multilingual plane. If we don't
-    // find a \u immediately afterwards we fail out anyhow, but if we do,
-    // this check catches both the case of the first code point being invalid
-    // or the second code point being invalid.
-    if ((code_point | code_point_2) >> 16) {
-      return false;
-    }
-
-    code_point =
-        (((code_point - 0xd800) << 10) | (code_point_2 - 0xdc00)) + 0x10000;
-    *src_ptr += 6;
-  }
-  size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr);
-  *dst_ptr += offset;
-  return offset > 0;
-}
-
-/**
- * Unescape a string from src to dst, stopping at a final unescaped quote. E.g., if src points at 'joe"', then
- * dst needs to have four free bytes.
- */
-simdjson_warn_unused simdjson_really_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst) {
-  while (1) {
-    // Copy the next n bytes, and find the backslash and quote in them.
-    auto bs_quote = backslash_and_quote::copy_and_find(src, dst);
-    // If the next thing is the end quote, copy and return
-    if (bs_quote.has_quote_first()) {
-      // we encountered quotes first. Move dst to point to quotes and exit
-      return dst + bs_quote.quote_index();
-    }
-    if (bs_quote.has_backslash()) {
-      /* find out where the backspace is */
-      auto bs_dist = bs_quote.backslash_index();
-      uint8_t escape_char = src[bs_dist + 1];
-      /* we encountered backslash first. Handle backslash */
-      if (escape_char == 'u') {
-        /* move src/dst up to the start; they will be further adjusted
-           within the unicode codepoint handling code. */
-        src += bs_dist;
-        dst += bs_dist;
-        if (!handle_unicode_codepoint(&src, &dst)) {
-          return nullptr;
-        }
-      } else {
-        /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and
-         * write bs_dist+1 characters to output
-         * note this may reach beyond the part of the buffer we've actually
-         * seen. I think this is ok */
-        uint8_t escape_result = escape_map[escape_char];
-        if (escape_result == 0u) {
-          return nullptr; /* bogus escape value is an error */
-        }
-        dst[bs_dist] = escape_result;
-        src += bs_dist + 2;
-        dst += bs_dist + 1;
-      }
-    } else {
-      /* they are the same. Since they can't co-occur, it means we
-       * encountered neither. */
-      src += backslash_and_quote::BYTES_PROCESSED;
-      dst += backslash_and_quote::BYTES_PROCESSED;
-    }
-  }
-  /* can't be reached */
-  return nullptr;
-}
-
-simdjson_unused simdjson_warn_unused simdjson_really_inline error_code parse_string_to_buffer(const uint8_t *src, uint8_t *&current_string_buf_loc, std::string_view &s) {
-  if (*(src++) != '"') { return STRING_ERROR; }
-  auto end = stringparsing::parse_string(src, current_string_buf_loc);
-  if (!end) { return STRING_ERROR; }
-  s = std::string_view(reinterpret_cast<const char *>(current_string_buf_loc), end-current_string_buf_loc);
-  current_string_buf_loc = end;
-  return SUCCESS;
-}
-
-} // namespace stringparsing
-} // unnamed namespace
-} // namespace SIMDJSON_IMPLEMENTATION
-} // namespace simdjson

include/simdjson/haswell/intrinsics.h

@@ -45,4 +45,6 @@
 #endif //  _blsr_u64
 #endif // SIMDJSON_CLANG_VISUAL_STUDIO
 
+static_assert(sizeof(__m256i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for haswell kernel.");
+
 #endif // SIMDJSON_HASWELL_INTRINSICS_H

include/simdjson/icelake/intrinsics.h

@@ -53,4 +53,6 @@
 #endif //  _blsr_u64
 #endif // SIMDJSON_CLANG_VISUAL_STUDIO
 
+static_assert(sizeof(__m512i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for icelake");
+
 #endif // SIMDJSON_ICELAKE_INTRINSICS_H

include/simdjson/implementation.h

@@ -18,8 +18,6 @@ namespace simdjson {
  * @return true if the string is valid UTF-8.
  */
 simdjson_warn_unused bool validate_utf8(const char * buf, size_t len) noexcept;
-
-
 /**
  * Validate the UTF-8 string.
  *

include/simdjson/internal/dom_parser_implementation.h

@@ -102,6 +102,22 @@ public:
    */
   simdjson_warn_unused virtual error_code stage2_next(dom::document &doc) noexcept = 0;
 
+  /**
+   * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There
+   * must be an unescaped quote terminating the string. It returns the final output
+   * position as pointer. In case of error (e.g., the string has bad escaped codes),
+   * then null_nullptrptr is returned. It is assumed that the output buffer is large
+   * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes +
+   * SIMDJSON_PADDING bytes.
+   *
+   * Overridden by each implementation.
+   *
+   * @param str pointer to the beginning of a valid UTF-8 JSON string, must end with an unescaped quote.
+   * @param dst pointer to a destination buffer, it must point a region in memory of sufficient size.
+   * @return end of the of the written region (exclusive) or nullptr in case of error.
+   */
+  simdjson_warn_unused virtual uint8_t *parse_string(const uint8_t *src, uint8_t *dst) const noexcept = 0;
+
   /**
    * Change the capacity of this parser.
    *
@@ -163,6 +179,7 @@ public:
    */
   simdjson_warn_unused inline error_code allocate(size_t capacity, size_t max_depth) noexcept;
 
+
 protected:
   /**
    * The maximum document length this parser supports.

include/simdjson/ppc64/intrinsics.h

@@ -16,4 +16,6 @@
 #undef vector
 #endif
 
+static_assert(sizeof(__vector unsigned char) <= simdjson::SIMDJSON_PADDING, "insufficient padding for ppc64");
+
 #endif //  SIMDJSON_PPC64_INTRINSICS_H

include/simdjson/simdjson_version.h

@@ -4,7 +4,7 @@
 #define SIMDJSON_SIMDJSON_VERSION_H
 
 /** The version of simdjson being used (major.minor.revision) */
-#define SIMDJSON_VERSION 2.0.4
+#define SIMDJSON_VERSION 2.2.0
 
 namespace simdjson {
 enum {
@@ -15,11 +15,11 @@ enum {
   /**
    * The minor version (major.MINOR.revision) of simdjson being used.
    */
-  SIMDJSON_VERSION_MINOR = 0,
+  SIMDJSON_VERSION_MINOR = 2,
   /**
    * The revision (major.minor.REVISION) of simdjson being used.
    */
-  SIMDJSON_VERSION_REVISION = 4
+  SIMDJSON_VERSION_REVISION = 0
 };
 } // namespace simdjson
 

include/simdjson/westmere/intrinsics.h

@@ -22,6 +22,6 @@
 #include <wmmintrin.h>  // for  _mm_clmulepi64_si128
 #endif
 
-
+static_assert(sizeof(__m128i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for westmere");
 
 #endif // SIMDJSON_WESTMERE_INTRINSICS_H

src/arm64/dom_parser_implementation.cpp

@@ -76,13 +76,13 @@ simdjson_really_inline json_character_block json_character_block::classify(const
 
 simdjson_really_inline bool is_ascii(const simd8x64<uint8_t>& input) {
     simd8<uint8_t> bits = input.reduce_or();
-    return bits.max_val() < 0b10000000u;
+    return bits.max_val() < 0x80u;
 }
 
 simdjson_unused simdjson_really_inline simd8<bool> must_be_continuation(const simd8<uint8_t> prev1, const simd8<uint8_t> prev2, const simd8<uint8_t> prev3) {
-    simd8<bool> is_second_byte = prev1 >= uint8_t(0b11000000u);
-    simd8<bool> is_third_byte  = prev2 >= uint8_t(0b11100000u);
-    simd8<bool> is_fourth_byte = prev3 >= uint8_t(0b11110000u);
+    simd8<bool> is_second_byte = prev1 >= uint8_t(0xc0u);
+    simd8<bool> is_third_byte  = prev2 >= uint8_t(0xe0u);
+    simd8<bool> is_fourth_byte = prev3 >= uint8_t(0xf0u);
     // Use ^ instead of | for is_*_byte, because ^ is commutative, and the caller is using ^ as well.
     // This will work fine because we only have to report errors for cases with 0-1 lead bytes.
     // Multiple lead bytes implies 2 overlapping multibyte characters, and if that happens, there is
@@ -92,8 +92,8 @@ simdjson_unused simdjson_really_inline simd8<bool> must_be_continuation(const si
 }
 
 simdjson_really_inline simd8<bool> must_be_2_3_continuation(const simd8<uint8_t> prev2, const simd8<uint8_t> prev3) {
-    simd8<bool> is_third_byte  = prev2 >= uint8_t(0b11100000u);
-    simd8<bool> is_fourth_byte = prev3 >= uint8_t(0b11110000u);
+    simd8<bool> is_third_byte  = prev2 >= uint8_t(0xe0u);
+    simd8<bool> is_fourth_byte = prev3 >= uint8_t(0xf0u);
     return is_third_byte ^ is_fourth_byte;
 }
 
@@ -109,6 +109,7 @@ simdjson_really_inline simd8<bool> must_be_2_3_continuation(const simd8<uint8_t>
 // Stage 2
 //
 
+#include "generic/stage2/stringparsing.h"
 #include "generic/stage2/tape_builder.h"
 
 //
@@ -151,6 +152,10 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu
   return stage2::tape_builder::parse_document<true>(*this, _doc);
 }
 
+simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst) const noexcept {
+  return arm64::stringparsing::parse_string(src, dst);
+}
+
 simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept {
   auto error = stage1(_buf, _len, stage1_mode::regular);
   if (error) { return error; }

src/fallback/dom_parser_implementation.cpp

@@ -27,12 +27,12 @@ simdjson_really_inline void add_structural() {
 }
 
 simdjson_really_inline bool is_continuation(uint8_t c) {
-  return (c & 0b11000000) == 0b10000000;
+  return (c & 0xc0) == 0x80;
 }
 
 simdjson_really_inline void validate_utf8_character() {
   // Continuation
-  if (simdjson_unlikely((buf[idx] & 0b01000000) == 0)) {
+  if (simdjson_unlikely((buf[idx] & 0x40) == 0)) {
     // extra continuation
     error = UTF8_ERROR;
     idx++;
@@ -40,7 +40,7 @@ simdjson_really_inline void validate_utf8_character() {
   }
 
   // 2-byte
-  if ((buf[idx] & 0b00100000) == 0) {
+  if ((buf[idx] & 0x20) == 0) {
     // missing continuation
     if (simdjson_unlikely(idx+1 > len || !is_continuation(buf[idx+1]))) {
       if (idx+1 > len && is_streaming(partial)) { idx = len; return; }
@@ -49,13 +49,13 @@ simdjson_really_inline void validate_utf8_character() {
       return;
     }
     // overlong: 1100000_ 10______
-    if (buf[idx] <= 0b11000001) { error = UTF8_ERROR; }
+    if (buf[idx] <= 0xc1) { error = UTF8_ERROR; }
     idx += 2;
     return;
   }
 
   // 3-byte
-  if ((buf[idx] & 0b00010000) == 0) {
+  if ((buf[idx] & 0x10) == 0) {
     // missing continuation
     if (simdjson_unlikely(idx+2 > len || !is_continuation(buf[idx+1]) || !is_continuation(buf[idx+2]))) {
       if (idx+2 > len && is_streaming(partial)) { idx = len; return; }
@@ -64,9 +64,9 @@ simdjson_really_inline void validate_utf8_character() {
       return;
     }
     // overlong: 11100000 100_____ ________
-    if (buf[idx] == 0b11100000 && buf[idx+1] <= 0b10011111) { error = UTF8_ERROR; }
+    if (buf[idx] == 0xe0 && buf[idx+1] <= 0x9f) { error = UTF8_ERROR; }
     // surrogates: U+D800-U+DFFF 11101101 101_____
-    if (buf[idx] == 0b11101101 && buf[idx+1] >= 0b10100000) { error = UTF8_ERROR; }
+    if (buf[idx] == 0xed && buf[idx+1] >= 0xa0) { error = UTF8_ERROR; }
     idx += 3;
     return;
   }
@@ -80,14 +80,14 @@ simdjson_really_inline void validate_utf8_character() {
     return;
   }
   // overlong: 11110000 1000____ ________ ________
-  if (buf[idx] == 0b11110000 && buf[idx+1] <= 0b10001111) { error = UTF8_ERROR; }
+  if (buf[idx] == 0xf0 && buf[idx+1] <= 0x8f) { error = UTF8_ERROR; }
   // too large: > U+10FFFF:
   // 11110100 (1001|101_)____
   // 1111(1___|011_|0101) 10______
   // also includes 5, 6, 7 and 8 byte characters:
   // 11111___
-  if (buf[idx] == 0b11110100 && buf[idx+1] >= 0b10010000) { error = UTF8_ERROR; }
-  if (buf[idx] >= 0b11110101) { error = UTF8_ERROR; }
+  if (buf[idx] == 0xf4 && buf[idx+1] >= 0x90) { error = UTF8_ERROR; }
+  if (buf[idx] >= 0xf5) { error = UTF8_ERROR; }
   idx += 4;
 }
 
@@ -97,7 +97,7 @@ simdjson_really_inline bool validate_string() {
   while (idx < len && buf[idx] != '"') {
     if (buf[idx] == '\\') {
       idx += 2;
-    } else if (simdjson_unlikely(buf[idx] & 0b10000000)) {
+    } else if (simdjson_unlikely(buf[idx] & 0x80)) {
       validate_utf8_character();
     } else {
       if (buf[idx] < 0x20) { error = UNESCAPED_CHARS; }
@@ -297,39 +297,39 @@ simdjson_warn_unused bool implementation::validate_utf8(const char *buf, size_t
       }
     }
     unsigned char byte = data[pos];
-    if (byte < 0b10000000) {
+    if (byte < 0x80) {
       pos++;
       continue;
-    } else if ((byte & 0b11100000) == 0b11000000) {
+    } else if ((byte & 0xe0) == 0xc0) {
       next_pos = pos + 2;
       if (next_pos > len) { return false; }
-      if ((data[pos + 1] & 0b11000000) != 0b10000000) { return false; }
+      if ((data[pos + 1] & 0xc0) != 0x80) { return false; }
       // range check
-      code_point = (byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111);
+      code_point = (byte & 0x1f) << 6 | (data[pos + 1] & 0x3f);
       if (code_point < 0x80 || 0x7ff < code_point) { return false; }
-    } else if ((byte & 0b11110000) == 0b11100000) {
+    } else if ((byte & 0xf0) == 0xe0) {
       next_pos = pos + 3;
       if (next_pos > len) { return false; }
-      if ((data[pos + 1] & 0b11000000) != 0b10000000) { return false; }
-      if ((data[pos + 2] & 0b11000000) != 0b10000000) { return false; }
+      if ((data[pos + 1] & 0xc0) != 0x80) { return false; }
+      if ((data[pos + 2] & 0xc0) != 0x80) { return false; }
       // range check
-      code_point = (byte & 0b00001111) << 12 |
-                   (data[pos + 1] & 0b00111111) << 6 |
-                   (data[pos + 2] & 0b00111111);
+      code_point = (byte & 0x0f) << 12 |
+                   (data[pos + 1] & 0x3f) << 6 |
+                   (data[pos + 2] & 0x3f);
       if (code_point < 0x800 || 0xffff < code_point ||
           (0xd7ff < code_point && code_point < 0xe000)) {
         return false;
       }
-    } else if ((byte & 0b11111000) == 0b11110000) { // 0b11110000
+    } else if ((byte & 0xf8) == 0xf0) { // 0b11110000
       next_pos = pos + 4;
       if (next_pos > len) { return false; }
-      if ((data[pos + 1] & 0b11000000) != 0b10000000) { return false; }
-      if ((data[pos + 2] & 0b11000000) != 0b10000000) { return false; }
-      if ((data[pos + 3] & 0b11000000) != 0b10000000) { return false; }
+      if ((data[pos + 1] & 0xc0) != 0x80) { return false; }
+      if ((data[pos + 2] & 0xc0) != 0x80) { return false; }
+      if ((data[pos + 3] & 0xc0) != 0x80) { return false; }
       // range check
       code_point =
-          (byte & 0b00000111) << 18 | (data[pos + 1] & 0b00111111) << 12 |
-          (data[pos + 2] & 0b00111111) << 6 | (data[pos + 3] & 0b00111111);
+          (byte & 0x07) << 18 | (data[pos + 1] & 0x3f) << 12 |
+          (data[pos + 2] & 0x3f) << 6 | (data[pos + 3] & 0x3f);
       if (code_point <= 0xffff || 0x10ffff < code_point) { return false; }
     } else {
       // we may have a continuation
@@ -346,6 +346,7 @@ simdjson_warn_unused bool implementation::validate_utf8(const char *buf, size_t
 //
 // Stage 2
 //
+#include "generic/stage2/stringparsing.h"
 #include "generic/stage2/tape_builder.h"
 
 namespace simdjson {
@@ -359,6 +360,10 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu
   return stage2::tape_builder::parse_document<true>(*this, _doc);
 }
 
+simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst) const noexcept {
+  return fallback::stringparsing::parse_string(src, dst);
+}
+
 simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept {
   auto error = stage1(_buf, _len, stage1_mode::regular);
   if (error) { return error; }

src/generic/stage1/json_structural_indexer.h

@@ -160,19 +160,19 @@ simdjson_really_inline size_t trim_partial_utf8(const uint8_t *buf, size_t len)
   if (simdjson_unlikely(len < 3)) {
     switch (len) {
       case 2:
-        if (buf[len-1] >= 0b11000000) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left
-        if (buf[len-2] >= 0b11100000) { return len-2; } // 3- and 4-byte characters with only 2 bytes left
+        if (buf[len-1] >= 0xc0) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left
+        if (buf[len-2] >= 0xe0) { return len-2; } // 3- and 4-byte characters with only 2 bytes left
         return len;
       case 1:
-        if (buf[len-1] >= 0b11000000) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left
+        if (buf[len-1] >= 0xc0) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left
         return len;
       case 0:
         return len;
     }
   }
-  if (buf[len-1] >= 0b11000000) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left
-  if (buf[len-2] >= 0b11100000) { return len-2; } // 3- and 4-byte characters with only 1 byte left
-  if (buf[len-3] >= 0b11110000) { return len-3; } // 4-byte characters with only 3 bytes left
+  if (buf[len-1] >= 0xc0) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left
+  if (buf[len-2] >= 0xe0) { return len-2; } // 3- and 4-byte characters with only 1 byte left
+  if (buf[len-3] >= 0xf0) { return len-3; } // 4-byte characters with only 3 bytes left
   return len;
 }
 

src/generic/stage1/utf8_lookup4_algorithm.h

@@ -120,14 +120,14 @@ using namespace simd;
       255, 255, 255, 255, 255, 255, 255, 255,
       255, 255, 255, 255, 255, 255, 255, 255,
       255, 255, 255, 255, 255, 255, 255, 255,
-      255, 255, 255, 255, 255, 0b11110000u-1, 0b11100000u-1, 0b11000000u-1
+      255, 255, 255, 255, 255, 0xf0u-1, 0xe0u-1, 0xc0u-1
     };
 #else
     static const uint8_t max_array[32] = {
       255, 255, 255, 255, 255, 255, 255, 255,
       255, 255, 255, 255, 255, 255, 255, 255,
       255, 255, 255, 255, 255, 255, 255, 255,
-      255, 255, 255, 255, 255, 0b11110000u-1, 0b11100000u-1, 0b11000000u-1
+      255, 255, 255, 255, 255, 0xf0u-1, 0xe0u-1, 0xc0u-1
     };
 #endif
     const simd8<uint8_t> max_value(&max_array[sizeof(max_array)-sizeof(simd8<uint8_t>)]);

src/generic/stage2/stringparsing.h

@@ -0,0 +1,133 @@
+// This file contains the common code every implementation uses
+// It is intended to be included multiple times and compiled multiple times
+
+namespace simdjson {
+namespace SIMDJSON_IMPLEMENTATION {
+namespace {
+/// @private
+namespace stringparsing {
+
+// begin copypasta
+// These chars yield themselves: " \ /
+// b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab
+// u not handled in this table as it's complex
+static const uint8_t escape_map[256] = {
+    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0, // 0x0.
+    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0,
+    0, 0, 0x22, 0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0x2f,
+    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0,
+
+    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0, // 0x4.
+    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0x5c, 0, 0,    0, // 0x5.
+    0, 0, 0x08, 0, 0,    0, 0x0c, 0, 0, 0, 0, 0, 0,    0, 0x0a, 0, // 0x6.
+    0, 0, 0x0d, 0, 0x09, 0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0, // 0x7.
+
+    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0,
+    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0,
+    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0,
+    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0,
+
+    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0,
+    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0,
+    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0,
+    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0,
+};
+
+// handle a unicode codepoint
+// write appropriate values into dest
+// src will advance 6 bytes or 12 bytes
+// dest will advance a variable amount (return via pointer)
+// return true if the unicode codepoint was valid
+// We work in little-endian then swap at write time
+simdjson_warn_unused
+simdjson_really_inline bool handle_unicode_codepoint(const uint8_t **src_ptr,
+                                            uint8_t **dst_ptr) {
+  // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the
+  // conversion isn't valid; we defer the check for this to inside the
+  // multilingual plane check
+  uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2);
+  *src_ptr += 6;
+  // check for low surrogate for characters outside the Basic
+  // Multilingual Plane.
+  if (code_point >= 0xd800 && code_point < 0xdc00) {
+    if (((*src_ptr)[0] != '\\') || (*src_ptr)[1] != 'u') {
+      return false;
+    }
+    uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2);
+
+    // if the first code point is invalid we will get here, as we will go past
+    // the check for being outside the Basic Multilingual plane. If we don't
+    // find a \u immediately afterwards we fail out anyhow, but if we do,
+    // this check catches both the case of the first code point being invalid
+    // or the second code point being invalid.
+    if ((code_point | code_point_2) >> 16) {
+      return false;
+    }
+
+    code_point =
+        (((code_point - 0xd800) << 10) | (code_point_2 - 0xdc00)) + 0x10000;
+    *src_ptr += 6;
+  }
+  size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr);
+  *dst_ptr += offset;
+  return offset > 0;
+}
+
+/**
+ * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There
+ * must be an unescaped quote terminating the string. It returns the final output
+ * position as pointer. In case of error (e.g., the string has bad escaped codes),
+ * then null_nullptrptr is returned. It is assumed that the output buffer is large
+ * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes +
+ * SIMDJSON_PADDING bytes.
+ */
+simdjson_warn_unused simdjson_really_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst) {
+  while (1) {
+    // Copy the next n bytes, and find the backslash and quote in them.
+    auto bs_quote = backslash_and_quote::copy_and_find(src, dst);
+    // If the next thing is the end quote, copy and return
+    if (bs_quote.has_quote_first()) {
+      // we encountered quotes first. Move dst to point to quotes and exit
+      return dst + bs_quote.quote_index();
+    }
+    if (bs_quote.has_backslash()) {
+      /* find out where the backspace is */
+      auto bs_dist = bs_quote.backslash_index();
+      uint8_t escape_char = src[bs_dist + 1];
+      /* we encountered backslash first. Handle backslash */
+      if (escape_char == 'u') {
+        /* move src/dst up to the start; they will be further adjusted
+           within the unicode codepoint handling code. */
+        src += bs_dist;
+        dst += bs_dist;
+        if (!handle_unicode_codepoint(&src, &dst)) {
+          return nullptr;
+        }
+      } else {
+        /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and
+         * write bs_dist+1 characters to output
+         * note this may reach beyond the part of the buffer we've actually
+         * seen. I think this is ok */
+        uint8_t escape_result = escape_map[escape_char];
+        if (escape_result == 0u) {
+          return nullptr; /* bogus escape value is an error */
+        }
+        dst[bs_dist] = escape_result;
+        src += bs_dist + 2;
+        dst += bs_dist + 1;
+      }
+    } else {
+      /* they are the same. Since they can't co-occur, it means we
+       * encountered neither. */
+      src += backslash_and_quote::BYTES_PROCESSED;
+      dst += backslash_and_quote::BYTES_PROCESSED;
+    }
+  }
+  /* can't be reached */
+  return nullptr;
+}
+
+} // namespace stringparsing
+} // unnamed namespace
+} // namespace SIMDJSON_IMPLEMENTATION
+} // namespace simdjson

src/haswell/dom_parser_implementation.cpp

@@ -87,16 +87,16 @@ simdjson_really_inline bool is_ascii(const simd8x64<uint8_t>& input) {
 }
 
 simdjson_unused simdjson_really_inline simd8<bool> must_be_continuation(const simd8<uint8_t> prev1, const simd8<uint8_t> prev2, const simd8<uint8_t> prev3) {
-  simd8<uint8_t> is_second_byte = prev1.saturating_sub(0b11000000u-1); // Only 11______ will be > 0
-  simd8<uint8_t> is_third_byte  = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0
-  simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0
+  simd8<uint8_t> is_second_byte = prev1.saturating_sub(0xc0u-1); // Only 11______ will be > 0
+  simd8<uint8_t> is_third_byte  = prev2.saturating_sub(0xe0u-1); // Only 111_____ will be > 0
+  simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0xf0u-1); // Only 1111____ will be > 0
   // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine.
   return simd8<int8_t>(is_second_byte | is_third_byte | is_fourth_byte) > int8_t(0);
 }
 
 simdjson_really_inline simd8<bool> must_be_2_3_continuation(const simd8<uint8_t> prev2, const simd8<uint8_t> prev3) {
-  simd8<uint8_t> is_third_byte  = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0
-  simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0
+  simd8<uint8_t> is_third_byte  = prev2.saturating_sub(0xe0u-1); // Only 111_____ will be > 0
+  simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0xf0u-1); // Only 1111____ will be > 0
   // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine.
   return simd8<int8_t>(is_third_byte | is_fourth_byte) > int8_t(0);
 }
@@ -112,6 +112,7 @@ simdjson_really_inline simd8<bool> must_be_2_3_continuation(const simd8<uint8_t>
 //
 // Stage 2
 //
+#include "generic/stage2/stringparsing.h"
 #include "generic/stage2/tape_builder.h"
 
 //
@@ -152,6 +153,10 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu
   return stage2::tape_builder::parse_document<true>(*this, _doc);
 }
 
+simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst) const noexcept {
+  return haswell::stringparsing::parse_string(src, dst);
+}
+
 simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept {
   auto error = stage1(_buf, _len, stage1_mode::regular);
   if (error) { return error; }

src/icelake/dom_parser_implementation.cpp

@@ -84,16 +84,16 @@ simdjson_really_inline bool is_ascii(const simd8x64<uint8_t>& input) {
 }
 
 simdjson_unused simdjson_really_inline simd8<bool> must_be_continuation(const simd8<uint8_t> prev1, const simd8<uint8_t> prev2, const simd8<uint8_t> prev3) {
-  simd8<uint8_t> is_second_byte = prev1.saturating_sub(0b11000000u-1); // Only 11______ will be > 0
-  simd8<uint8_t> is_third_byte  = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0
-  simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0
+  simd8<uint8_t> is_second_byte = prev1.saturating_sub(0xc0u-1); // Only 11______ will be > 0
+  simd8<uint8_t> is_third_byte  = prev2.saturating_sub(0xe0u-1); // Only 111_____ will be > 0
+  simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0xf0u-1); // Only 1111____ will be > 0
   // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine.
   return simd8<int8_t>(is_second_byte | is_third_byte | is_fourth_byte) > int8_t(0);
 }
 
 simdjson_really_inline simd8<bool> must_be_2_3_continuation(const simd8<uint8_t> prev2, const simd8<uint8_t> prev3) {
-  simd8<uint8_t> is_third_byte  = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0
-  simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0
+  simd8<uint8_t> is_third_byte  = prev2.saturating_sub(0xe0u-1); // Only 111_____ will be > 0
+  simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0xf0u-1); // Only 1111____ will be > 0
   // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine.
   return simd8<int8_t>(is_third_byte | is_fourth_byte) > int8_t(0);
 }
@@ -158,6 +158,7 @@ SIMDJSON_POP_DISABLE_WARNINGS
 //
 // Stage 2
 //
+#include "generic/stage2/stringparsing.h"
 #include "generic/stage2/tape_builder.h"
 
 //
@@ -198,6 +199,10 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu
   return stage2::tape_builder::parse_document<true>(*this, _doc);
 }
 
+simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst) const noexcept {
+  return icelake::stringparsing::parse_string(src, dst);
+}
+
 simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept {
   auto error = stage1(_buf, _len, stage1_mode::regular);
   if (error) { return error; }

src/implementation.cpp

@@ -188,7 +188,6 @@ simdjson_warn_unused error_code minify(const char *buf, size_t len, char *dst, s
 simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) noexcept {
   return get_active_implementation()->validate_utf8(buf, len);
 }
-
 const implementation * builtin_implementation() {
   static const implementation * builtin_impl = get_available_implementations()[SIMDJSON_STRINGIFY(SIMDJSON_BUILTIN_IMPLEMENTATION)];
   assert(builtin_impl);

src/ppc64/dom_parser_implementation.cpp

@@ -50,20 +50,20 @@ simdjson_really_inline json_character_block json_character_block::classify(const
 
 simdjson_really_inline bool is_ascii(const simd8x64<uint8_t>& input) {
   // careful: 0x80 is not ascii.
-  return input.reduce_or().saturating_sub(0b01111111u).bits_not_set_anywhere();
+  return input.reduce_or().saturating_sub(0x7fu).bits_not_set_anywhere();
 }
 
 simdjson_unused simdjson_really_inline simd8<bool> must_be_continuation(const simd8<uint8_t> prev1, const simd8<uint8_t> prev2, const simd8<uint8_t> prev3) {
-  simd8<uint8_t> is_second_byte = prev1.saturating_sub(0b11000000u-1); // Only 11______ will be > 0
-  simd8<uint8_t> is_third_byte  = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0
-  simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0
+  simd8<uint8_t> is_second_byte = prev1.saturating_sub(0xc0u-1); // Only 11______ will be > 0
+  simd8<uint8_t> is_third_byte  = prev2.saturating_sub(0xe0u-1); // Only 111_____ will be > 0
+  simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0xf0u-1); // Only 1111____ will be > 0
   // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine.
   return simd8<int8_t>(is_second_byte | is_third_byte | is_fourth_byte) > int8_t(0);
 }
 
 simdjson_really_inline simd8<bool> must_be_2_3_continuation(const simd8<uint8_t> prev2, const simd8<uint8_t> prev3) {
-  simd8<uint8_t> is_third_byte  = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0
-  simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0
+  simd8<uint8_t> is_third_byte  = prev2.saturating_sub(0xe0u-1); // Only 111_____ will be > 0
+  simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0xf0u-1); // Only 1111____ will be > 0
   // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine.
   return simd8<int8_t>(is_third_byte | is_fourth_byte) > int8_t(0);
 }
@@ -79,7 +79,7 @@ simdjson_really_inline simd8<bool> must_be_2_3_continuation(const simd8<uint8_t>
 //
 // Stage 2
 //
-
+#include "generic/stage2/stringparsing.h"
 #include "generic/stage2/tape_builder.h"
 
 //
@@ -122,6 +122,10 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu
   return stage2::tape_builder::parse_document<true>(*this, _doc);
 }
 
+simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst) const noexcept {
+  return ppc64::stringparsing::parse_string(src, dst);
+}
+
 simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept {
   auto error = stage1(_buf, _len, stage1_mode::regular);
   if (error) { return error; }

src/westmere/dom_parser_implementation.cpp

@@ -85,16 +85,16 @@ simdjson_really_inline bool is_ascii(const simd8x64<uint8_t>& input) {
 }
 
 simdjson_unused simdjson_really_inline simd8<bool> must_be_continuation(const simd8<uint8_t> prev1, const simd8<uint8_t> prev2, const simd8<uint8_t> prev3) {
-  simd8<uint8_t> is_second_byte = prev1.saturating_sub(0b11000000u-1); // Only 11______ will be > 0
-  simd8<uint8_t> is_third_byte  = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0
-  simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0
+  simd8<uint8_t> is_second_byte = prev1.saturating_sub(0xc0u-1); // Only 11______ will be > 0
+  simd8<uint8_t> is_third_byte  = prev2.saturating_sub(0xe0u-1); // Only 111_____ will be > 0
+  simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0xf0u-1); // Only 1111____ will be > 0
   // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine.
   return simd8<int8_t>(is_second_byte | is_third_byte | is_fourth_byte) > int8_t(0);
 }
 
 simdjson_really_inline simd8<bool> must_be_2_3_continuation(const simd8<uint8_t> prev2, const simd8<uint8_t> prev3) {
-  simd8<uint8_t> is_third_byte  = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0
-  simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0
+  simd8<uint8_t> is_third_byte  = prev2.saturating_sub(0xe0u-1); // Only 111_____ will be > 0
+  simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0xf0u-1); // Only 1111____ will be > 0
   // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine.
   return simd8<int8_t>(is_third_byte | is_fourth_byte) > int8_t(0);
 }
@@ -110,6 +110,7 @@ simdjson_really_inline simd8<bool> must_be_2_3_continuation(const simd8<uint8_t>
 //
 // Stage 2
 //
+#include "generic/stage2/stringparsing.h"
 #include "generic/stage2/tape_builder.h"
 
 //
@@ -151,6 +152,10 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu
   return stage2::tape_builder::parse_document<true>(*this, _doc);
 }
 
+simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst) const noexcept {
+  return westmere::stringparsing::parse_string(src, dst);
+}
+
 simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept {
   auto error = stage1(_buf, _len, stage1_mode::regular);
   if (error) { return error; }

tests/dom/errortests.cpp

@@ -146,7 +146,7 @@ namespace parser_load {
 }
 
 namespace adversarial {
-  #define PADDING_FILLED_WITH_NUMBERS "222222222222222222222222222222222"
+  #define PADDING_FILLED_WITH_NUMBERS "22222222222222222222222222222222222222222222222222222222222222222"
   bool number_overrun_at_root() {
     TEST_START();
     constexpr const char *json = "1" PADDING_FILLED_WITH_NUMBERS ",";
@@ -179,8 +179,9 @@ namespace adversarial {
     TEST_SUCCEED();
   }
   bool run() {
-    static_assert(33 > SIMDJSON_PADDING, "corruption test doesn't have enough padding"); // 33 = std::strlen(PADDING_FILLED_WITH_NUMBERS)
-    return true
+    constexpr size_t filler_size = 65;
+    static_assert(filler_size > SIMDJSON_PADDING, "corruption test doesn't have enough padding"); // 33 = std::strlen(PADDING_FILLED_WITH_NUMBERS)
+    return (std::strlen(PADDING_FILLED_WITH_NUMBERS) == filler_size)
       && number_overrun_at_root()
       && number_overrun_in_array()
       && number_overrun_in_object()

tests/ondemand/ondemand_parse_api_tests.cpp

@@ -46,8 +46,8 @@ namespace parse_api_tests {
   bool parser_iterate_padded() {
     TEST_START();
     ondemand::parser parser;
-    const char json_str[] = "12\0                              "; // 32 padding
-    ASSERT_EQUAL(sizeof(json_str), 34);
+    const char json_str[] = "12\0                                                              ";// 64 bytes of padding
+    ASSERT_EQUAL(sizeof(json_str), 66);
     ASSERT_EQUAL(strlen(json_str), 2);
 
     {
@@ -84,8 +84,8 @@ namespace parse_api_tests {
   bool parser_iterate_padded_string_view() {
     TEST_START();
     ondemand::parser parser;
-    const char json_str[] = "12\0                              "; // 32 padding
-    ASSERT_EQUAL(sizeof(json_str), 34);
+    const char json_str[] = "12\0                                                              "; // 64 bytes of padding
+    ASSERT_EQUAL(sizeof(json_str), 66);
     ASSERT_EQUAL(strlen(json_str), 2);
 
     {
@@ -122,14 +122,14 @@ namespace parse_api_tests {
   bool parser_iterate_insufficient_padding() {
     TEST_START();
     ondemand::parser parser;
-    constexpr char json_str[] = "12\0                             "; // 31 padding
-    ASSERT_EQUAL(sizeof(json_str), 33);
+    constexpr char json_str[] = "12\0                                                             "; // 63 bytes of padding
+    ASSERT_EQUAL(sizeof(json_str), 65);
     ASSERT_EQUAL(strlen(json_str), 2);
-    ASSERT_EQUAL(padded_string_view(json_str, strlen(json_str), sizeof(json_str)).padding(), 31);
-    ASSERT_EQUAL(SIMDJSON_PADDING, 32);
+    ASSERT_EQUAL(padded_string_view(json_str, strlen(json_str), sizeof(json_str)).padding(), 63);
+    ASSERT_EQUAL(SIMDJSON_PADDING, 64);
 
     {
-      cout << "- char*, 31 padding" << endl;
+      cout << "- char*, 63 padding" << endl;
       ASSERT_ERROR( parser.iterate(json_str, strlen(json_str), sizeof(json_str)), INSUFFICIENT_PADDING );
       cout << "- char*, 0 padding" << endl;
       ASSERT_ERROR( parser.iterate(json_str, strlen(json_str), strlen(json_str)), INSUFFICIENT_PADDING );
@@ -137,7 +137,7 @@ namespace parse_api_tests {
 
     {
       std::string_view json(json_str);
-      cout << "- string_view, 31 padding" << endl;
+      cout << "- string_view, 63 padding" << endl;
       ASSERT_ERROR( parser.iterate(json, sizeof(json_str)), INSUFFICIENT_PADDING );
       cout << "- string_view, 0 padding" << endl;
       ASSERT_ERROR( parser.iterate(json, strlen(json_str)), INSUFFICIENT_PADDING );

tests/ondemand/ondemand_readme_examples.cpp

@@ -39,7 +39,6 @@ using error_code=simdjson::error_code;
       }
     }
     return true;
-
   }
 
 void recursive_print_json(ondemand::value element) {
@@ -792,6 +791,29 @@ bool simple_error_example() {
 
 
 #if SIMDJSON_EXCEPTIONS
+  bool raw_string() {
+    TEST_START();
+    auto json = R"( {"name": "Jack The Ripper \u0033"} )"_padded;
+    // We create a buffer large enough to store all strings we need:
+    std::unique_ptr<uint8_t[]> buffer(new uint8_t[json.size() + simdjson::SIMDJSON_PADDING]);
+    uint8_t * ptr = buffer.get();
+    ondemand::parser parser;
+    ondemand::document doc = parser.iterate(json);
+    // We store our strings as 'string_view' instances in a vector:
+    std::vector<std::string_view> mystrings;
+    for (auto key_value : doc.get_object()) {
+      std::string_view keysv = parser.unescape(key_value.key(), ptr);// writes 'name'
+      mystrings.push_back(keysv);
+      std::string_view valuesv = parser.unescape(key_value.value().get_raw_json_string(), ptr);
+      // writes 'Jack The Ripper 3', escaping the \u0033
+      mystrings.push_back(valuesv);
+    }
+    ASSERT_EQUAL(mystrings[0],"name");
+    ASSERT_EQUAL(mystrings[1],"Jack The Ripper 3");
+    TEST_SUCCEED();
+  }
+
+
   bool simple_error_example_except() {
     TEST_START();
     ondemand::parser parser;
@@ -800,30 +822,34 @@ bool simple_error_example() {
       ondemand::document doc = parser.iterate(json);
       double x = doc["bad number"].get_double();
       std::cout << "Got " << x << std::endl;
-      return true;
+      TEST_SUCCEED();
     } catch(simdjson_error& e) {
       // e.error() == NUMBER_ERROR
       std::cout << e.error() << std::endl;
-      return false;
+      TEST_FAIL("I did not expect an exception");
     }
   }
 
   int64_t current_location_tape_error_with_except() {
+    TEST_START();
     auto broken_json = R"( {"double": 13.06, false, "integer": -343} )"_padded;
     ondemand::parser parser;
-    ondemand::document doc = parser.iterate(broken_json);
+    ondemand::document doc;
     try {
+      doc = parser.iterate(broken_json);
       return int64_t(doc["integer"]);
     } catch(simdjson_error& err) {
-      std::cerr << err.error() << std::endl;
-      std::cerr << doc.current_location() << std::endl;
-      return -1;
+      std::cout << err.error() << std::endl;
+      std::cout << doc.current_location() << std::endl;
+      TEST_SUCCEED();
     }
+    TEST_FAIL("I expected an exception!");
   }
 
 #endif
 
 int load_example() {
+  TEST_START();
   simdjson::ondemand::parser parser;
   simdjson::ondemand::document tweets;
   padded_string json;
@@ -872,6 +898,7 @@ int example_1() {
 }
 #if SIMDJSON_EXCEPTIONS
 int load_example_except() {
+  TEST_START();
   simdjson::ondemand::parser parser;
   padded_string json = padded_string::load("twitter.json");
   simdjson::ondemand::document tweets = parser.iterate(json);
@@ -958,15 +985,12 @@ bool current_location_no_error() {
   TEST_SUCCEED();
 }
 
-int main() {
-#if SIMDJSON_EXCEPTIONS
-  basics_treewalk();
-  basics_treewalk_breakline();
-#endif
-  if (
-    true
+bool run() {
+  return true
 #if SIMDJSON_EXCEPTIONS
 //    && basics_1() // Fails because twitter.json isn't in current directory. Compile test only.
+    &&  basics_treewalk()
+    &&  basics_treewalk_breakline()
     && json_value_with_array_count()
     && json_array_with_array_count()
     && json_array_count_complex()
@@ -999,12 +1023,13 @@ int main() {
     && current_location_out_of_bounds()
     && current_location_no_error()
   #if SIMDJSON_EXCEPTIONS
+    && raw_string()
     && number_tests()
     && current_location_tape_error_with_except()
   #endif
-  ) {
-    return 0;
-  } else {
-    return 1;
-  }
+  ;
+}
+
+int main(int argc, char *argv[]) {
+  return test_main(argc, argv, run);
 }