Compare commits

...

12 Commits

Author SHA1 Message Date
Daniel Lemire 90e9d3c90f fix 2025-07-31 13:08:21 -04:00
Daniel Lemire 3d1ab87ecb Merge branch 'master' into release_candidate_4_0_0 2025-07-31 10:24:54 -04:00
Borislav Stanimirov d365cfb4a1 remove cmake_policy (#2404)
* properly gitignore Visual Studio artifacts

* remove cmake_policy
2025-07-31 10:18:28 -04:00
Daniel Lemire b9e308a727 marking them as 'really inline' 2025-07-28 11:47:09 -04:00
Brad Bramble aa6817d5aa Fix linker errors for downstream users caused by non-inline symbols in header (#2403) 2025-07-28 11:44:32 -04:00
Francisco Geiman Thiesen 90e4a66c93 Bringing a bit more use-cases for reflection based serializations (optional). Also adding string-based enum handling as requested on X. (#2395)
* Adding type validation, enhancing optional type support and adding test a few more tests.

* Adding support for string-based enum serlalization and deserialization.

* Removing unintentional endline.

* Removing trailing whitespace.

* Adding simpler api as suggested by moisrex.

* Removing explicit optiona<int> and optional<std::string> references and using concepts instead! Credit goes to Lemire for pointing this out and suggesting a concepts based approach here.

* Removing tests that are not relevant for this branch.

* Removing api related changes. That will be done by moisrex.

* Removing unnecessary new endlines.

* Removing tests related to api changes and cleaning-up irrelevant tests.

* removing broken reference

* Removing trailing whitespace
2025-07-23 09:24:05 +02:00
evbse e6240f18c4 Add version to amalgamated files (#2400) 2025-07-23 09:20:06 +02:00
Daniel Lemire 97eb557388 Merge branch 'release_candidate_4_0_0' of github.com:simdjson/simdjson into release_candidate_4_0_0 2025-07-16 12:08:03 -04:00
Daniel Lemire 41ced28821 adding a space 2025-07-16 12:07:49 -04:00
Daniel Lemire 1078eb4034 minor update to the release candidate (#2394) 2025-07-16 12:05:09 -04:00
Daniel Lemire 0cd774097f [skip ci] doc fixes 2025-07-14 23:46:30 -04:00
Daniel Lemire b5e27af4da release candidate 4.0.0 2025-07-14 15:55:00 -04:00
51 changed files with 13443 additions and 376 deletions
+2 -2
View File
@@ -38,7 +38,7 @@ cmake-build-release/
.history/
# Visual Studio artifacts
/VS/
/.vs/
# C/C++ build outputs
.build/
@@ -106,4 +106,4 @@ objs
!.vscode/extensions.json
# clangd
.cache
.cache
+3 -5
View File
@@ -1,11 +1,9 @@
cmake_minimum_required(VERSION 3.14)
cmake_policy(VERSION 3.5) # For doctest
project(
simdjson
# The version number is modified by tools/release.py
VERSION 3.13.0
VERSION 4.0.0
DESCRIPTION "Parsing gigabytes of JSON per second"
HOMEPAGE_URL "https://simdjson.org/"
LANGUAGES CXX C
@@ -22,8 +20,8 @@ string(
# ---- Options, variables ----
# These version numbers are modified by tools/release.py
set(SIMDJSON_LIB_VERSION "26.0.0" CACHE STRING "simdjson library version")
set(SIMDJSON_LIB_SOVERSION "26" CACHE STRING "simdjson library soversion")
set(SIMDJSON_LIB_VERSION "28.0.0" CACHE STRING "simdjson library version")
set(SIMDJSON_LIB_SOVERSION "28" CACHE STRING "simdjson library soversion")
option(SIMDJSON_BUILD_STATIC_LIB "Build simdjson_static library along with simdjson (only makes sense if BUILD_SHARED_LIBS=ON)" OFF)
if(SIMDJSON_BUILD_STATIC_LIB AND NOT BUILD_SHARED_LIBS)
+1 -1
View File
@@ -38,7 +38,7 @@ PROJECT_NAME = simdjson
# could be handy for archiving the generated documentation or if some version
# control system is used.
PROJECT_NUMBER = "3.13.0"
PROJECT_NUMBER = "4.0.0"
# Using the PROJECT_BRIEF tag one can provide an optional one line description
# for a project that appears at the top of each page and should give viewer a
+46 -42
View File
@@ -1,47 +1,50 @@
The Basics
==========
An overview of what you need to know to use simdjson, with examples.
An overview of what you need to know to use simdjson to parse JSON documents, with examples.
[Our documentation regarding the generation (serialization) of JSON documents is in a
separate document](https://github.com/simdjson/simdjson/blob/master/doc/builder.md).
- [The Basics](#the-basics)
- [Requirements](#requirements)
- [Including simdjson](#including-simdjson)
- [Using simdjson with package managers](#using-simdjson-with-package-managers)
- [Using simdjson as a CMake dependency](#using-simdjson-as-a-cmake-dependency)
- [Versions](#versions)
- [The basics: loading and parsing JSON documents](#the-basics-loading-and-parsing-json-documents)
- [Documents are iterators](#documents-are-iterators)
- [Parser, document and JSON scope](#parser-document-and-json-scope)
- [string_view](#string_view)
- [Avoiding pitfalls: enable development checks](#avoiding-pitfalls-enable-development-checks)
- [Using the parsed JSON](#using-the-parsed-json)
- [Using the parsed JSON: additional examples](#using-the-parsed-json-additional-examples)
- [Adding support for custom types](#adding-support-for-custom-types)
- [1. Specialize `simdjson::ondemand::value::get` to get custom types (pre-C++20)](#1-specialize-simdjsonondemandvalueget-to-get-custom-types-pre-c20)
- [2. Use `tag_invoke` for custom types (C++20)](#2-use-tag_invoke-for-custom-types-c20)
- [Minifying JSON strings without parsing](#minifying-json-strings-without-parsing)
- [UTF-8 validation (alone)](#utf-8-validation-alone)
- [JSON Pointer](#json-pointer)
- [JSONPath](#jsonpath)
- [Error handling](#error-handling)
- [Error handling examples without exceptions](#error-handling-examples-without-exceptions)
- [Disabling exceptions](#disabling-exceptions)
- [Exceptions](#exceptions)
- [Current location in document](#current-location-in-document)
- [Checking for trailing content](#checking-for-trailing-content)
- [Rewinding](#rewinding)
- [Newline-Delimited JSON (ndjson) and JSON lines](#newline-delimited-json-ndjson-and-json-lines)
- [Parsing numbers inside strings](#parsing-numbers-inside-strings)
- [Dynamic Number Types](#dynamic-number-types)
- [Raw strings from keys](#raw-strings-from-keys)
- [General direct access to the raw JSON string](#general-direct-access-to-the-raw-json-string)
- [Storing directly into an existing string instance](#storing-directly-into-an-existing-string-instance)
- [Thread safety](#thread-safety)
- [Standard compliance](#standard-compliance)
- [Backwards compatibility](#backwards-compatibility)
- [Examples](#examples)
- [Performance tips](#performance-tips)
- [Further reading](#further-reading)
* [Requirements](#requirements)
* [Including simdjson](#including-simdjson)
* [Using simdjson with package managers](#using-simdjson-with-package-managers)
* [Using simdjson as a CMake dependency](#using-simdjson-as-a-cmake-dependency)
* [Versions](#versions)
* [The basics: loading and parsing JSON documents](#the-basics--loading-and-parsing-json-documents)
* [Documents are iterators](#documents-are-iterators)
+ [Parser, document and JSON scope](#parser--document-and-json-scope)
* [string_view](#string-view)
* [Avoiding pitfalls: enable development checks](#avoiding-pitfalls--enable-development-checks)
* [Using the parsed JSON](#using-the-parsed-json)
+ [Using the parsed JSON: additional examples](#using-the-parsed-json--additional-examples)
* [Adding support for custom types](#adding-support-for-custom-types)
+ [1. Specialize `simdjson::ondemand::value::get` to get custom types (pre-C++20)](#1-specialize--simdjson--ondemand--value--get--to-get-custom-types--pre-c--20-)
+ [2. Use `tag_invoke` for custom types (C++20)](#2-use--tag-invoke--for-custom-types--c--20-)
+ [3. Using static reflection (C++26)](#3-using-static-reflection--c--26-)
* [Minifying JSON strings without parsing](#minifying-json-strings-without-parsing)
* [UTF-8 validation (alone)](#utf-8-validation--alone-)
* [JSON Pointer](#json-pointer)
* [JSONPath](#jsonpath)
* [Error handling](#error-handling)
+ [Error handling examples without exceptions](#error-handling-examples-without-exceptions)
+ [Disabling exceptions](#disabling-exceptions)
+ [Exceptions](#exceptions)
+ [Current location in document](#current-location-in-document)
+ [Checking for trailing content](#checking-for-trailing-content)
* [Rewinding](#rewinding)
* [Newline-Delimited JSON (ndjson) and JSON lines](#newline-delimited-json--ndjson--and-json-lines)
* [Parsing numbers inside strings](#parsing-numbers-inside-strings)
* [Dynamic Number Types](#dynamic-number-types)
* [Raw strings from keys](#raw-strings-from-keys)
* [General direct access to the raw JSON string](#general-direct-access-to-the-raw-json-string)
* [Storing directly into an existing string instance](#storing-directly-into-an-existing-string-instance)
* [Thread safety](#thread-safety)
* [Standard compliance](#standard-compliance)
* [Backwards compatibility](#backwards-compatibility)
* [Examples](#examples)
* [Performance tips](#performance-tips)
* [Further reading](#further-reading)
Requirements
@@ -826,7 +829,7 @@ There are 3 main ways provided by simdjson to deserialize a value into a custom
1. Specialize `simdjson::ondemand::document::get` for the whole document
2. Specialize `simdjson::ondemand::value::get` for each value
2. Using `tag_invoke` *(the recommended way if your system supports C++20 or better)*
3. Using static reflectioin (requires C++26 or better)
3. Using static reflection (requires C++26 or better)
We describe all of them in the following sections. Most users who have systems compatible with
C++20 or better should skip ahead to [using `tag_invoke` for custom types (C++20)](#2-use-tag_invoke-for-custom-types-c20) as it is more powerful and simpler.
@@ -1140,7 +1143,7 @@ struct Car {
};
```
Observe how we defined the class to use types that simdjson does not directly support (`float`, `int`).
Observe how we define the class to use types that simdjson does not directly support (`float`, `int`).
With C++20 support, the library grabs from the JSON the generic type (`double`, `int`) and then it
casts it automatically.
@@ -1355,11 +1358,12 @@ your code with the `SIMDJSON_STATIC_REFLECTION` macro set:
```
Then you can deserialize a type such as `Car` automatically:
```cpp
std::string json = R"( { "make": "Toyota", "model": "Camry", "year": 2018,
"tire_pressure": [ 40.1, 39.9 ] } )";
simdjson::ondemand::parser parser;
simdjson::ondemand::document doc = parser.iterate(simdjson::pad(json)).get(doc);
simdjson::ondemand::document doc = parser.iterate(simdjson::pad(json));
Car c = doc.get<Car>();
```
+114 -44
View File
@@ -4,16 +4,29 @@ Builder
Sometimes you want to generate JSON string outputs efficiently.
The simdjson library provides high-performance low-level facilities.
When using these low-level functionalities, you are responsible to
define the structure of your JSON document. However, string escaping
and UTF-8 validation is automated.
define the structure of your JSON document. Our more advanced interface
automates the process using C++26 static reflection: you get both high
speed and high convenience.
- [Builder](#builder)
* [Overview: string_builder](#overview--string-builder)
* [Example: string_builder](#example--string-builder)
* [C++26 static reflection](#c--26-static-reflection)
+ [Without `string_buffer` instance](#without--string-buffer--instance)
+ [Without `string_buffer` instance but with explicit error handling](#without--string-buffer--instance-but-with-explicit-error-handling)
Overview: string_builder
---------------------------
The string_builder class is a low-level utility for constructing JSON strings representing documents. It is optimized for performance, potentially leveraging kernel-specific features like SIMD instructions for tasks such as string escaping. This class supports atomic types (e.g., booleans, numbers, strings) but does not handle composed types directly (like arrays or objects).
Note that JSON strings are always encoded as UTF-8.
An `string_builder` is created with an initial buffer capacity (e.g., 1kB). The memory
is reallocated when needed. It has the following methods to add content to the string:
is reallocated when needed.
The efficiency of `string_builder` stems from its internal use of a resizable array or buffer. When you append data, it adds the characters to this buffer, resizing it only when necessary, typically in a way that minimizes reallocations. This approach contrasts with regular string concatenation, where each operation creates a new string, copying all previous content, leading to quadratic time complexity for repeated concatenations.
It has the following methods to add content to the string:
- `append(number_type v)`: Appends a number (including booleans) to the JSON buffer. Booleans are converted to the strings "false" or "true". Numbers are formatted according to the JSON standard, with floating-point numbers using the shortest representation that accurately reflects the value.
@@ -32,6 +45,9 @@ After writting the content, if you have reasons to believe that the content migh
- `validate_unicode()`: Checks if the content in the JSON buffer is valid UTF-8. Returns: true if the content is valid UTF-8, false otherwise.
You might need to do unicode validation if you have strings in your data structures containing
malformed UTF-8.
Once you are satisfied, you can recover the string as follows:
- `operator std::string()`: Converts the JSON buffer to an std::string. (Might throw if an error occurred.)
@@ -44,55 +60,75 @@ Example: string_builder
---------------------------
```C++
struct Car {
std::string make;
std::string model;
int64_t year;
std::vector<double> tire_pressure;
};
void serialize_car(const Car& car, simdjson::builder::string_builder& builder) {
// start of JSON
builder.start_object();
void serialize_car(const Car& car, simdjson::builder::string_builder& builder) {
// start of JSON
builder.start_object();
// "make"
builder.append_key_value("make", car.make);
builder.append_comma();
// "make"
builder.append_key_value("make", car.make);
builder.append_comma();
// "model"
builder.append_key_value("model", car.model);
builder.append_comma();
// "model"
builder.append_key_value("model", car.model);
builder.append_comma();
// "year"
builder.append_key_value("year", car.year);
builder.append_comma();
// "year"
builder.append_key_value("year", car.year);
builder.append_comma();
// "tire_pressure"
builder.escape_and_append_with_quotes("tire_pressure");
builder.append_colon();
builder.start_array();
// vector tire_pressure
for (size_t i = 0; i < car.tire_pressure.size(); ++i) {
builder.append(car.tire_pressure[i]);
if (i < car.tire_pressure.size() - 1) {
builder.append_comma();
}
// "tire_pressure"
builder.escape_and_append_with_quotes("tire_pressure");
builder.append_colon();
builder.start_array();
// vector tire_pressure
for (size_t i = 0; i < car.tire_pressure.size(); ++i) {
builder.append(car.tire_pressure[i]);
if (i < car.tire_pressure.size() - 1) {
builder.append_comma();
}
builder.end_array();
builder.end_object();
}
builder.end_array();
builder.end_object();
}
bool car_test() {
simdjson::builder::string_builder sb;
Car c = {"Toyota", "Corolla", 2017, {30.0,30.2,30.513,30.79}};
serialize_car(c, sb);
std::string_view p;
if(sb.view().get(p)) {
return false; // there was an error
}
// p holds the JSON:
// "{\"make\":\"Toyota\",\"model\":\"Corolla\",\"year\":2017,\"tire_pressure\":[30.0,30.2,30.513,30.79]}"
return true;
bool car_test() {
simdjson::builder::string_builder sb;
Car c = {"Toyota", "Corolla", 2017, {30.0,30.2,30.513,30.79}};
serialize_car(c, sb);
std::string_view p{sb};
// p holds the JSON:
// "{\"make\":\"Toyota\",\"model\":\"Corolla\",\"year\":2017,\"tire_pressure\":[30.0,30.2,30.513,30.79]}"
return true;
}
```
The `string_builder` constructor takes an optional parameter which specifies the initial
memory allocation in byte. If you know approximately the size of your JSON output, you can
pass this value as a parameter (e.g., `simdjson::builder::string_builder sb{1233213}`).
The `string_builder` might throw an exception in case of error when you cast it result to `std::string_view`. If you wish to avoid exceptions, you can use the following programming pattern:
```cpp
std::string_view p;
if(sb.view().get(p)) {
return false; // there was an error
}
```
In all cases, the `std::string_view` instance depends the corresponding `string_builder` instance.
C++26 static reflection
------------------------
Static reflection (or compile-time reflection) in C++26 introduces a powerful compile-time mechanism that allows a program to inspect and manipulate its own structure, such as types, variables, functions, and other program elements, during compilation. Unlike runtime reflection in languages like Java or Python, C++26s static reflection operates entirely at compile time, aligning with C++s emphasis on zero-overhead abstractions and high performance. It means
that you can delegate much of the work to the library.
If you have a compiler with support C++26 static reflection, you can compile
your code with the `SIMDJSON_STATIC_REFLECTION` macro set:
@@ -106,21 +142,55 @@ And then you can append your data structures to a `string_builder` instance
automatically. In most cases, it should work automatically:
```cpp
struct Car {
std::string make;
std::string model;
int64_t year;
std::vector<double> tire_pressure;
};
bool car_test() {
simdjson::builder::string_builder sb;
Car c = {"Toyota", "Corolla", 2017, {30.0,30.2,30.513,30.79}};
append(sb, c);
std::string_view p;
if(sb.view().get(p)) {
return false; // there was an error
}
sb << c;
std::string_view p{sb};
// p holds the JSON:
// "{\"make\":\"Toyota\",\"model\":\"Corolla\",\"year\":2017,\"tire_pressure\":[30.0,30.2,30.513,30.79]}"
return true;
}
```
If you prefer, you can also create a string directly:
### Without `string_buffer` instance
In some instances, you might want to create a string directly from your own data type.
You can create a string directly, without an explicit `string_builder` instance
with the `simdjson::builder::to_json_string` function.
(Under the hood a `string_builder` instance may still be created.)
```cpp
struct Car {
std::string make;
std::string model;
int64_t year;
std::vector<double> tire_pressure;
};
void f() {
Car c = {"Toyota", "Corolla", 2017, {30.0,30.2,30.513,30.79}};
std::string json = simdjson::builder::to_json_string(c);
}
```
If you know the output size, in bytes, of your JSON string, you may
pass it as a second parameter (e.g., `simdjson::builder::to_json_string(c, 31123)`).
### Without `string_buffer` instance but with explicit error handling
If prefer a version without exceptions and explicit error handling, you can use the following
pattern:
```cpp
std::string json;
+8 -1
View File
@@ -1,7 +1,10 @@
The Document-Object-Model (DOM) front-end
==========
An overview of what you need to know to use simdjson, with examples.
An overview of what you need to know to use simdjson to parse JSON documents with
our DOM API, with examples. [Our documentation regarding the generation (serialization) of JSON documents is in a
separate document](https://github.com/simdjson/simdjson/blob/master/doc/builder.md).
* [DOM vs On-Demand](#dom-vs-on-demand)
* [The Basics: Loading and Parsing JSON Documents](#the-basics-loading-and-parsing-json-documents-using-the-dom-front-end)
@@ -28,6 +31,10 @@ a conventional Document-Object-Model (DOM) front-end. In such a scenario, the JS
entirely parsed, validated and materialized in memory as the first step. The programmer may
then access the parsed data using this in-memory model.
On-Demand is a different model where you parse just what you need, directly into your own
data structure. The On-Demand approach, when well tuned, can provide superior performance.
[We refer you to the On-Demand documentation for further details](https://github.com/simdjson/simdjson/blob/master/doc/basics.md).
The Basics: Loading and Parsing JSON Documents using the DOM front-end
----------------------------------------------
-1
View File
@@ -23,7 +23,6 @@ public:
) const noexcept final;
simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final;
simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final;
simdjson_warn_unused size_t write_string_escaped(const std::string_view input, char *out) const noexcept final;
};
} // namespace arm64
+12
View File
@@ -356,4 +356,16 @@ namespace std {
#define SIMDJSON_AVX512_ALLOWED 1
#endif
#ifndef __has_cpp_attribute
#define simdjson_lifetime_bound
#elif __has_cpp_attribute(msvc::lifetimebound)
#define simdjson_lifetime_bound [[msvc::lifetimebound]]
#elif __has_cpp_attribute(clang::lifetimebound)
#define simdjson_lifetime_bound [[clang::lifetimebound]]
#elif __has_cpp_attribute(lifetimebound)
#define simdjson_lifetime_bound [[lifetimebound]]
#else
#define simdjson_lifetime_bound
#endif
#endif // SIMDJSON_COMMON_DEFS_H
-1
View File
@@ -115,7 +115,6 @@ concept optional_type = requires(std::remove_cvref_t<T> obj) {
{ obj.value() } -> std::same_as<typename std::remove_cvref_t<T>::value_type&>;
requires requires(typename std::remove_cvref_t<T>::value_type &&val) {
obj.emplace(std::move(val));
obj = std::move(val);
{
obj.value_or(val)
} -> std::convertible_to<typename std::remove_cvref_t<T>::value_type>;
+3
View File
@@ -107,6 +107,9 @@ protected:
* by a "formatter" which handles the details. Thus
* the string_builder template could support both minification
* and prettification, and various other tradeoffs.
*
* This is not to be confused with the simdjson::builder::string_builder
* which is a different class.
*/
template <class formatter = mini_formatter>
class string_builder {
@@ -26,7 +26,6 @@ public:
) const noexcept final;
simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final;
simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final;
simdjson_warn_unused size_t write_string_escaped(const std::string_view input, char *out) const noexcept final;
};
} // namespace fallback
@@ -77,7 +77,7 @@ template <typename T, typename ValT = SIMDJSON_IMPLEMENTATION::ondemand::value>
concept custom_deserializable = tag_invocable<deserialize_tag, ValT&, T&>;
template <typename T, typename ValT = SIMDJSON_IMPLEMENTATION::ondemand::value>
concept deserializable = custom_deserializable<T, ValT> || is_builtin_deserializable_v<T>;
concept deserializable = custom_deserializable<T, ValT> || is_builtin_deserializable_v<T> || concepts::optional_type<T>;
template <typename T, typename ValT = SIMDJSON_IMPLEMENTATION::ondemand::value>
concept nothrow_custom_deserializable = nothrow_tag_invocable<deserialize_tag, ValT&, T&>;
@@ -195,8 +195,8 @@ simdjson_inline document::operator object() & noexcept(false) { return get_objec
simdjson_inline document::operator uint64_t() noexcept(false) { return get_uint64(); }
simdjson_inline document::operator int64_t() noexcept(false) { return get_int64(); }
simdjson_inline document::operator double() noexcept(false) { return get_double(); }
simdjson_inline document::operator std::string_view() noexcept(false) { return get_string(false); }
simdjson_inline document::operator raw_json_string() noexcept(false) { return get_raw_json_string(); }
simdjson_inline document::operator std::string_view() noexcept(false) simdjson_lifetime_bound { return get_string(false); }
simdjson_inline document::operator raw_json_string() noexcept(false) simdjson_lifetime_bound { return get_raw_json_string(); }
simdjson_inline document::operator bool() noexcept(false) { return get_bool(); }
simdjson_inline document::operator value() noexcept(false) { return get_value(); }
+2 -2
View File
@@ -322,7 +322,7 @@ public:
* time it parses a document or when it is destroyed.
* @exception simdjson_error(INCORRECT_TYPE) if the JSON value is not a string.
*/
simdjson_inline operator std::string_view() noexcept(false);
simdjson_inline operator std::string_view() noexcept(false) simdjson_lifetime_bound;
/**
* Cast this JSON value to a raw_json_string.
*
@@ -331,7 +331,7 @@ public:
* @returns A pointer to the raw JSON for the given string.
* @exception simdjson_error(INCORRECT_TYPE) if the JSON value is not a string.
*/
simdjson_inline operator raw_json_string() noexcept(false);
simdjson_inline operator raw_json_string() noexcept(false) simdjson_lifetime_bound;
/**
* Cast this JSON value to a bool.
*
@@ -14,6 +14,8 @@
#include <charconv>
#include <cstring>
#include <experimental/meta>
#include <memory>
#include <optional>
#include <string_view>
#include <type_traits>
#include <utility>
@@ -94,8 +96,13 @@ consteval std::string consteval_to_quoted_escaped(std::string_view input);
template <class T>
requires(std::is_class_v<T> && !container_but_not_string<T> &&
!concepts::string_view_keyed_map<T> &&
!concepts::optional_type<T> &&
!concepts::smart_pointer<T> &&
!concepts::appendable_containers<T> &&
!std::is_same_v<T, std::string> &&
!std::is_same_v<T, std::string_view>)
!std::is_same_v<T, std::string_view> &&
!std::is_same_v<T, const char*> &&
!std::is_same_v<T, char>)
constexpr void atom(string_builder &b, const T &t) {
int i = 0;
b.append('{');
@@ -111,8 +118,127 @@ constexpr void atom(string_builder &b, const T &t) {
b.append('}');
}
// Support for optional types (std::optional, etc.)
template <concepts::optional_type T>
constexpr void atom(string_builder &b, const T &opt) {
if (opt) {
atom(b, opt.value());
} else {
b.append_raw("null");
}
}
// Support for smart pointers (std::unique_ptr, std::shared_ptr, etc.)
template <concepts::smart_pointer T>
constexpr void atom(string_builder &b, const T &ptr) {
if (ptr) {
atom(b, *ptr);
} else {
b.append_raw("null");
}
}
// Support for enums - serialize as string representation using expand approach from P2996R12
template <typename T>
requires(std::is_enum_v<T>)
void atom(string_builder &b, const T &e) {
#if SIMDJSON_STATIC_REFLECTION
std::string_view result = "<unnamed>";
[:expand(std::meta::enumerators_of(^^T)):] >> [&]<auto enum_val>{
if (e == [:enum_val:]) {
result = std::meta::identifier_of(enum_val);
}
};
if (result != "<unnamed>") {
b.append_raw("\"");
b.append_raw(result);
b.append_raw("\"");
} else {
// Fallback to integer if enum value not found
atom(b, static_cast<std::underlying_type_t<T>>(e));
}
#else
// Fallback: serialize as integer if reflection not available
atom(b, static_cast<std::underlying_type_t<T>>(e));
#endif
}
// Support for appendable containers that don't have operator[] (sets, etc.)
template <concepts::appendable_containers T>
requires(!container_but_not_string<T> && !concepts::string_view_keyed_map<T> &&
!concepts::optional_type<T> && !concepts::smart_pointer<T> &&
!std::is_same_v<T, std::string> &&
!std::is_same_v<T, std::string_view> && !std::is_same_v<T, const char*>)
constexpr void atom(string_builder &b, const T &container) {
if (container.empty()) {
b.append_raw("[]");
return;
}
b.append('[');
bool first = true;
for (const auto& item : container) {
if (!first) {
b.append(',');
}
first = false;
atom(b, item);
}
b.append(']');
}
// append functions that delegate to atom functions for primitive types
template <class T>
requires(std::is_arithmetic_v<T> && !std::is_same_v<T, char>)
void append(string_builder &b, const T &t) {
atom(b, t);
}
template <class T>
requires(std::is_same_v<T, std::string> ||
std::is_same_v<T, std::string_view> ||
std::is_same_v<T, const char *> ||
std::is_same_v<T, char>)
void append(string_builder &b, const T &t) {
atom(b, t);
}
template <concepts::optional_type T>
void append(string_builder &b, const T &t) {
atom(b, t);
}
template <concepts::smart_pointer T>
void append(string_builder &b, const T &t) {
atom(b, t);
}
template <concepts::appendable_containers T>
requires(!container_but_not_string<T> && !concepts::string_view_keyed_map<T> &&
!concepts::optional_type<T> && !concepts::smart_pointer<T> &&
!std::is_same_v<T, std::string> &&
!std::is_same_v<T, std::string_view> && !std::is_same_v<T, const char*>)
void append(string_builder &b, const T &t) {
atom(b, t);
}
template <concepts::string_view_keyed_map T>
void append(string_builder &b, const T &t) {
atom(b, t);
}
// works for struct
template <class Z> void append(string_builder &b, const Z &z) {
template <class Z>
requires(std::is_class_v<Z> && !container_but_not_string<Z> &&
!concepts::string_view_keyed_map<Z> &&
!concepts::optional_type<Z> &&
!concepts::smart_pointer<Z> &&
!concepts::appendable_containers<Z> &&
!std::is_same_v<Z, std::string> &&
!std::is_same_v<Z, std::string_view> &&
!std::is_same_v<Z, const char*> &&
!std::is_same_v<Z, char>)
void append(string_builder &b, const Z &z) {
int i = 0;
b.append('{');
[:expand(std::meta::nonstatic_data_members_of(^^Z, std::meta::access_context::unchecked())):] >> [&]<auto dm>() {
@@ -145,8 +271,8 @@ void append(string_builder &b, const Z &z) {
}
template <class Z>
simdjson_result<std::string> to_json_string(const Z &z) {
string_builder b;
simdjson_result<std::string> to_json_string(const Z &z, size_t initial_capacity = 1024) {
string_builder b(initial_capacity);
append(b, z);
std::string_view s;
if(auto e = b.view().get(s); e) { return e; }
@@ -162,7 +288,13 @@ simdjson_error to_json(const Z &z, std::string &s) {
s.assign(view);
return SUCCESS;
}
} // namespace json_builder
template <class Z>
string_builder& operator<<(string_builder& b, const Z& z) {
append(b, z);
return b;
}
} // namespace builder
} // namespace SIMDJSON_IMPLEMENTATION
} // namespace simdjson
#endif // SIMDJSON_STATIC_REFLECTION
@@ -303,9 +303,9 @@ simdjson_inline void string_builder::clear() noexcept {
namespace internal {
// We could specialize further for 32-bit integers.
int int_log2(uint32_t x) { return (63 - leading_zeroes(x | 1)); }
simdjson_really_inline int int_log2(uint32_t x) { return (63 - leading_zeroes(x | 1)); }
int fast_digit_count(uint32_t x) {
simdjson_really_inline int fast_digit_count(uint32_t x) {
static uint64_t table[] = {
4294967296, 8589934582, 8589934582, 8589934582, 12884901788,
12884901788, 12884901788, 17179868184, 17179868184, 17179868184,
@@ -317,9 +317,9 @@ int fast_digit_count(uint32_t x) {
return uint32_t((x + table[int_log2(x)]) >> 32);
}
int int_log2(uint64_t x) { return 63 - leading_zeroes(x | 1); }
simdjson_really_inline int int_log2(uint64_t x) { return 63 - leading_zeroes(x | 1); }
int fast_digit_count(uint64_t x) {
simdjson_really_inline int fast_digit_count(uint64_t x) {
static uint64_t table[] = {9,
99,
999,
@@ -346,7 +346,7 @@ int fast_digit_count(uint64_t x) {
template <typename number_type, typename = typename std::enable_if<
std::is_unsigned<number_type>::value>::type>
simdjson_inline size_t digit_count(number_type v) noexcept {
simdjson_really_inline size_t digit_count(number_type v) noexcept {
static_assert(sizeof(number_type) == 8 || sizeof(number_type) == 4 ||
sizeof(number_type) == 2 || sizeof(number_type) == 1,
"We only support 8-bit, 16-bit, 32-bit and 64-bit numbers");
@@ -516,7 +516,7 @@ simdjson_inline string_builder::operator std::string() const noexcept(false) {
}
simdjson_inline string_builder::operator std::string_view() const
noexcept(false) {
noexcept(false) simdjson_lifetime_bound {
return view();
}
#endif
@@ -19,7 +19,7 @@ namespace builder {
* supports atomic types (Booleans, strings), it does not support composed
* types (arrays and objects).
*
* Ultimately, this class should support kernel-specific optimizations. E.g.,
* Ultimately, this class can support kernel-specific optimizations. E.g.,
* it may make use of SIMD instructions to escape strings faster.
*/
class string_builder {
@@ -146,7 +146,7 @@ public:
* The result may not be valid UTF-8 if some of your content was not valid UTF-8.
* Use validate_unicode() to check the content if needed.
*/
simdjson_inline operator std::string_view() const noexcept(false);
simdjson_inline operator std::string_view() const noexcept(false) simdjson_lifetime_bound;
#endif
/**
@@ -9,6 +9,9 @@
#include "simdjson/generic/ondemand/object.h"
#include "simdjson/generic/ondemand/serialization.h"
#include "simdjson/generic/ondemand/value.h"
#if SIMDJSON_STATIC_REFLECTION
#include "simdjson/generic/ondemand/json_builder.h"
#endif
#endif // SIMDJSON_CONDITIONAL_INCLUDE
namespace simdjson {
@@ -35,6 +35,20 @@ inline simdjson_result<std::string_view> to_json_string(simdjson_result<SIMDJSON
inline simdjson_result<std::string_view> to_json_string(simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> x);
inline simdjson_result<std::string_view> to_json_string(simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::object> x);
inline simdjson_result<std::string_view> to_json_string(simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array> x);
#if SIMDJSON_STATIC_REFLECTION
/**
* Create a JSON string from any user-defined type using static reflection.
* Only available when SIMDJSON_STATIC_REFLECTION is enabled.
*/
template<typename T>
requires(!std::same_as<T, SIMDJSON_IMPLEMENTATION::ondemand::document> &&
!std::same_as<T, SIMDJSON_IMPLEMENTATION::ondemand::value> &&
!std::same_as<T, SIMDJSON_IMPLEMENTATION::ondemand::object> &&
!std::same_as<T, SIMDJSON_IMPLEMENTATION::ondemand::array>)
inline std::string to_json_string(const T& obj);
#endif
} // namespace simdjson
/**
@@ -248,17 +248,16 @@ error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept(nothrow_deser
/**
* This CPO (Customization Point Object) will help deserialize into optional types.
*/
template <concepts::optional_type T, typename ValT>
template <concepts::optional_type T>
requires(!require_custom_serialization<T>)
error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept(nothrow_deserializable<typename std::remove_cvref_t<T>::value_type, ValT>) {
error_code tag_invoke(deserialize_tag, auto &val, T &out) noexcept(nothrow_deserializable<typename std::remove_cvref_t<T>::value_type, decltype(val)>) {
using value_type = typename std::remove_cvref_t<T>::value_type;
static_assert(
deserializable<value_type, ValT>,
"The specified type inside the unique_ptr must itself be deserializable");
static_assert(
std::is_default_constructible_v<value_type>,
"The specified type inside the unique_ptr must default constructible.");
// Check if the value is null
if (val.is_null()) {
out.reset(); // Set to nullopt
return SUCCESS;
}
if (!out) {
out.emplace();
@@ -297,7 +296,7 @@ template<typename R>
consteval auto expand(R range) {
std::vector<std::meta::info> args;
for (auto r : range) {
args.push_back(reflect_value(r));
args.push_back(reflect_constant(r));
}
return substitute(^^__impl::replicator, args);
}
@@ -317,17 +316,42 @@ error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept {
[:expand(std::meta::nonstatic_data_members_of(^^T, std::meta::access_context::unchecked())):] >> [&]<auto mem>() {
if constexpr (!std::meta::is_const(mem) && std::meta::is_public(mem)) {
constexpr std::string_view key = std::define_static_string(std::meta::identifier_of(mem));
static_assert(
deserializable<decltype(out.[:mem:]), SIMDJSON_IMPLEMENTATION::ondemand::object>,
"The specified type inside the class must itself be deserializable");
// Note: removed static assert as optional types are now handled generically
// as long we are succesful or the field is not found, we continue
if(e == simdjson::SUCCESS || e == simdjson::NO_SUCH_FIELD) {
obj[key].get(out.[:mem:]);
e = obj[key].get(out.[:mem:]);
}
}
};
return e;
}
// Support for enum deserialization - deserialize from string representation using expand approach from P2996R12
template <typename T, typename ValT>
requires(std::is_enum_v<T>)
error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept {
#if SIMDJSON_STATIC_REFLECTION
std::string_view str;
SIMDJSON_TRY(val.get_string().get(str));
bool found = false;
[:expand(std::meta::enumerators_of(^^T)):] >> [&]<auto enum_val>{
if (!found && str == std::meta::identifier_of(enum_val)) {
out = [:enum_val:];
found = true;
}
};
return found ? SUCCESS : INCORRECT_TYPE;
#else
// Fallback: deserialize as integer if reflection not available
std::underlying_type_t<T> int_val;
SIMDJSON_TRY(val.get(int_val));
out = static_cast<T>(int_val);
return SUCCESS;
#endif
}
template <typename simdjson_value, typename T>
requires(user_defined_type<std::remove_cvref_t<T>>)
error_code tag_invoke(deserialize_tag, simdjson_value &val, std::unique_ptr<T> &out) noexcept {
@@ -366,6 +390,10 @@ error_code tag_invoke(deserialize_tag, simdjson_value &val, std::shared_ptr<T> &
// Unique pointers
////////////////////////////////////////
error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr<bool> &out) noexcept {
if (val.is_null()) {
out.reset();
return SUCCESS;
}
if (!out) {
out = std::make_unique<bool>();
if (!out) { return MEMALLOC; }
@@ -375,6 +403,10 @@ error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr<bool> &out) no
}
error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr<int64_t> &out) noexcept {
if (val.is_null()) {
out.reset();
return SUCCESS;
}
if (!out) {
out = std::make_unique<int64_t>();
if (!out) { return MEMALLOC; }
@@ -384,6 +416,10 @@ error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr<int64_t> &out)
}
error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr<uint64_t> &out) noexcept {
if (val.is_null()) {
out.reset();
return SUCCESS;
}
if (!out) {
out = std::make_unique<uint64_t>();
if (!out) { return MEMALLOC; }
@@ -393,6 +429,10 @@ error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr<uint64_t> &out
}
error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr<double> &out) noexcept {
if (val.is_null()) {
out.reset();
return SUCCESS;
}
if (!out) {
out = std::make_unique<double>();
if (!out) { return MEMALLOC; }
@@ -402,6 +442,10 @@ error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr<double> &out)
}
error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr<std::string_view> &out) noexcept {
if (val.is_null()) {
out.reset();
return SUCCESS;
}
if (!out) {
out = std::make_unique<std::string_view>();
if (!out) { return MEMALLOC; }
@@ -415,6 +459,10 @@ error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr<std::string_vi
// Shared pointers
////////////////////////////////////////
error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr<bool> &out) noexcept {
if (val.is_null()) {
out.reset();
return SUCCESS;
}
if (!out) {
out = std::make_shared<bool>();
if (!out) { return MEMALLOC; }
@@ -424,6 +472,10 @@ error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr<bool> &out) no
}
error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr<int64_t> &out) noexcept {
if (val.is_null()) {
out.reset();
return SUCCESS;
}
if (!out) {
out = std::make_shared<int64_t>();
if (!out) { return MEMALLOC; }
@@ -433,6 +485,10 @@ error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr<int64_t> &out)
}
error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr<uint64_t> &out) noexcept {
if (val.is_null()) {
out.reset();
return SUCCESS;
}
if (!out) {
out = std::make_shared<uint64_t>();
if (!out) { return MEMALLOC; }
@@ -442,6 +498,10 @@ error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr<uint64_t> &out
}
error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr<double> &out) noexcept {
if (val.is_null()) {
out.reset();
return SUCCESS;
}
if (!out) {
out = std::make_shared<double>();
if (!out) { return MEMALLOC; }
@@ -451,6 +511,10 @@ error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr<double> &out)
}
error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr<std::string_view> &out) noexcept {
if (val.is_null()) {
out.reset();
return SUCCESS;
}
if (!out) {
out = std::make_shared<std::string_view>();
if (!out) { return MEMALLOC; }
@@ -460,6 +524,61 @@ error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr<std::string_vi
}
////////////////////////////////////////
// Explicit optional specializations
////////////////////////////////////////
////////////////////////////////////////
// Explicit smart pointer specializations for string and int types
////////////////////////////////////////
error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr<std::string> &out) noexcept {
// Check if the value is null
if (val.is_null()) {
out.reset(); // Set to nullptr
return SUCCESS;
}
if (!out) {
out = std::make_unique<std::string>();
}
std::string_view str;
SIMDJSON_TRY(val.get_string().get(str));
*out = std::string{str};
return SUCCESS;
}
error_code tag_invoke(deserialize_tag, auto &val, std::shared_ptr<std::string> &out) noexcept {
// Check if the value is null
if (val.is_null()) {
out.reset(); // Set to nullptr
return SUCCESS;
}
if (!out) {
out = std::make_shared<std::string>();
}
std::string_view str;
SIMDJSON_TRY(val.get_string().get(str));
*out = std::string{str};
return SUCCESS;
}
error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr<int> &out) noexcept {
// Check if the value is null
if (val.is_null()) {
out.reset(); // Set to nullptr
return SUCCESS;
}
if (!out) {
out = std::make_unique<int>();
}
int64_t temp;
SIMDJSON_TRY(val.get_int64().get(temp));
*out = static_cast<int>(temp);
return SUCCESS;
}
} // namespace simdjson
#endif // SIMDJSON_ONDEMAND_DESERIALIZE_H
+13
View File
@@ -75,6 +75,19 @@ public:
#if SIMDJSON_SUPPORTS_DESERIALIZATION
if constexpr (custom_deserializable<T, value>) {
return deserialize(*this, out);
} else if constexpr (concepts::optional_type<T>) {
using value_type = typename std::remove_cvref_t<T>::value_type;
// Check if the value is null
if (is_null()) {
out.reset(); // Set to nullopt
return SUCCESS;
}
if (!out) {
out.emplace();
}
return get<value_type>(out.value());
} else {
static_assert(!sizeof(T), "The get<T> method with type T is not implemented by the simdjson library. "
"And you do not seem to have added support for it. Indeed, we have that "
@@ -28,7 +28,6 @@ public:
) const noexcept final;
simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final;
simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final;
simdjson_warn_unused size_t write_string_escaped(const std::string_view input, char *out) const noexcept final;
};
} // namespace haswell
@@ -28,7 +28,6 @@ public:
) const noexcept final;
simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final;
simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final;
simdjson_warn_unused size_t write_string_escaped(const std::string_view input, char *out) const noexcept final;
};
} // namespace icelake
-17
View File
@@ -26,15 +26,6 @@ simdjson_inline simdjson_warn_unused bool validate_utf8(const std::string_view s
return validate_utf8(sv.data(), sv.size());
}
/**
* Write the string to the output buffer while escaping double-quote, backlash and ascii control characters.
*
* @param input the string_view to escape
* @param out output buffer (for escaped string): to be safe, it should have 6 * input.size() allocated bytes.
* @return number of bytes written
*/
simdjson_warn_unused size_t write_string_escaped(const std::string_view input, char *out) noexcept;
/**
* Validate the UTF-8 string.
*
@@ -136,14 +127,6 @@ public:
*/
simdjson_warn_unused virtual bool validate_utf8(const char *buf, size_t len) const noexcept = 0;
/**
* Write the string to the output buffer while escaping double-quote, backlash and ascii control characters.
*
* @param input the string_view to escape
* @param out output buffer (for escaped string): to be safe, it should have 6 * input.size() allocated bytes.
* @return number of bytes written
*/
simdjson_warn_unused virtual size_t write_string_escaped(const std::string_view input, char *out) const noexcept = 0;
protected:
/** @private Construct an implementation with the given name and description. For subclasses. */
simdjson_inline implementation(
-1
View File
@@ -23,7 +23,6 @@ public:
) const noexcept final;
simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final;
simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final;
simdjson_warn_unused size_t write_string_escaped(const std::string_view input, char *out) const noexcept final;
};
} // namespace lasx
-1
View File
@@ -23,7 +23,6 @@ public:
) const noexcept final;
simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final;
simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final;
simdjson_warn_unused size_t write_string_escaped(const std::string_view input, char *out) const noexcept final;
};
} // namespace lsx
+22
View File
@@ -12,6 +12,28 @@ namespace simdjson {
* @copydoc simdjson::builtin::builder
*/
namespace builder = builtin::builder;
#if SIMDJSON_STATIC_REFLECTION
/**
* Create a JSON string from any user-defined type using static reflection.
* Only available when SIMDJSON_STATIC_REFLECTION is enabled.
*/
template<typename T>
requires(!std::same_as<T, builtin::ondemand::document> &&
!std::same_as<T, builtin::ondemand::value> &&
!std::same_as<T, builtin::ondemand::object> &&
!std::same_as<T, builtin::ondemand::array>)
inline std::string to_json_string(const T& obj) {
builder::string_builder str_builder;
append(str_builder, obj);
std::string_view view;
if (str_builder.view().get(view) == SUCCESS) {
return std::string(view);
}
return "";
}
#endif
} // namespace simdjson
#endif // SIMDJSON_ONDEMAND_H
+2 -2
View File
@@ -125,9 +125,9 @@ inline const char *padded_string::data() const noexcept { return data_ptr; }
inline char *padded_string::data() noexcept { return data_ptr; }
inline padded_string::operator std::string_view() const { return std::string_view(data(), length()); }
inline padded_string::operator std::string_view() const simdjson_lifetime_bound { return std::string_view(data(), length()); }
inline padded_string::operator padded_string_view() const noexcept {
inline padded_string::operator padded_string_view() const noexcept simdjson_lifetime_bound {
return padded_string_view(data(), length(), length() + SIMDJSON_PADDING);
}
-1
View File
@@ -32,7 +32,6 @@ public:
size_t &dst_len) const noexcept final;
simdjson_warn_unused bool validate_utf8(const char *buf,
size_t len) const noexcept final;
simdjson_warn_unused size_t write_string_escaped(const std::string_view input, char *out) const noexcept final;
};
} // namespace ppc64
+3 -3
View File
@@ -4,18 +4,18 @@
#define SIMDJSON_SIMDJSON_VERSION_H
/** The version of simdjson being used (major.minor.revision) */
#define SIMDJSON_VERSION "3.13.0"
#define SIMDJSON_VERSION "4.0.0"
namespace simdjson {
enum {
/**
* The major version (MAJOR.minor.revision) of simdjson being used.
*/
SIMDJSON_VERSION_MAJOR = 3,
SIMDJSON_VERSION_MAJOR = 4,
/**
* The minor version (major.MINOR.revision) of simdjson being used.
*/
SIMDJSON_VERSION_MINOR = 13,
SIMDJSON_VERSION_MINOR = 0,
/**
* The revision (major.minor.REVISION) of simdjson being used.
*/
@@ -24,7 +24,6 @@ public:
) const noexcept final;
simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final;
simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final;
simdjson_warn_unused size_t write_string_escaped(const std::string_view input, char *out) const noexcept final;
};
} // namespace westmere
+9 -4
View File
@@ -296,10 +296,10 @@ class SimdjsonRepository:
class Amalgamator:
@classmethod
def amalgamate(cls, output_path: str, filename: str, roots: List[RelativeRoot], timestamp: str):
def amalgamate(cls, output_path: str, filename: str, roots: List[RelativeRoot], timestamp: str, version: str):
print(f"Creating {output_path}")
fid = open(output_path, 'w')
print(f"/* auto-generated on {timestamp}. Do not edit! */", file=fid)
print(f"/* auto-generated on {timestamp}. version {version} Do not edit! */", file=fid)
amalgamator = cls(fid, SimdjsonRepository(PROJECTPATH, roots))
file = amalgamator.repository[filename]
assert file, f"{filename} not found in {[os.path.join(PROJECTPATH, root) for root in roots]}!"
@@ -480,8 +480,13 @@ AMAL_C = os.path.join(AMALGAMATE_OUTPUT_PATH, "simdjson.cpp")
DEMOCPP = os.path.join(AMALGAMATE_OUTPUT_PATH, "amalgamate_demo.cpp")
README = os.path.join(AMALGAMATE_OUTPUT_PATH, "README.md")
Amalgamator.amalgamate(AMAL_H, "simdjson.h", ['include'], timestamp).validate_all_files_used('include')
Amalgamator.amalgamate(AMAL_C, "simdjson.cpp", ['src', 'include'], timestamp).validate_all_files_used('src')
def read_version():
with open(os.path.join(PROJECTPATH, 'include/simdjson/simdjson_version.h')) as f:
return re.search(r'\d+\.\d+\.\d+', f.read()).group(0)
version = read_version()
Amalgamator.amalgamate(AMAL_H, "simdjson.h", ['include'], timestamp, version).validate_all_files_used('include')
Amalgamator.amalgamate(AMAL_C, "simdjson.cpp", ['src', 'include'], timestamp, version).validate_all_files_used('src')
# copy the README and DEMOCPP
if SCRIPTPATH != AMALGAMATE_OUTPUT_PATH:
+482 -11
View File
@@ -1,4 +1,4 @@
/* auto-generated on 2025-06-24 15:13:44 -0400. Do not edit! */
/* auto-generated on 2025-07-14 15:43:52 -0400. Do not edit! */
/* including simdjson.cpp: */
/* begin file simdjson.cpp */
#define SIMDJSON_SRC_SIMDJSON_CPP
@@ -77,12 +77,31 @@
#endif
#endif
#ifndef SIMDJSON_CONSTEXPR_LAMBDA
#if SIMDJSON_CPLUSPLUS17
#define SIMDJSON_CONSTEXPR_LAMBDA constexpr
#else
#define SIMDJSON_CONSTEXPR_LAMBDA
#endif
#endif
#ifdef __has_include
#if __has_include(<version>)
#include <version>
#endif
#endif
// The current specification is unclear on how we detect
// static reflection, both __cpp_lib_reflection and
// __cpp_impl_reflection are proposed in the draft specification.
// For now, we disable static reflect by default. It must be
// specified at compiler time.
#ifndef SIMDJSON_STATIC_REFLECTION
#define SIMDJSON_STATIC_REFLECTION 0 // disabled by default.
#endif
#if defined(__apple_build_version__)
#if __apple_build_version__ < 14000000
#define SIMDJSON_CONCEPT_DISABLED 1 // apple-clang/13 doesn't support std::convertible_to
@@ -101,6 +120,14 @@
#define SIMDJSON_SUPPORTS_DESERIALIZATION 0
#endif // defined(__cpp_concepts) && !defined(SIMDJSON_CONCEPT_DISABLED)
#if !defined(SIMDJSON_CONSTEVAL)
#if defined(__cpp_consteval) && __cpp_consteval >= 201811L
#define SIMDJSON_CONSTEVAL 1
#else
#define SIMDJSON_CONSTEVAL 0
#endif // defined(__cpp_consteval) && __cpp_consteval >= 201811L
#endif // !defined(SIMDJSON_CONSTEVAL)
#endif // SIMDJSON_COMPILER_CHECK_H
/* end file simdjson/compiler_check.h */
/* including simdjson/portability.h: #include "simdjson/portability.h" */
@@ -298,6 +325,7 @@ using std::size_t;
#if defined(NDEBUG) || defined(__OPTIMIZE__) || (defined(_MSC_VER) && !defined(_DEBUG))
// If NDEBUG is set, or __OPTIMIZE__ is set, or we are under MSVC in release mode,
// then do away with asserts and use __assume.
// We still recommend that our users set NDEBUG in release mode.
#if SIMDJSON_VISUAL_STUDIO
#define SIMDJSON_UNREACHABLE() __assume(0)
#define SIMDJSON_ASSUME(COND) __assume(COND)
@@ -2342,16 +2370,25 @@ namespace std {
// It could also wrongly set SIMDJSON_DEVELOPMENT_CHECKS (e.g., if the programmer
// sets _DEBUG in a release build under Visual Studio, or if some compiler fails to
// set the __OPTIMIZE__ macro).
// We make it so that if NDEBUG is defined, then SIMDJSON_DEVELOPMENT_CHECKS
// is not defined, irrespective of the compiler.
// We recommend that users set NDEBUG in release builds, so that
// SIMDJSON_DEVELOPMENT_CHECKS is not defined in release builds by default,
// irrespective of the compiler.
#ifndef SIMDJSON_DEVELOPMENT_CHECKS
#ifdef _MSC_VER
// Visual Studio seems to set _DEBUG for debug builds.
#ifdef _DEBUG
// We set SIMDJSON_DEVELOPMENT_CHECKS to 1 if _DEBUG is defined
// and NDEBUG is not defined.
#if defined(_DEBUG) && !defined(NDEBUG)
#define SIMDJSON_DEVELOPMENT_CHECKS 1
#endif // _DEBUG
#else // _MSC_VER
// All other compilers appear to set __OPTIMIZE__ to a positive integer
// when the compiler is optimizing.
#ifndef __OPTIMIZE__
// We only set SIMDJSON_DEVELOPMENT_CHECKS if both __OPTIMIZE__
// and NDEBUG are not defined.
#if !defined(__OPTIMIZE__) && !defined(NDEBUG)
#define SIMDJSON_DEVELOPMENT_CHECKS 1
#endif // __OPTIMIZE__
#endif // _MSC_VER
@@ -2463,7 +2500,8 @@ enum error_code {
SCALAR_DOCUMENT_AS_VALUE, ///< A scalar document is treated as a value.
OUT_OF_BOUNDS, ///< Attempted to access location outside of document.
TRAILING_CONTENT, ///< Unexpected trailing content in the JSON input
NUM_ERROR_CODES
OUT_OF_CAPACITY, ///< The capacity was exceeded, we cannot allocate enough memory.
NUM_ERROR_CODES ///< Placeholder for end of error code list.
};
/**
@@ -8209,6 +8247,12 @@ namespace {
tmp = vpaddq_u8(tmp, tmp);
return vgetq_lane_u16(vreinterpretq_u16_u8(tmp), 0);
}
// Returns 4-bit out of each byte, alternating between the high 4 bits and low
// bits result it is 64 bit.
simdjson_inline uint64_t to_bitmask64() const {
return vget_lane_u64(
vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(*this), 4)), 0);
}
simdjson_inline bool any() const { return vmaxvq_u32(vreinterpretq_u32_u8(*this)) != 0; }
};
@@ -8285,7 +8329,7 @@ namespace {
// Bit-specific operations
simdjson_inline simd8<bool> any_bits_set(simd8<uint8_t> bits) const { return vtstq_u8(*this, bits); }
simdjson_inline bool any_bits_set_anywhere() const { return this->max_val() != 0; }
simdjson_inline bool any_bits_set_anywhere() const { return vmaxvq_u32(vreinterpretq_u32_u8(*this)) != 0; }
simdjson_inline bool any_bits_set_anywhere(simd8<uint8_t> bits) const { return (*this & bits).any_bits_set_anywhere(); }
template<int N>
simdjson_inline simd8<uint8_t> shr() const { return vshrq_n_u8(*this, N); }
@@ -8298,7 +8342,12 @@ namespace {
return lookup_table.apply_lookup_16_to(*this);
}
// Returns 4-bit out of each byte, alternating between the high 4 bits and low
// bits result it is 64 bit.
simdjson_inline uint64_t to_bitmask64() const {
return vget_lane_u64(
vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(*this), 4)), 0);
}
// Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset).
// Passing a 0 value for mask would be equivalent to writing out every byte to output.
// Only the first 16 - count_ones(mask) bytes of the result are significant but 16 bytes
@@ -8621,6 +8670,32 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin
};
}
struct escaping {
static constexpr uint32_t BYTES_PROCESSED = 16;
simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst);
simdjson_inline bool has_escape() { return escape_bits != 0; }
simdjson_inline int escape_index() { return trailing_zeroes(escape_bits) / 4; }
uint64_t escape_bits;
}; // struct escaping
simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) {
static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes");
simd8<uint8_t> v(src);
v.store(dst);
simd8<bool> is_quote = (v == '"');
simd8<bool> is_backslash = (v == '\\');
simd8<bool> is_control = (v < 32);
return {
(is_backslash | is_quote | is_control).to_bitmask64()
};
}
} // unnamed namespace
} // namespace arm64
} // namespace simdjson
@@ -10991,6 +11066,12 @@ namespace {
tmp = vpaddq_u8(tmp, tmp);
return vgetq_lane_u16(vreinterpretq_u16_u8(tmp), 0);
}
// Returns 4-bit out of each byte, alternating between the high 4 bits and low
// bits result it is 64 bit.
simdjson_inline uint64_t to_bitmask64() const {
return vget_lane_u64(
vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(*this), 4)), 0);
}
simdjson_inline bool any() const { return vmaxvq_u32(vreinterpretq_u32_u8(*this)) != 0; }
};
@@ -11067,7 +11148,7 @@ namespace {
// Bit-specific operations
simdjson_inline simd8<bool> any_bits_set(simd8<uint8_t> bits) const { return vtstq_u8(*this, bits); }
simdjson_inline bool any_bits_set_anywhere() const { return this->max_val() != 0; }
simdjson_inline bool any_bits_set_anywhere() const { return vmaxvq_u32(vreinterpretq_u32_u8(*this)) != 0; }
simdjson_inline bool any_bits_set_anywhere(simd8<uint8_t> bits) const { return (*this & bits).any_bits_set_anywhere(); }
template<int N>
simdjson_inline simd8<uint8_t> shr() const { return vshrq_n_u8(*this, N); }
@@ -11080,7 +11161,12 @@ namespace {
return lookup_table.apply_lookup_16_to(*this);
}
// Returns 4-bit out of each byte, alternating between the high 4 bits and low
// bits result it is 64 bit.
simdjson_inline uint64_t to_bitmask64() const {
return vget_lane_u64(
vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(*this), 4)), 0);
}
// Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset).
// Passing a 0 value for mask would be equivalent to writing out every byte to output.
// Only the first 16 - count_ones(mask) bytes of the result are significant but 16 bytes
@@ -11403,6 +11489,32 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin
};
}
struct escaping {
static constexpr uint32_t BYTES_PROCESSED = 16;
simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst);
simdjson_inline bool has_escape() { return escape_bits != 0; }
simdjson_inline int escape_index() { return trailing_zeroes(escape_bits) / 4; }
uint64_t escape_bits;
}; // struct escaping
simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) {
static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes");
simd8<uint8_t> v(src);
v.store(dst);
simd8<bool> is_quote = (v == '"');
simd8<bool> is_backslash = (v == '\\');
simd8<bool> is_control = (v < 32);
return {
(is_backslash | is_quote | is_control).to_bitmask64()
};
}
} // unnamed namespace
} // namespace arm64
} // namespace simdjson
@@ -13508,6 +13620,7 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V
/* end file generic/stage2/json_iterator.h for arm64 */
/* including generic/stage2/stringparsing.h for arm64: #include <generic/stage2/stringparsing.h> */
/* begin file generic/stage2/stringparsing.h for arm64 */
#include <cstdint>
#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H
/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */
@@ -13747,6 +13860,7 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t
}
} // namespace stringparsing
} // unnamed namespace
} // namespace arm64
} // namespace simdjson
@@ -14998,6 +15112,31 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin
};
}
struct escaping {
static constexpr uint32_t BYTES_PROCESSED = 32;
simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst);
simdjson_inline bool has_escape() { return escape_bits != 0; }
simdjson_inline int escape_index() { return trailing_zeroes(escape_bits); }
uint64_t escape_bits;
}; // struct escaping
simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) {
static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes");
simd8<uint8_t> v(src);
v.store(dst);
simd8<bool> is_quote = (v == '"');
simd8<bool> is_backslash = (v == '\\');
simd8<bool> is_control = (v < 32);
return {
uint64_t((is_backslash | is_quote | is_control).to_bitmask())
};
}
} // unnamed namespace
} // namespace haswell
} // namespace simdjson
@@ -17651,6 +17790,31 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin
};
}
struct escaping {
static constexpr uint32_t BYTES_PROCESSED = 32;
simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst);
simdjson_inline bool has_escape() { return escape_bits != 0; }
simdjson_inline int escape_index() { return trailing_zeroes(escape_bits); }
uint64_t escape_bits;
}; // struct escaping
simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) {
static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes");
simd8<uint8_t> v(src);
v.store(dst);
simd8<bool> is_quote = (v == '"');
simd8<bool> is_backslash = (v == '\\');
simd8<bool> is_control = (v < 32);
return {
uint64_t((is_backslash | is_quote | is_control).to_bitmask())
};
}
} // unnamed namespace
} // namespace haswell
} // namespace simdjson
@@ -19754,6 +19918,7 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V
/* end file generic/stage2/json_iterator.h for haswell */
/* including generic/stage2/stringparsing.h for haswell: #include <generic/stage2/stringparsing.h> */
/* begin file generic/stage2/stringparsing.h for haswell */
#include <cstdint>
#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H
/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */
@@ -19993,6 +20158,7 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t
}
} // namespace stringparsing
} // unnamed namespace
} // namespace haswell
} // namespace simdjson
@@ -20834,7 +21000,6 @@ namespace simd {
friend simdjson_really_inline uint64_t operator==(const simd8<T> lhs, const simd8<T> rhs) {
return _mm512_cmpeq_epi8_mask(lhs, rhs);
}
static const int SIZE = sizeof(base<T>::value);
template<int N=1>
@@ -21177,6 +21342,35 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin
};
}
struct escaping {
static constexpr uint32_t BYTES_PROCESSED = 64;
simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst);
simdjson_inline bool has_escape() { return escape_bits != 0; }
simdjson_inline int escape_index() { return trailing_zeroes(uint64_t(escape_bits)); }
__mmask64 escape_bits;
}; // struct escaping
simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) {
static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes");
simd8<uint8_t> v(src);
v.store(dst);
__mmask64 is_quote = _mm512_cmpeq_epi8_mask(v, _mm512_set1_epi8('"'));
__mmask64 is_backslash = _mm512_cmpeq_epi8_mask(v, _mm512_set1_epi8('\\'));
__mmask64 is_control = _mm512_cmplt_epi8_mask(v, _mm512_set1_epi8(32));
return {
(is_backslash | is_quote | is_control)
};
}
} // unnamed namespace
} // namespace icelake
} // namespace simdjson
@@ -23483,7 +23677,6 @@ namespace simd {
friend simdjson_really_inline uint64_t operator==(const simd8<T> lhs, const simd8<T> rhs) {
return _mm512_cmpeq_epi8_mask(lhs, rhs);
}
static const int SIZE = sizeof(base<T>::value);
template<int N=1>
@@ -23826,6 +24019,35 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin
};
}
struct escaping {
static constexpr uint32_t BYTES_PROCESSED = 64;
simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst);
simdjson_inline bool has_escape() { return escape_bits != 0; }
simdjson_inline int escape_index() { return trailing_zeroes(uint64_t(escape_bits)); }
__mmask64 escape_bits;
}; // struct escaping
simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) {
static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes");
simd8<uint8_t> v(src);
v.store(dst);
__mmask64 is_quote = _mm512_cmpeq_epi8_mask(v, _mm512_set1_epi8('"'));
__mmask64 is_backslash = _mm512_cmpeq_epi8_mask(v, _mm512_set1_epi8('\\'));
__mmask64 is_control = _mm512_cmplt_epi8_mask(v, _mm512_set1_epi8(32));
return {
(is_backslash | is_quote | is_control)
};
}
} // unnamed namespace
} // namespace icelake
} // namespace simdjson
@@ -25989,6 +26211,7 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V
/* end file generic/stage2/json_iterator.h for icelake */
/* including generic/stage2/stringparsing.h for icelake: #include <generic/stage2/stringparsing.h> */
/* begin file generic/stage2/stringparsing.h for icelake */
#include <cstdint>
#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H
/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */
@@ -26228,6 +26451,7 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t
}
} // namespace stringparsing
} // unnamed namespace
} // namespace icelake
} // namespace simdjson
@@ -27630,6 +27854,32 @@ backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) {
};
}
struct escaping {
static constexpr uint32_t BYTES_PROCESSED = 16;
simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst);
simdjson_inline bool has_escape() { return escape_bits != 0; }
simdjson_inline int escape_index() { return trailing_zeroes(escape_bits); }
uint64_t escape_bits;
}; // struct escaping
simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) {
static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes");
simd8<uint8_t> v(src);
v.store(dst);
simd8<bool> is_quote = (v == '"');
simd8<bool> is_backslash = (v == '\\');
simd8<bool> is_control = (v < 32);
return {
// We store it as a 64-bit bitmask even though we only need 16 bits.
uint64_t((is_backslash | is_quote | is_control).to_bitmask())
};
}
} // unnamed namespace
} // namespace ppc64
} // namespace simdjson
@@ -30394,6 +30644,32 @@ backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) {
};
}
struct escaping {
static constexpr uint32_t BYTES_PROCESSED = 16;
simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst);
simdjson_inline bool has_escape() { return escape_bits != 0; }
simdjson_inline int escape_index() { return trailing_zeroes(escape_bits); }
uint64_t escape_bits;
}; // struct escaping
simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) {
static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes");
simd8<uint8_t> v(src);
v.store(dst);
simd8<bool> is_quote = (v == '"');
simd8<bool> is_backslash = (v == '\\');
simd8<bool> is_control = (v < 32);
return {
// We store it as a 64-bit bitmask even though we only need 16 bits.
uint64_t((is_backslash | is_quote | is_control).to_bitmask())
};
}
} // unnamed namespace
} // namespace ppc64
} // namespace simdjson
@@ -32499,6 +32775,7 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V
/* end file generic/stage2/json_iterator.h for ppc64 */
/* including generic/stage2/stringparsing.h for ppc64: #include <generic/stage2/stringparsing.h> */
/* begin file generic/stage2/stringparsing.h for ppc64 */
#include <cstdint>
#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H
/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */
@@ -32738,6 +33015,7 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t
}
} // namespace stringparsing
} // unnamed namespace
} // namespace ppc64
} // namespace simdjson
@@ -34389,6 +34667,31 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin
};
}
struct escaping {
static constexpr uint32_t BYTES_PROCESSED = 16;
simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst);
simdjson_inline bool has_escape() { return escape_bits != 0; }
simdjson_inline int escape_index() { return trailing_zeroes(escape_bits); }
uint64_t escape_bits;
}; // struct escaping
simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) {
static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes");
simd8<uint8_t> v(src);
v.store(dst);
simd8<bool> is_quote = (v == '"');
simd8<bool> is_backslash = (v == '\\');
simd8<bool> is_control = (v < 32);
return {
uint64_t((is_backslash | is_quote | is_control).to_bitmask())
};
}
} // unnamed namespace
} // namespace westmere
} // namespace simdjson
@@ -37468,6 +37771,31 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin
};
}
struct escaping {
static constexpr uint32_t BYTES_PROCESSED = 16;
simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst);
simdjson_inline bool has_escape() { return escape_bits != 0; }
simdjson_inline int escape_index() { return trailing_zeroes(escape_bits); }
uint64_t escape_bits;
}; // struct escaping
simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) {
static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes");
simd8<uint8_t> v(src);
v.store(dst);
simd8<bool> is_quote = (v == '"');
simd8<bool> is_backslash = (v == '\\');
simd8<bool> is_control = (v < 32);
return {
uint64_t((is_backslash | is_quote | is_control).to_bitmask())
};
}
} // unnamed namespace
} // namespace westmere
} // namespace simdjson
@@ -39571,6 +39899,7 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V
/* end file generic/stage2/json_iterator.h for westmere */
/* including generic/stage2/stringparsing.h for westmere: #include <generic/stage2/stringparsing.h> */
/* begin file generic/stage2/stringparsing.h for westmere */
#include <cstdint>
#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H
/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */
@@ -39810,6 +40139,7 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t
}
} // namespace stringparsing
} // unnamed namespace
} // namespace westmere
} // namespace simdjson
@@ -40338,7 +40668,6 @@ simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *
if (error) { return error; }
return stage2(_doc);
}
} // namespace westmere
} // namespace simdjson
@@ -40968,6 +41297,31 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin
};
}
struct escaping {
static constexpr uint32_t BYTES_PROCESSED = 16;
simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst);
simdjson_inline bool has_escape() { return escape_bits != 0; }
simdjson_inline int escape_index() { return trailing_zeroes(escape_bits); }
uint64_t escape_bits;
}; // struct escaping
simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) {
static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes");
simd8<uint8_t> v(src);
v.store(dst);
simd8<bool> is_quote = (v == '"');
simd8<bool> is_backslash = (v == '\\');
simd8<bool> is_control = (v < 32);
return {
(is_backslash | is_quote | is_control).to_bitmask()
};
}
} // unnamed namespace
} // namespace lsx
} // namespace simdjson
@@ -43517,6 +43871,31 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin
};
}
struct escaping {
static constexpr uint32_t BYTES_PROCESSED = 16;
simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst);
simdjson_inline bool has_escape() { return escape_bits != 0; }
simdjson_inline int escape_index() { return trailing_zeroes(escape_bits); }
uint64_t escape_bits;
}; // struct escaping
simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) {
static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes");
simd8<uint8_t> v(src);
v.store(dst);
simd8<bool> is_quote = (v == '"');
simd8<bool> is_backslash = (v == '\\');
simd8<bool> is_control = (v < 32);
return {
(is_backslash | is_quote | is_control).to_bitmask()
};
}
} // unnamed namespace
} // namespace lsx
} // namespace simdjson
@@ -45622,6 +46001,7 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V
/* end file generic/stage2/json_iterator.h for lsx */
/* including generic/stage2/stringparsing.h for lsx: #include <generic/stage2/stringparsing.h> */
/* begin file generic/stage2/stringparsing.h for lsx */
#include <cstdint>
#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H
/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */
@@ -45861,6 +46241,7 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t
}
} // namespace stringparsing
} // unnamed namespace
} // namespace lsx
} // namespace simdjson
@@ -46994,6 +47375,31 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin
};
}
struct escaping {
static constexpr uint32_t BYTES_PROCESSED = 16;
simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst);
simdjson_inline bool has_escape() { return escape_bits != 0; }
simdjson_inline int escape_index() { return trailing_zeroes(escape_bits); }
uint64_t escape_bits;
}; // struct escaping
simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) {
static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes");
simd8<uint8_t> v(src);
v.store(dst);
simd8<bool> is_quote = (v == '"');
simd8<bool> is_backslash = (v == '\\');
simd8<bool> is_control = (v < 32);
return {
(is_backslash | is_quote | is_control).to_bitmask()
};
}
} // unnamed namespace
} // namespace lasx
} // namespace simdjson
@@ -49559,6 +49965,31 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin
};
}
struct escaping {
static constexpr uint32_t BYTES_PROCESSED = 16;
simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst);
simdjson_inline bool has_escape() { return escape_bits != 0; }
simdjson_inline int escape_index() { return trailing_zeroes(escape_bits); }
uint64_t escape_bits;
}; // struct escaping
simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) {
static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "escaping finder must process fewer than SIMDJSON_PADDING bytes");
simd8<uint8_t> v(src);
v.store(dst);
simd8<bool> is_quote = (v == '"');
simd8<bool> is_backslash = (v == '\\');
simd8<bool> is_control = (v < 32);
return {
(is_backslash | is_quote | is_control).to_bitmask()
};
}
} // unnamed namespace
} // namespace lasx
} // namespace simdjson
@@ -51664,6 +52095,7 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V
/* end file generic/stage2/json_iterator.h for lasx */
/* including generic/stage2/stringparsing.h for lasx: #include <generic/stage2/stringparsing.h> */
/* begin file generic/stage2/stringparsing.h for lasx */
#include <cstdint>
#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H
/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */
@@ -51903,6 +52335,7 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t
}
} // namespace stringparsing
} // unnamed namespace
} // namespace lasx
} // namespace simdjson
@@ -52532,6 +52965,24 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin
return { src[0] };
}
struct escaping {
static constexpr uint32_t BYTES_PROCESSED = 1;
simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst);
simdjson_inline bool has_escape() { return escape_bits; }
simdjson_inline int escape_index() { return 0; }
bool escape_bits;
}; // struct escaping
simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) {
dst[0] = src[0];
return { (src[0] == '\\') || (src[0] == '"') || (src[0] < 32) };
}
} // unnamed namespace
} // namespace fallback
} // namespace simdjson
@@ -54686,6 +55137,24 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin
return { src[0] };
}
struct escaping {
static constexpr uint32_t BYTES_PROCESSED = 1;
simdjson_inline static escaping copy_and_find(const uint8_t *src, uint8_t *dst);
simdjson_inline bool has_escape() { return escape_bits; }
simdjson_inline int escape_index() { return 0; }
bool escape_bits;
}; // struct escaping
simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *dst) {
dst[0] = src[0];
return { (src[0] == '\\') || (src[0] == '"') || (src[0] < 32) };
}
} // unnamed namespace
} // namespace fallback
} // namespace simdjson
@@ -54892,6 +55361,7 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par
/* end file generic/stage1/find_next_document_index.h for fallback */
/* including generic/stage2/stringparsing.h for fallback: #include <generic/stage2/stringparsing.h> */
/* begin file generic/stage2/stringparsing.h for fallback */
#include <cstdint>
#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H
/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */
@@ -55131,6 +55601,7 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t
}
} // namespace stringparsing
} // unnamed namespace
} // namespace fallback
} // namespace simdjson
+11642 -53
View File
File diff suppressed because it is too large Load Diff
Binary file not shown.
-4
View File
@@ -165,10 +165,6 @@ simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *
return stage2(_doc);
}
simdjson_warn_unused size_t implementation::write_string_escaped(const std::string_view input, char *out) const noexcept {
return arm64::stringparsing::write_string_escaped(input, out);
}
} // namespace arm64
} // namespace simdjson
-4
View File
@@ -459,10 +459,6 @@ simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *
return stage2(_doc);
}
simdjson_warn_unused size_t implementation::write_string_escaped(const std::string_view input, char *out) const noexcept {
return fallback::stringparsing::write_string_escaped(input, out);
}
} // namespace fallback
} // namespace simdjson
-98
View File
@@ -237,104 +237,6 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t
}
}
/////////////
/// TODO: This function is not used in the codebase. It is not clear if it is needed.
/////////////
simdjson_warn_unused size_t write_string_escaped(const std::string_view input, char *out) noexcept {
// We are making the following assumption: most strings will either be very short or they will not
// need escaping.
size_t i = 0;
size_t pos = 0;
/*if(input.size() >= escaping::BYTES_PROCESSED) {
auto vec_processing = [input,out]() -> size_t {
size_t index = 0;
size_t position = 0;
for(;input.size() - index >= escaping::BYTES_PROCESSED; index += escaping::BYTES_PROCESSED) {
escaping vinput = escaping::copy_and_find(reinterpret_cast<const uint8_t *>(input.data()) + index, reinterpret_cast<uint8_t *>(out) + position);
if(vinput.has_escape()) {
return index + vinput.escape_index(); // We have a character that needs escaping
}
position += escaping::BYTES_PROCESSED;
}
if(index == input.size()) { return input.size(); }
// We virtually backtrack so we can load a full vector register
index = input.size() - escaping::BYTES_PROCESSED;
position = index;
escaping vinput = escaping::copy_and_find(reinterpret_cast<const uint8_t *>(input.data()) + index, reinterpret_cast<uint8_t *>(out) + position);
if(vinput.has_escape()) {
return index + vinput.escape_index(); // We have a character that needs escaping
}
return input.size();
};
i = vec_processing();
pos = i;
if(i == input.size()) { return pos; }
// Here we only continue if there was a character that needed escaping.
}*/
static std::string_view control_chars[] = {
"\\x0000", "\\x0001", "\\x0002", "\\x0003", "\\x0004", "\\x0005", "\\x0006",
"\\x0007", "\\x0008", "\\t", "\\n", "\\x000b", "\\f", "\\r",
"\\x000e", "\\x000f", "\\x0010", "\\x0011", "\\x0012", "\\x0013", "\\x0014",
"\\x0015", "\\x0016", "\\x0017", "\\x0018", "\\x0019", "\\x001a", "\\x001b",
"\\x001c", "\\x001d", "\\x001e", "\\x001f"};
static std::array<uint8_t, 256> json_quotable_character =
[]() SIMDJSON_CONSTEXPR_LAMBDA {
std::array<uint8_t, 256> result{};
for (int index = 0; index < 32; index++) {
result[index] = 1;
}
for (int index : {'"', '\\'}) {
result[index] = 1;
}
return result;
}();
// The rest could possibly be vectorized, but consider that we expect most strings
// to be short or not to require escaping.
for (; i < input.size(); i++) {
uint8_t c = static_cast<uint8_t>(input[i]);
if(json_quotable_character[c]) {
switch (c) {
case '"':
out[pos++] = '\\';
out[pos++] = '"';
break;
case '\\':
out[pos++] = '\\';
out[pos++] = '\\';
break;
case '\b':
out[pos++] = '\\';
out[pos++] = 'b';
break;
case '\f':
out[pos++] = '\\';
out[pos++] = 'f';
break;
case '\n':
out[pos++] = '\\';
out[pos++] = 'n';
break;
case '\r':
out[pos++] = '\\';
out[pos++] = 'r';
break;
case '\t':
out[pos++] = '\\';
out[pos++] = 't';
break;
default:
control_chars[c].copy(out + pos, 6);
pos += 6;
}
} else {
out[pos++] = c;
}
}
return pos;
}
} // namespace stringparsing
} // unnamed namespace
-4
View File
@@ -162,10 +162,6 @@ simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *
return stage2(_doc);
}
simdjson_warn_unused size_t implementation::write_string_escaped(const std::string_view input, char *out) const noexcept {
return haswell::stringparsing::write_string_escaped(input, out);
}
} // namespace SIMDJSON_IMPLEMENTATION
} // namespace simdjson
-4
View File
@@ -208,10 +208,6 @@ simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *
return stage2(_doc);
}
simdjson_warn_unused size_t implementation::write_string_escaped(const std::string_view input, char *out) const noexcept {
return icelake::stringparsing::write_string_escaped(input, out);
}
} // namespace icelake
} // namespace simdjson
-9
View File
@@ -186,9 +186,6 @@ public:
simdjson_warn_unused bool validate_utf8(const char * buf, size_t len) const noexcept final override {
return set_best()->validate_utf8(buf, len);
}
simdjson_warn_unused size_t write_string_escaped(const std::string_view input, char *out) const noexcept final {
return set_best()->write_string_escaped(input, out);
}
simdjson_inline detect_best_supported_implementation_on_first_use() noexcept : implementation("best_supported_detector", "Detects the best supported implementation and sets it", 0) {}
private:
const implementation *set_best() const noexcept;
@@ -239,9 +236,6 @@ public:
simdjson_warn_unused error_code minify(const uint8_t *, size_t, uint8_t *, size_t &) const noexcept final override {
return UNSUPPORTED_ARCHITECTURE;
}
simdjson_warn_unused size_t write_string_escaped(const std::string_view, char *) const noexcept final override {
return 0; // TODO: Evaluate whether this is the right thing to do for unsupported architecture.
}
simdjson_warn_unused bool validate_utf8(const char *, size_t) const noexcept final override {
return false; // Just refuse to validate. Given that we have a fallback implementation
// it seems unlikely that unsupported_implementation will ever be used. If it is used,
@@ -325,9 +319,6 @@ simdjson_warn_unused error_code minify(const char *buf, size_t len, char *dst, s
simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) noexcept {
return get_active_implementation()->validate_utf8(buf, len);
}
simdjson_warn_unused size_t write_string_escaped(const std::string_view input, char *out) noexcept {
return get_active_implementation()->write_string_escaped(input, out);
}
const implementation * builtin_implementation() {
static const implementation * builtin_impl = get_available_implementations()[SIMDJSON_STRINGIFY(SIMDJSON_BUILTIN_IMPLEMENTATION)];
assert(builtin_impl);
-4
View File
@@ -125,10 +125,6 @@ simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *
return stage2(_doc);
}
simdjson_warn_unused size_t implementation::write_string_escaped(const std::string_view input, char *out) const noexcept {
return lasx::stringparsing::write_string_escaped(input, out);
}
} // namespace lasx
} // namespace simdjson
-4
View File
@@ -129,10 +129,6 @@ simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *
return stage2(_doc);
}
simdjson_warn_unused size_t implementation::write_string_escaped(const std::string_view input, char *out) const noexcept {
return lsx::stringparsing::write_string_escaped(input, out);
}
} // namespace lsx
} // namespace simdjson
-4
View File
@@ -135,10 +135,6 @@ simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *
return stage2(_doc);
}
simdjson_warn_unused size_t implementation::write_string_escaped(const std::string_view input, char *out) const noexcept {
return ppc64::stringparsing::write_string_escaped(input, out);
}
} // namespace ppc64
} // namespace simdjson
-5
View File
@@ -166,11 +166,6 @@ simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *
if (error) { return error; }
return stage2(_doc);
}
simdjson_warn_unused size_t implementation::write_string_escaped(const std::string_view input, char *out) const noexcept {
return westmere::stringparsing::write_string_escaped(input, out);
}
} // namespace westmere
} // namespace simdjson
+3
View File
@@ -3,6 +3,9 @@ include_directories(..)
add_cpp_test(builder_string_builder_tests LABELS ondemand acceptance per_implementation)
if(SIMDJSON_STATIC_REFLECTION)
add_cpp_test(static_reflection_builder_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(static_reflection_comprehensive_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(static_reflection_edge_cases_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(static_reflection_enum_tests LABELS ondemand acceptance per_implementation)
endif(SIMDJSON_STATIC_REFLECTION)
# Copy the simdjson dll into the tests directory
if(MSVC AND BUILD_SHARED_LIBS)
@@ -270,6 +270,7 @@ namespace builder_tests {
}
void serialize_car(const Car& car, simdjson::builder::string_builder& builder) {
// start of JSON
builder.start_object();
@@ -318,6 +319,17 @@ namespace builder_tests {
TEST_SUCCEED();
}
#if SIMDJSON_EXCEPTIONS
bool car_test_exception() {
TEST_START();
simdjson::builder::string_builder sb;
Car c = {"Toyota", "Corolla", 2017, {30.0,30.2,30.513,30.79}};
serialize_car_long(c, sb);
std::string_view p{sb};
ASSERT_EQUAL(p, "{\"make\":\"Toyota\",\"model\":\"Corolla\",\"year\":2017,\"tire_pressure\":[30.0,30.2,30.513,30.79]}");
TEST_SUCCEED();
}
#endif
bool run() {
return
various_integers() &&
@@ -325,6 +337,7 @@ namespace builder_tests {
car_test_long() &&
car_test() &&
#if SIMDJSON_EXCEPTIONS
car_test_exception() &&
string_convertion_except() &&
#endif
append_char() &&
@@ -48,14 +48,11 @@ namespace builder_tests {
bool car_test() {
TEST_START();
simdjson::builder::string_builder sb;
#if SIMDJSON_STATIC_REFLECTION
Car c = {"Toyota", "Corolla", 2017, {30.0,30.2,30.513,30.79}};
append(sb, c);
std::string_view p;
auto result = sb.view().get(p);
auto result = builder::to_json_string(c);
ASSERT_SUCCESS(result);
ASSERT_EQUAL(p, "{\"make\":\"Toyota\",\"model\":\"Corolla\",\"year\":2017,\"tire_pressure\":[30.0,30.2,30.513,30.79]}");
std::string pstr(p.begin(), p.end());
std::string pstr = result.value();
ASSERT_EQUAL(pstr, "{\"make\":\"Toyota\",\"model\":\"Corolla\",\"year\":2017,\"tire_pressure\":[30.0,30.2,30.513,30.79]}");
simdjson::ondemand::parser parser;
simdjson::ondemand::document doc;
@@ -70,12 +67,38 @@ namespace builder_tests {
ASSERT_EQUAL(c2.tire_pressure[1], 30.2);
ASSERT_EQUAL(c2.tire_pressure[2], 30.513);
ASSERT_EQUAL(c2.tire_pressure[3], 30.79);
#endif
TEST_SUCCEED();
}
#if SIMDJSON_EXCEPTIONS
bool car_test_exception() {
TEST_START();
simdjson::builder::string_builder sb;
Car c = {"Toyota", "Corolla", 2017, {30.0,30.2,30.513,30.79}};
append(sb, c);
std::string_view p{sb};
TEST_SUCCEED();
}
bool car_test_exception2() {
TEST_START();
simdjson::builder::string_builder sb;
Car c = {"Toyota", "Corolla", 2017, {30.0,30.2,30.513,30.79}};
sb << c;
std::string_view p{sb};
TEST_SUCCEED();
}
void car_test_to_json_exception() {
TEST_START();
Car c = {"Toyota", "Corolla", 2017, {30.0,30.2,30.513,30.79}};
std::string json = simdjson::builder::to_json_string(c);
TEST_SUCCEED();
}
#endif // SIMDJSON_EXCEPTIONS
bool serialize_deserialize_kid() {
TEST_START();
#if SIMDJSON_STATIC_REFLECTION
simdjson::padded_string json_str =
R"({"age": 12, "name": "John", "toys": ["car", "ball"]})"_padded;
simdjson::ondemand::parser parser;
@@ -90,7 +113,7 @@ bool serialize_deserialize_kid() {
ASSERT_EQUAL(k.toys[1], "ball");
// Now, go the other direction:
std::string json;
ASSERT_SUCCESS(simdjson::builder::to_json_string(k).get(json));
ASSERT_SUCCESS(builder::to_json_string(k).get(json));
std::cout << json << std::endl;
// Now we parse it back:
simdjson::ondemand::parser parser2;
@@ -103,11 +126,13 @@ bool serialize_deserialize_kid() {
ASSERT_EQUAL(k2.toys.size(), 2);
ASSERT_EQUAL(k2.toys[0], "car");
ASSERT_EQUAL(k2.toys[1], "ball");
#endif
TEST_SUCCEED();
}
bool serialize_deserialize_x_y_z() {
TEST_START();
#if SIMDJSON_STATIC_REFLECTION
X s1 = {.a = '1',
.b = 10,
.c = 0,
@@ -119,7 +144,7 @@ bool serialize_deserialize_x_y_z() {
.i = {1, 2, 3},
.z = {.x = 1000}}};
std::string pstr;
ASSERT_SUCCESS(simdjson::builder::to_json_string(s1).get(pstr));
ASSERT_SUCCESS(builder::to_json_string(s1).get(pstr));
ASSERT_EQUAL(
pstr,
R"({"a":"1","b":10,"c":0,"d":"test string\n\r\"","e":[1,2,3],"f":["ab","cd","fg"],"y":{"g":100,"h":"test string\n\r\"","i":[1,2,3],"z":{"x":1000}}})");
@@ -129,12 +154,23 @@ bool serialize_deserialize_x_y_z() {
X s2;
ASSERT_SUCCESS(doc.get<X>().get(s2));
ASSERT_TRUE(s1 == s2);
#endif
TEST_SUCCEED();
}
bool run() {
return car_test() && serialize_deserialize_kid() && serialize_deserialize_x_y_z() && true;
}
bool run() {
return
#if SIMDJSON_EXCEPTIONS
car_test_exception() &&
car_test_exception2() &&
car_test_to_json_exception() &&
#endif // SIMDJSON_EXCEPTIONS
car_test() &&
serialize_deserialize_kid() &&
serialize_deserialize_x_y_z() &&
true;
}
} // namespace builder_tests
@@ -0,0 +1,231 @@
#include "simdjson.h"
#include "test_builder.h"
#include <string>
#include <string_view>
#include <vector>
#include <set>
#include <map>
#include <optional>
#include <memory>
using namespace simdjson;
namespace builder_tests {
bool test_primitive_types() {
TEST_START();
#if SIMDJSON_STATIC_REFLECTION
struct PrimitiveTypes {
bool bool_val;
char char_val;
int int_val;
double double_val;
float float_val;
};
PrimitiveTypes test{true, 'X', 42, 3.14159, 2.71f};
auto result = builder::to_json_string(test);
ASSERT_SUCCESS(result);
std::string json = result.value();
ASSERT_TRUE(json.find("\"bool_val\":true") != std::string::npos);
ASSERT_TRUE(json.find("\"char_val\":\"X\"") != std::string::npos);
ASSERT_TRUE(json.find("\"int_val\":42") != std::string::npos);
ASSERT_TRUE(json.find("\"double_val\":3.14159") != std::string::npos);
// Test round-trip
ondemand::parser parser;
auto doc_result = parser.iterate(pad(json));
ASSERT_SUCCESS(doc_result);
auto get_result = doc_result.value().get<PrimitiveTypes>();
ASSERT_SUCCESS(get_result);
PrimitiveTypes deserialized = std::move(get_result.value());
ASSERT_EQUAL(deserialized.bool_val, test.bool_val);
ASSERT_EQUAL(deserialized.char_val, test.char_val);
ASSERT_EQUAL(deserialized.int_val, test.int_val);
ASSERT_EQUAL(deserialized.double_val, test.double_val);
#endif
TEST_SUCCEED();
}
bool test_string_types() {
TEST_START();
#if SIMDJSON_STATIC_REFLECTION
struct StringTypes {
std::string string_val;
std::string_view string_view_val;
};
StringTypes test{"hello world", "test_view"};
auto result = builder::to_json_string(test);
ASSERT_SUCCESS(result);
std::string json = result.value();
ASSERT_TRUE(json.find("\"string_val\":\"hello world\"") != std::string::npos);
ASSERT_TRUE(json.find("\"string_view_val\":\"test_view\"") != std::string::npos);
// Test round-trip
ondemand::parser parser;
auto doc_result = parser.iterate(pad(json));
ASSERT_SUCCESS(doc_result);
auto get_result = doc_result.value().get<StringTypes>();
ASSERT_SUCCESS(get_result);
StringTypes deserialized = std::move(get_result.value());
ASSERT_EQUAL(deserialized.string_val, test.string_val);
#endif
TEST_SUCCEED();
}
bool test_optional_types() {
TEST_START();
#if SIMDJSON_STATIC_REFLECTION
struct OptionalTypes {
std::optional<int> opt_int_with_value;
std::optional<std::string> opt_string_with_value;
std::optional<int> opt_int_null;
std::optional<std::string> opt_string_null;
};
OptionalTypes test;
test.opt_int_with_value = 42;
test.opt_string_with_value = "optional_test";
test.opt_int_null = std::nullopt;
test.opt_string_null = std::nullopt;
auto result = builder::to_json_string(test);
ASSERT_SUCCESS(result);
std::string json = result.value();
ASSERT_TRUE(json.find("\"opt_int_with_value\":42") != std::string::npos);
ASSERT_TRUE(json.find("\"opt_string_with_value\":\"optional_test\"") != std::string::npos);
ASSERT_TRUE(json.find("\"opt_int_null\":null") != std::string::npos);
ASSERT_TRUE(json.find("\"opt_string_null\":null") != std::string::npos);
// Test round-trip
ondemand::parser parser;
auto doc_result = parser.iterate(pad(json));
ASSERT_SUCCESS(doc_result);
auto get_result = doc_result.value().get<OptionalTypes>();
ASSERT_SUCCESS(get_result);
OptionalTypes deserialized = std::move(get_result.value());
ASSERT_TRUE(deserialized.opt_int_with_value.has_value());
ASSERT_EQUAL(*deserialized.opt_int_with_value, 42);
ASSERT_TRUE(deserialized.opt_string_with_value.has_value());
ASSERT_EQUAL(*deserialized.opt_string_with_value, "optional_test");
ASSERT_FALSE(deserialized.opt_int_null.has_value());
ASSERT_FALSE(deserialized.opt_string_null.has_value());
#endif
TEST_SUCCEED();
}
bool test_smart_pointer_types() {
TEST_START();
#if SIMDJSON_STATIC_REFLECTION
struct SmartPointerTypes {
std::unique_ptr<int> unique_int_with_value;
std::shared_ptr<std::string> shared_string_with_value;
std::unique_ptr<bool> unique_bool_with_value;
std::unique_ptr<int> unique_int_null;
std::shared_ptr<std::string> shared_string_null;
};
SmartPointerTypes test;
test.unique_int_with_value = std::make_unique<int>(123);
test.shared_string_with_value = std::make_shared<std::string>("shared_test");
test.unique_bool_with_value = std::make_unique<bool>(true);
test.unique_int_null = nullptr;
test.shared_string_null = nullptr;
auto result = builder::to_json_string(test);
ASSERT_SUCCESS(result);
std::string json = result.value();
ASSERT_TRUE(json.find("\"unique_int_with_value\":123") != std::string::npos);
ASSERT_TRUE(json.find("\"shared_string_with_value\":\"shared_test\"") != std::string::npos);
ASSERT_TRUE(json.find("\"unique_bool_with_value\":true") != std::string::npos);
ASSERT_TRUE(json.find("\"unique_int_null\":null") != std::string::npos);
ASSERT_TRUE(json.find("\"shared_string_null\":null") != std::string::npos);
// Test round-trip
ondemand::parser parser;
auto doc_result = parser.iterate(pad(json));
ASSERT_SUCCESS(doc_result);
auto get_result = doc_result.value().get<SmartPointerTypes>();
ASSERT_SUCCESS(get_result);
SmartPointerTypes deserialized = std::move(get_result.value());
ASSERT_TRUE(deserialized.unique_int_with_value != nullptr);
ASSERT_EQUAL(*deserialized.unique_int_with_value, 123);
ASSERT_TRUE(deserialized.shared_string_with_value != nullptr);
ASSERT_EQUAL(*deserialized.shared_string_with_value, "shared_test");
ASSERT_TRUE(deserialized.unique_bool_with_value != nullptr);
ASSERT_EQUAL(*deserialized.unique_bool_with_value, true);
ASSERT_TRUE(deserialized.unique_int_null == nullptr);
ASSERT_TRUE(deserialized.shared_string_null == nullptr);
#endif
TEST_SUCCEED();
}
bool test_container_types() {
TEST_START();
#if SIMDJSON_STATIC_REFLECTION
struct ContainerTypes {
std::vector<int> int_vector;
std::set<std::string> string_set;
std::map<std::string, int> string_map;
};
ContainerTypes test;
test.int_vector = {1, 2, 3, 4, 5};
test.string_set = {"apple", "banana", "cherry"};
test.string_map = {{"key1", 10}, {"key2", 20}, {"key3", 30}};
auto result = builder::to_json_string(test);
ASSERT_SUCCESS(result);
std::string json = result.value();
ASSERT_TRUE(json.find("\"int_vector\":[1,2,3,4,5]") != std::string::npos);
ASSERT_TRUE(json.find("\"string_set\":[") != std::string::npos);
ASSERT_TRUE(json.find("\"string_map\":{") != std::string::npos);
// Test round-trip
ondemand::parser parser;
auto doc_result = parser.iterate(pad(json));
ASSERT_SUCCESS(doc_result);
auto get_result = doc_result.value().get<ContainerTypes>();
ASSERT_SUCCESS(get_result);
ContainerTypes deserialized = std::move(get_result.value());
ASSERT_EQUAL(deserialized.int_vector.size(), 5);
ASSERT_EQUAL(deserialized.string_set.size(), 3);
ASSERT_EQUAL(deserialized.string_map.size(), 3);
ASSERT_EQUAL(deserialized.int_vector[0], 1);
ASSERT_EQUAL(deserialized.int_vector[4], 5);
#endif
TEST_SUCCEED();
}
bool run() {
return test_primitive_types() &&
test_string_types() &&
test_optional_types() &&
test_smart_pointer_types() &&
test_container_types();
}
} // namespace builder_tests
int main(int argc, char *argv[]) {
return test_main(argc, argv, builder_tests::run);
}
@@ -0,0 +1,230 @@
#include "simdjson.h"
#include "test_builder.h"
#include <string>
#include <vector>
#include <optional>
#include <memory>
#include <limits>
using namespace simdjson;
namespace builder_tests {
bool test_empty_values() {
TEST_START();
#if SIMDJSON_STATIC_REFLECTION
struct EmptyValues {
std::string empty_string;
std::vector<int> empty_vector;
std::optional<int> null_optional;
std::unique_ptr<int> null_unique_ptr;
std::shared_ptr<std::string> null_shared_ptr;
};
EmptyValues test;
test.empty_string = "";
// empty_vector is already empty by default
test.null_optional = std::nullopt;
test.null_unique_ptr = nullptr;
test.null_shared_ptr = nullptr;
auto result = builder::to_json_string(test);
ASSERT_SUCCESS(result);
std::string json = result.value();
ASSERT_TRUE(json.find("\"empty_string\":\"\"") != std::string::npos);
ASSERT_TRUE(json.find("\"empty_vector\":[]") != std::string::npos);
ASSERT_TRUE(json.find("\"null_optional\":null") != std::string::npos);
ASSERT_TRUE(json.find("\"null_unique_ptr\":null") != std::string::npos);
ASSERT_TRUE(json.find("\"null_shared_ptr\":null") != std::string::npos);
// Test round-trip
ondemand::parser parser;
auto doc_result = parser.iterate(pad(json));
ASSERT_SUCCESS(doc_result);
auto get_result = doc_result.value().get<EmptyValues>();
ASSERT_SUCCESS(get_result);
EmptyValues deserialized = std::move(get_result.value());
ASSERT_EQUAL(deserialized.empty_string, "");
ASSERT_EQUAL(deserialized.empty_vector.size(), 0);
ASSERT_FALSE(deserialized.null_optional.has_value());
ASSERT_TRUE(deserialized.null_unique_ptr == nullptr);
ASSERT_TRUE(deserialized.null_shared_ptr == nullptr);
#endif
TEST_SUCCEED();
}
bool test_special_characters() {
TEST_START();
#if SIMDJSON_STATIC_REFLECTION
struct SpecialChars {
std::string quotes;
std::string backslashes;
std::string newlines;
std::string unicode;
char null_char;
};
SpecialChars test;
test.quotes = "He said \"Hello\"";
test.backslashes = "Path\\to\\file";
test.newlines = "Line1\nLine2\tTabbed";
test.unicode = "Café résumé";
test.null_char = '\0';
auto result = builder::to_json_string(test);
ASSERT_SUCCESS(result);
std::string json = result.value();
// Test that quotes are properly escaped
ASSERT_TRUE(json.find("\\\"Hello\\\"") != std::string::npos);
// Test that backslashes are properly escaped
ASSERT_TRUE(json.find("\\\\to\\\\") != std::string::npos);
// Test that newlines are properly escaped
ASSERT_TRUE(json.find("\\n") != std::string::npos);
ASSERT_TRUE(json.find("\\t") != std::string::npos);
// Test round-trip (excluding null char which has special handling)
struct SpecialCharsNoNull {
std::string quotes;
std::string backslashes;
std::string newlines;
std::string unicode;
};
SpecialCharsNoNull test_no_null;
test_no_null.quotes = test.quotes;
test_no_null.backslashes = test.backslashes;
test_no_null.newlines = test.newlines;
test_no_null.unicode = test.unicode;
auto result_no_null = builder::to_json_string(test_no_null);
ASSERT_SUCCESS(result_no_null);
ondemand::parser parser;
auto doc_result = parser.iterate(pad(result_no_null.value()));
ASSERT_SUCCESS(doc_result);
auto get_result = doc_result.value().get<SpecialCharsNoNull>();
ASSERT_SUCCESS(get_result);
SpecialCharsNoNull deserialized = std::move(get_result.value());
ASSERT_EQUAL(deserialized.quotes, test.quotes);
ASSERT_EQUAL(deserialized.backslashes, test.backslashes);
ASSERT_EQUAL(deserialized.newlines, test.newlines);
ASSERT_EQUAL(deserialized.unicode, test.unicode);
#endif
TEST_SUCCEED();
}
bool test_numeric_limits() {
TEST_START();
#if SIMDJSON_STATIC_REFLECTION
struct NumericLimits {
int max_int;
int min_int;
double max_double;
double min_double;
bool true_val;
bool false_val;
};
NumericLimits test;
test.max_int = std::numeric_limits<int>::max();
test.min_int = std::numeric_limits<int>::min();
test.max_double = 1e100; // Large but safe double value
test.min_double = -1e100; // Large negative but safe double value
test.true_val = true;
test.false_val = false;
auto result = builder::to_json_string(test);
ASSERT_SUCCESS(result);
std::string json = result.value();
ASSERT_TRUE(json.find("\"true_val\":true") != std::string::npos);
ASSERT_TRUE(json.find("\"false_val\":false") != std::string::npos);
// Test round-trip
ondemand::parser parser;
auto doc_result = parser.iterate(pad(json));
ASSERT_SUCCESS(doc_result);
auto get_result = doc_result.value().get<NumericLimits>();
ASSERT_SUCCESS(get_result);
NumericLimits deserialized = std::move(get_result.value());
ASSERT_EQUAL(deserialized.max_int, test.max_int);
ASSERT_EQUAL(deserialized.min_int, test.min_int);
ASSERT_EQUAL(deserialized.true_val, true);
ASSERT_EQUAL(deserialized.false_val, false);
#endif
TEST_SUCCEED();
}
bool test_nested_structures() {
TEST_START();
#if SIMDJSON_STATIC_REFLECTION
struct Inner {
int value;
std::string name;
};
struct Outer {
Inner inner_obj;
std::vector<Inner> inner_vector;
std::optional<Inner> optional_inner;
std::unique_ptr<Inner> unique_inner;
};
Outer test;
test.inner_obj = {42, "inner"};
test.inner_vector = {{1, "first"}, {2, "second"}};
test.optional_inner = Inner{99, "optional"};
test.unique_inner = std::make_unique<Inner>(Inner{123, "unique"});
auto result = builder::to_json_string(test);
ASSERT_SUCCESS(result);
std::string json = result.value();
ASSERT_TRUE(json.find("\"inner_obj\":{") != std::string::npos);
ASSERT_TRUE(json.find("\"inner_vector\":[") != std::string::npos);
ASSERT_TRUE(json.find("\"optional_inner\":{") != std::string::npos);
ASSERT_TRUE(json.find("\"unique_inner\":{") != std::string::npos);
// Test round-trip
ondemand::parser parser;
auto doc_result = parser.iterate(pad(json));
ASSERT_SUCCESS(doc_result);
auto get_result = doc_result.value().get<Outer>();
ASSERT_SUCCESS(get_result);
Outer deserialized = std::move(get_result.value());
ASSERT_EQUAL(deserialized.inner_obj.value, 42);
ASSERT_EQUAL(deserialized.inner_obj.name, "inner");
ASSERT_EQUAL(deserialized.inner_vector.size(), 2);
ASSERT_EQUAL(deserialized.inner_vector[0].value, 1);
ASSERT_EQUAL(deserialized.inner_vector[1].name, "second");
ASSERT_TRUE(deserialized.optional_inner.has_value());
ASSERT_EQUAL(deserialized.optional_inner->value, 99);
ASSERT_TRUE(deserialized.unique_inner != nullptr);
ASSERT_EQUAL(deserialized.unique_inner->value, 123);
#endif
TEST_SUCCEED();
}
bool run() {
return test_empty_values() &&
test_special_characters() &&
test_numeric_limits() &&
test_nested_structures();
}
} // namespace builder_tests
int main(int argc, char *argv[]) {
return test_main(argc, argv, builder_tests::run);
}
@@ -0,0 +1,258 @@
#include "simdjson.h"
#include "test_builder.h"
#include <string>
using namespace simdjson;
namespace builder_tests {
bool test_enum_serialization() {
TEST_START();
#if SIMDJSON_STATIC_REFLECTION
enum class Color {
Red,
Green,
Blue
};
struct EnumStruct {
Color color;
int value;
};
EnumStruct test{Color::Red, 42};
auto result = builder::to_json_string(test);
ASSERT_SUCCESS(result);
std::string json = result.value();
// Enum should be serialized as string (Red)
ASSERT_TRUE(json.find("\"color\":\"Red\"") != std::string::npos);
ASSERT_TRUE(json.find("\"value\":42") != std::string::npos);
// Test different enum values
test.color = Color::Green;
auto result2 = builder::to_json_string(test);
ASSERT_SUCCESS(result2);
std::string json2 = result2.value();
ASSERT_TRUE(json2.find("\"color\":\"Green\"") != std::string::npos);
test.color = Color::Blue;
auto result3 = builder::to_json_string(test);
ASSERT_SUCCESS(result3);
std::string json3 = result3.value();
ASSERT_TRUE(json3.find("\"color\":\"Blue\"") != std::string::npos);
#endif
TEST_SUCCEED();
}
bool test_enum_deserialization() {
TEST_START();
#if SIMDJSON_STATIC_REFLECTION
enum class Status {
Active,
Inactive,
Pending
};
struct StatusStruct {
Status status;
std::string name;
};
// Test deserialization of different enum values with string representation
std::string json1 = "{\"status\":\"Active\",\"name\":\"test1\"}";
ondemand::parser parser1;
auto doc_result1 = parser1.iterate(pad(json1));
ASSERT_SUCCESS(doc_result1);
auto get_result1 = doc_result1.value().get<StatusStruct>();
ASSERT_SUCCESS(get_result1);
StatusStruct deserialized1 = std::move(get_result1.value());
ASSERT_TRUE(deserialized1.status == Status::Active);
ASSERT_EQUAL(deserialized1.name, "test1");
// Test Status::Inactive
std::string json2 = "{\"status\":\"Inactive\",\"name\":\"test2\"}";
ondemand::parser parser2;
auto doc_result2 = parser2.iterate(pad(json2));
ASSERT_SUCCESS(doc_result2);
auto get_result2 = doc_result2.value().get<StatusStruct>();
ASSERT_SUCCESS(get_result2);
StatusStruct deserialized2 = std::move(get_result2.value());
ASSERT_TRUE(deserialized2.status == Status::Inactive);
ASSERT_EQUAL(deserialized2.name, "test2");
// Test Status::Pending
std::string json3 = "{\"status\":\"Pending\",\"name\":\"test3\"}";
ondemand::parser parser3;
auto doc_result3 = parser3.iterate(pad(json3));
ASSERT_SUCCESS(doc_result3);
auto get_result3 = doc_result3.value().get<StatusStruct>();
ASSERT_SUCCESS(get_result3);
StatusStruct deserialized3 = std::move(get_result3.value());
ASSERT_TRUE(deserialized3.status == Status::Pending);
ASSERT_EQUAL(deserialized3.name, "test3");
#endif
TEST_SUCCEED();
}
bool test_enum_round_trip() {
TEST_START();
#if SIMDJSON_STATIC_REFLECTION
enum class Priority {
Low,
Medium,
High,
Critical
};
struct Task {
Priority priority;
std::string description;
int id;
};
Task original{Priority::High, "Important task", 123};
// Serialize
auto serialize_result = builder::to_json_string(original);
ASSERT_SUCCESS(serialize_result);
std::string json = serialize_result.value();
ASSERT_TRUE(json.find("\"priority\":\"High\"") != std::string::npos); // High as string
ASSERT_TRUE(json.find("\"description\":\"Important task\"") != std::string::npos);
ASSERT_TRUE(json.find("\"id\":123") != std::string::npos);
// Deserialize
ondemand::parser parser;
auto doc_result = parser.iterate(pad(json));
ASSERT_SUCCESS(doc_result);
auto get_result = doc_result.value().get<Task>();
ASSERT_SUCCESS(get_result);
Task deserialized = std::move(get_result.value());
ASSERT_TRUE(deserialized.priority == Priority::High);
ASSERT_EQUAL(deserialized.description, "Important task");
ASSERT_EQUAL(deserialized.id, 123);
#endif
TEST_SUCCEED();
}
bool test_enum_with_underlying_type() {
TEST_START();
#if SIMDJSON_STATIC_REFLECTION
enum class ErrorCode : int {
Success = 0,
NotFound = 404,
ServerError = 500
};
struct Response {
ErrorCode error;
std::string message;
};
Response test{ErrorCode::NotFound, "Resource not found"};
auto result = builder::to_json_string(test);
ASSERT_SUCCESS(result);
std::string json = result.value();
ASSERT_TRUE(json.find("\"error\":\"NotFound\"") != std::string::npos);
ASSERT_TRUE(json.find("\"message\":\"Resource not found\"") != std::string::npos);
// Test round-trip
ondemand::parser parser;
auto doc_result = parser.iterate(pad(json));
ASSERT_SUCCESS(doc_result);
auto get_result = doc_result.value().get<Response>();
ASSERT_SUCCESS(get_result);
Response deserialized = std::move(get_result.value());
ASSERT_TRUE(deserialized.error == ErrorCode::NotFound);
ASSERT_EQUAL(deserialized.message, "Resource not found");
#endif
TEST_SUCCEED();
}
bool test_multiple_enums() {
TEST_START();
#if SIMDJSON_STATIC_REFLECTION
enum class Day {
Monday,
Tuesday,
Wednesday,
Thursday,
Friday,
Saturday,
Sunday
};
enum class Month {
January,
February,
March,
April,
May,
June,
July,
August,
September,
October,
November,
December
};
struct Date {
Day day;
Month month;
int year;
};
Date test{Day::Friday, Month::July, 2024};
auto result = builder::to_json_string(test);
ASSERT_SUCCESS(result);
std::string json = result.value();
ASSERT_TRUE(json.find("\"day\":\"Friday\"") != std::string::npos); // Friday as string
ASSERT_TRUE(json.find("\"month\":\"July\"") != std::string::npos); // July as string
ASSERT_TRUE(json.find("\"year\":2024") != std::string::npos);
// Test round-trip
ondemand::parser parser;
auto doc_result = parser.iterate(pad(json));
ASSERT_SUCCESS(doc_result);
auto get_result = doc_result.value().get<Date>();
ASSERT_SUCCESS(get_result);
Date deserialized = std::move(get_result.value());
ASSERT_TRUE(deserialized.day == Day::Friday);
ASSERT_TRUE(deserialized.month == Month::July);
ASSERT_EQUAL(deserialized.year, 2024);
#endif
TEST_SUCCEED();
}
bool run() {
return test_enum_serialization() &&
test_enum_deserialization() &&
test_enum_round_trip() &&
test_enum_with_underlying_type() &&
test_multiple_enums();
}
} // namespace builder_tests
int main(int argc, char *argv[]) {
return test_main(argc, argv, builder_tests::run);
}