Adding the ability to write directly to an std::string. It is mostly syntaxic sugar (#2075)

This commit is contained in:
Daniel Lemire
2023-10-25 19:32:12 -04:00
committed by GitHub
parent 35e87896f0
commit 993ac4b87c
13 changed files with 1098 additions and 36 deletions
+35
View File
@@ -1722,6 +1722,41 @@ obj.reset(); // revise the object
uint64_t x = obj["value"]; // gives me 123
```
Storing Directly into an Existing std::string Instance
-----------------------------------------------------
The simdjson library favours the use of `std::string_view` instances because
it tends to lead to better performance due to causing fewer memory allocations.
However, they are cases where you need to store a string result in an `std::string``
instance. You can do so with a version of the `to_string()` method which takes as
a parameter a reference to an `std::string`.
```C++
auto json = R"({
"name": "Daniel",
"age": 42
})"_padded;
ondemand::parser parser;
ondemand::document doc = parser.iterate(json);
std::string name;
doc["name"].get_string(name);
```
The same routine can be written without exceptions handling:
```C++
std::string name;
auto err = doc["name"].get_string(name);
if(err) { /* handle error */ }
```
The `std::string` instance, once created, is independent. Unlike our `std::string_view` instances, it does not point at data that is
within our `parser` instance. The same caveat applies: you should
only consume a JSON string once.
You should be mindful of the trade-off: allocating multiple
`std::string` instances can become expensive.
Thread Safety
-------------
@@ -128,6 +128,9 @@ simdjson_inline simdjson_result<double> document::get_double_in_string() noexcep
simdjson_inline simdjson_result<std::string_view> document::get_string(bool allow_replacement) noexcept {
return get_root_value_iterator().get_root_string(true, allow_replacement);
}
simdjson_inline error_code document::get_string(std::string& receiver, bool allow_replacement) noexcept {
return get_root_value_iterator().get_root_string(receiver, true, allow_replacement);
}
simdjson_inline simdjson_result<std::string_view> document::get_wobbly_string() noexcept {
return get_root_value_iterator().get_root_wobbly_string(true);
}
@@ -397,6 +400,10 @@ simdjson_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLE
if (error()) { return error(); }
return first.get_string(allow_replacement);
}
simdjson_inline error_code simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::get_string(std::string& receiver, bool allow_replacement) noexcept {
if (error()) { return error(); }
return first.get_string(receiver, allow_replacement);
}
simdjson_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::get_wobbly_string() noexcept {
if (error()) { return error(); }
return first.get_wobbly_string();
@@ -583,6 +590,7 @@ simdjson_inline simdjson_result<int64_t> document_reference::get_int64_in_string
simdjson_inline simdjson_result<double> document_reference::get_double() noexcept { return doc->get_root_value_iterator().get_root_double(false); }
simdjson_inline simdjson_result<double> document_reference::get_double_in_string() noexcept { return doc->get_root_value_iterator().get_root_double(false); }
simdjson_inline simdjson_result<std::string_view> document_reference::get_string(bool allow_replacement) noexcept { return doc->get_root_value_iterator().get_root_string(false, allow_replacement); }
simdjson_inline error_code document_reference::get_string(std::string& receiver, bool allow_replacement) noexcept { return doc->get_root_value_iterator().get_root_string(receiver, false, allow_replacement); }
simdjson_inline simdjson_result<std::string_view> document_reference::get_wobbly_string() noexcept { return doc->get_root_value_iterator().get_root_wobbly_string(false); }
simdjson_inline simdjson_result<raw_json_string> document_reference::get_raw_json_string() noexcept { return doc->get_root_value_iterator().get_root_raw_json_string(false); }
simdjson_inline simdjson_result<bool> document_reference::get_bool() noexcept { return doc->get_root_value_iterator().get_root_bool(false); }
@@ -719,6 +727,10 @@ simdjson_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLE
if (error()) { return error(); }
return first.get_string(allow_replacement);
}
simdjson_inline error_code simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::get_string(std::string& receiver, bool allow_replacement) noexcept {
if (error()) { return error(); }
return first.get_string(receiver, allow_replacement);
}
simdjson_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::get_wobbly_string() noexcept {
if (error()) { return error(); }
return first.get_wobbly_string();
@@ -100,6 +100,20 @@ public:
* @returns INCORRECT_TYPE if the JSON value is not a string.
*/
simdjson_inline simdjson_result<std::string_view> get_string(bool allow_replacement = false) noexcept;
/**
* Attempts to fill the provided std::string reference with the parsed value of the current string.
*
* The string is guaranteed to be valid UTF-8.
*
* Important: a value should be consumed once. Calling get_string() twice on the same value
* is an error.
*
* Performance: This method may be slower than get_string() or get_string(bool) because it may need to allocate memory.
* We recommend you avoid allocating an std::string unless you need to.
*
* @returns INCORRECT_TYPE if the JSON value is not a string. Otherwise, we return SUCCESS.
*/
simdjson_inline error_code get_string(std::string& receiver, bool allow_replacement = false) noexcept;
/**
* Cast this JSON value to a string.
*
@@ -624,6 +638,7 @@ public:
simdjson_inline simdjson_result<double> get_double() noexcept;
simdjson_inline simdjson_result<double> get_double_in_string() noexcept;
simdjson_inline simdjson_result<std::string_view> get_string(bool allow_replacement = false) noexcept;
simdjson_inline error_code get_string(std::string& receiver, bool allow_replacement = false) noexcept;
simdjson_inline simdjson_result<std::string_view> get_wobbly_string() noexcept;
simdjson_inline simdjson_result<raw_json_string> get_raw_json_string() noexcept;
simdjson_inline simdjson_result<bool> get_bool() noexcept;
@@ -693,6 +708,7 @@ public:
simdjson_inline simdjson_result<double> get_double() noexcept;
simdjson_inline simdjson_result<double> get_double_in_string() noexcept;
simdjson_inline simdjson_result<std::string_view> get_string(bool allow_replacement = false) noexcept;
simdjson_inline error_code get_string(std::string& receiver, bool allow_replacement = false) noexcept;
simdjson_inline simdjson_result<std::string_view> get_wobbly_string() noexcept;
simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::raw_json_string> get_raw_json_string() noexcept;
simdjson_inline simdjson_result<bool> get_bool() noexcept;
@@ -765,6 +781,7 @@ public:
simdjson_inline simdjson_result<double> get_double() noexcept;
simdjson_inline simdjson_result<double> get_double_in_string() noexcept;
simdjson_inline simdjson_result<std::string_view> get_string(bool allow_replacement = false) noexcept;
simdjson_inline error_code get_string(std::string& receiver, bool allow_replacement = false) noexcept;
simdjson_inline simdjson_result<std::string_view> get_wobbly_string() noexcept;
simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::raw_json_string> get_raw_json_string() noexcept;
simdjson_inline simdjson_result<bool> get_bool() noexcept;
@@ -254,6 +254,7 @@ public:
*/
simdjson_inline simdjson_result<std::string_view> unescape(raw_json_string in, bool allow_replacement) noexcept;
simdjson_inline simdjson_result<std::string_view> unescape_wobbly(raw_json_string in) noexcept;
simdjson_inline void reenter_child(token_position position, depth_t child_depth) noexcept;
simdjson_inline error_code consume_character(char c) noexcept;
@@ -47,6 +47,9 @@ simdjson_inline simdjson_result<raw_json_string> value::get_raw_json_string() no
simdjson_inline simdjson_result<std::string_view> value::get_string(bool allow_replacement) noexcept {
return iter.get_string(allow_replacement);
}
simdjson_inline error_code value::get_string(std::string& receiver, bool allow_replacement) noexcept {
return iter.get_string(receiver, allow_replacement);
}
simdjson_inline simdjson_result<std::string_view> value::get_wobbly_string() noexcept {
return iter.get_wobbly_string();
}
@@ -319,6 +322,10 @@ simdjson_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLE
if (error()) { return error(); }
return first.get_string(allow_replacement);
}
simdjson_inline error_code simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::get_string(std::string& receiver, bool allow_replacement) noexcept {
if (error()) { return error(); }
return first.get_string(receiver, allow_replacement);
}
simdjson_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::get_wobbly_string() noexcept {
if (error()) { return error(); }
return first.get_wobbly_string();
+15
View File
@@ -135,6 +135,20 @@ public:
*/
simdjson_inline simdjson_result<std::string_view> get_string(bool allow_replacement = false) noexcept;
/**
* Attempts to fill the provided std::string reference with the parsed value of the current string.
*
* The string is guaranteed to be valid UTF-8.
*
* Important: a value should be consumed once. Calling get_string() twice on the same value
* is an error.
*
* Performance: This method may be slower than get_string() or get_string(bool) because it may need to allocate memory.
* We recommend you avoid allocating an std::string unless you need to.
*
* @returns INCORRECT_TYPE if the JSON value is not a string. Otherwise, we return SUCCESS.
*/
simdjson_inline error_code get_string(std::string& receiver, bool allow_replacement = false) noexcept;
/**
* Cast this JSON value to a "wobbly" string.
@@ -605,6 +619,7 @@ public:
simdjson_inline simdjson_result<double> get_double() noexcept;
simdjson_inline simdjson_result<double> get_double_in_string() noexcept;
simdjson_inline simdjson_result<std::string_view> get_string(bool allow_replacement = false) noexcept;
simdjson_inline error_code get_string(std::string& receiver, bool allow_replacement = false) noexcept;
simdjson_inline simdjson_result<std::string_view> get_wobbly_string() noexcept;
simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::raw_json_string> get_raw_json_string() noexcept;
simdjson_inline simdjson_result<bool> get_bool() noexcept;
@@ -513,6 +513,13 @@ simdjson_warn_unused simdjson_inline simdjson_result<bool> value_iterator::parse
simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> value_iterator::get_string(bool allow_replacement) noexcept {
return get_raw_json_string().unescape(json_iter(), allow_replacement);
}
simdjson_warn_unused simdjson_inline error_code value_iterator::get_string(std::string& receiver, bool allow_replacement) noexcept {
std::string_view content;
auto err = get_string(allow_replacement).get(content);
if (err) { return err; }
receiver = std::string(content);
return SUCCESS;
}
simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> value_iterator::get_wobbly_string() noexcept {
return get_raw_json_string().unescape_wobbly(json_iter());
}
@@ -636,6 +643,13 @@ simdjson_inline simdjson_result<number> value_iterator::get_root_number(bool che
simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> value_iterator::get_root_string(bool check_trailing, bool allow_replacement) noexcept {
return get_root_raw_json_string(check_trailing).unescape(json_iter(), allow_replacement);
}
simdjson_warn_unused simdjson_inline error_code value_iterator::get_root_string(std::string& receiver, bool check_trailing, bool allow_replacement) noexcept {
std::string_view content;
auto err = get_root_string(check_trailing, allow_replacement).get(content);
if (err) { return err; }
receiver = std::string(content);
return SUCCESS;
}
simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> value_iterator::get_root_wobbly_string(bool check_trailing) noexcept {
return get_root_raw_json_string(check_trailing).unescape_wobbly(json_iter());
}
@@ -296,6 +296,7 @@ public:
*/
simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> get_string(bool allow_replacement) noexcept;
simdjson_warn_unused simdjson_inline error_code get_string(std::string& receiver, bool allow_replacement) noexcept;
simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> get_wobbly_string() noexcept;
simdjson_warn_unused simdjson_inline simdjson_result<raw_json_string> get_raw_json_string() noexcept;
simdjson_warn_unused simdjson_inline simdjson_result<uint64_t> get_uint64() noexcept;
@@ -312,7 +313,8 @@ public:
simdjson_warn_unused simdjson_inline simdjson_result<number> get_number() noexcept;
simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> get_root_string(bool check_trailing, bool allow_replacement) noexcept;
simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> get_root_wobbly_string(bool check_trailing) noexcept;
simdjson_warn_unused simdjson_inline error_code get_root_string(std::string& receiver, bool check_trailing, bool allow_replacement) noexcept;
simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> get_root_wobbly_string(bool check_trailing) noexcept;
simdjson_warn_unused simdjson_inline simdjson_result<raw_json_string> get_root_raw_json_string(bool check_trailing) noexcept;
simdjson_warn_unused simdjson_inline simdjson_result<uint64_t> get_root_uint64(bool check_trailing) noexcept;
simdjson_warn_unused simdjson_inline simdjson_result<uint64_t> get_root_uint64_in_string(bool check_trailing) noexcept;
+2 -5
View File
@@ -131,11 +131,8 @@
#if SIMDJSON_IS_32BITS
#ifndef SIMDJSON_NO_PORTABILITY_WARNING
#pragma message("The simdjson library is designed \
for 64-bit processors and it seems that you are not \
compiling for a known 64-bit platform. All fast kernels \
will be disabled and performance may be poor. Please \
use a 64-bit target such as x64, 64-bit ARM or 64-bit PPC.")
// In the future, we should allow programmers
// to get warning.
#endif // SIMDJSON_NO_PORTABILITY_WARNING
#endif // SIMDJSON_IS_32BITS
+749 -30
View File
File diff suppressed because it is too large Load Diff
+1
View File
@@ -23,6 +23,7 @@ add_cpp_test(ondemand_ordering_tests LABELS ondemand acceptance per_impl
add_cpp_test(ondemand_parse_api_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_readme_examples LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_scalar_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_to_string LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_twitter_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_wrong_type_error_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_iterate_many_csv LABELS ondemand acceptance per_implementation)
@@ -20,7 +20,37 @@ bool string2() {
}
bool to_string_example_no_except() {
TEST_START();
auto json = R"({
"name": "Daniel",
"age": 42
})"_padded;
ondemand::parser parser;
ondemand::document doc;
auto err = parser.iterate(json).get(doc);
if(err) { return false; }
std::string name;
err = doc["name"].get_string(name);
if(err) { return false; }
TEST_SUCCEED();
}
#if SIMDJSON_EXCEPTIONS
bool to_string_example() {
TEST_START();
auto json = R"({
"name": "Daniel",
"age": 42
})"_padded;
ondemand::parser parser;
ondemand::document doc = parser.iterate(json);
std::string name;
doc["name"].get_string(name);
ASSERT_EQUAL(name, "Daniel");
TEST_SUCCEED();
}
bool gen_raw1() {
TEST_START();
simdjson::ondemand::parser parser;
@@ -1420,7 +1450,9 @@ bool run() {
&& current_location_user_error()
&& current_location_out_of_bounds()
&& current_location_no_error()
&& to_string_example_no_except()
#if SIMDJSON_EXCEPTIONS
&& to_string_example()
&& raw_string()
&& number_tests()
&& current_location_tape_error_with_except()
+210
View File
@@ -0,0 +1,210 @@
#include "simdjson.h"
#include "test_ondemand.h"
using namespace simdjson;
namespace json_package_tests {
using namespace std;
bool baby() {
TEST_START();
auto json = R"({
"name": "Daniel",
"age": 42
})"_padded;
ondemand::parser parser;
ondemand::document doc;
ASSERT_SUCCESS(parser.iterate(json).get(doc));
simdjson::ondemand::object main_object;
ASSERT_SUCCESS(doc.get_object().get(main_object));
std::string name;
ASSERT_SUCCESS(main_object["name"].get_string(name));
ASSERT_EQUAL(name, "Daniel");
uint64_t age;
ASSERT_SUCCESS(main_object["age"].get_uint64().get(age));
ASSERT_EQUAL(age, 42);
TEST_SUCCEED();
}
bool thirtysecondsofcode() {
TEST_START();
auto json = R"({
"name": "30-seconds-of-code",
"private": true,
"version": "10.0.0",
"description": "30 seconds of code website.",
"exports": "./index.js",
"author": "chalarangelo",
"type": "module",
"devDependencies": {
"@jsiqle/core": "^3.0.0",
"astro": "^3.2.0",
"chalk": "^5.3.0",
"eslint": "^8.50.0",
"eslint-config-prettier": "^9.0.0",
"front-matter": "^4.0.2",
"fs-extra": "^11.1.1",
"glob": "^10.3.10",
"hast-util-to-html": "^9.0.0",
"js-yaml": "^4.1.0",
"mdast-util-to-hast": "^13.0.2",
"prettier": "^3.0.3",
"prettier-plugin-astro": "^0.12.0",
"prismjs": "^1.29.0",
"remark": "^15.0.1",
"remark-gfm": "^4.0.0",
"sass": "^1.68.0",
"sharp": "^0.32.6",
"unist-util-select": "^5.0.0",
"unist-util-visit": "^5.0.0",
"unist-util-visit-parents": "^6.0.1",
"webfonts-generator": "^0.4.0"
},
"imports": {
"#blocks/*": "./src/blocks/*.js",
"#components/*": "./src/components/*.astro",
"#layouts/*": "./src/layouts/*.astro",
"#settings/*": "./src/settings/*.js",
"#prefabs": "./src/prefabs/index.js",
"#utils": "./src/utils/index.js",
"#utils/search": "./src/utils/search.js"
},
"scripts": {
"predev": "NODE_ENV=development node ./src/scripts/develop.js",
"dev": "astro dev --port 8000",
"start": "astro dev --port 8000",
"prebuild": "NODE_ENV=production node ./src/scripts/build.js",
"build": "astro build",
"preview": "astro preview --port 9000",
"watch": "NODE_ENV=development node ./src/scripts/watch.js",
"console": "NODE_ENV=production node ./src/scripts/console.js",
"create": "NODE_ENV=production node ./src/scripts/create.js",
"icons": "NODE_ENV=production node ./src/scripts/icons.js",
"manifest": "NODE_ENV=production node ./src/scripts/manifest.js"
},
"license": "MIT",
"repository": {
"type": "git",
"url": "https://github.com/30-seconds/30-seconds-of-code"
},
"bugs": {
"url": "https://github.com/30-seconds/30-seconds-of-code/issues"
},
"browserslist": [
"> 0.5% and last 4 versions and not dead and not ie>0 and not op_mini all and not and_uc>0 and not edge<79"
],
"engines": {
"node": ">=18.14.2"
}
})"_padded;
ondemand::parser parser;
ondemand::document doc;
ASSERT_SUCCESS(parser.iterate(json).get(doc));
simdjson::ondemand::object main_object;
ASSERT_SUCCESS(doc.get_object().get(main_object));
simdjson::ondemand::raw_json_string key;
simdjson::ondemand::value value;
for (auto field : main_object) {
// Throw error if getting key or value fails.
ASSERT_SUCCESS(field.key().get(key));
ASSERT_SUCCESS(field.value().get(value));
if (key == "name") {
std::string name;
ASSERT_SUCCESS(value.get_string(name));
ASSERT_EQUAL(name, "30-seconds-of-code");
} else if (key == "main") {
std::string main;
ASSERT_SUCCESS(value.get_string(main));
// unused
} else if (key == "exports") {
simdjson::ondemand::json_type exports_type;
if (!value.type().get(exports_type)) {
std::string_view exports;
switch (exports_type) {
case simdjson::ondemand::json_type::object: {
simdjson::ondemand::object exports_object;
if (!value.get_object().get(exports_object) &&
!exports_object.raw_json().get(exports)) {
// unused
}
break;
}
case simdjson::ondemand::json_type::array: {
simdjson::ondemand::array exports_array;
if (!value.get_array().get(exports_array) &&
!exports_array.raw_json().get(exports)) {
// unused
}
break;
}
case simdjson::ondemand::json_type::string: {
if (!value.get_string().get(exports)) {
ASSERT_EQUAL(exports, "./index.js");
}
break;
}
default:
break;
}
}
} else if (key == "imports") {
simdjson::ondemand::json_type imports_type;
if (!value.type().get(imports_type)) {
std::string_view imports;
switch (imports_type) {
case simdjson::ondemand::json_type::object: {
simdjson::ondemand::object imports_object;
if (!value.get_object().get(imports_object) &&
!imports_object.raw_json().get(imports)) {
ASSERT_EQUAL(imports, R"({
"#blocks/*": "./src/blocks/*.js",
"#components/*": "./src/components/*.astro",
"#layouts/*": "./src/layouts/*.astro",
"#settings/*": "./src/settings/*.js",
"#prefabs": "./src/prefabs/index.js",
"#utils": "./src/utils/index.js",
"#utils/search": "./src/utils/search.js"
})");
}
break;
}
case simdjson::ondemand::json_type::array: {
simdjson::ondemand::array imports_array;
if (!value.get_array().get(imports_array) &&
!imports_array.raw_json().get(imports)) {
// unused
}
break;
}
case simdjson::ondemand::json_type::string: {
if (!value.get_string().get(imports)) {
// unused
}
break;
}
default:
break;
}
}
} else if (key == "type") {
std::string_view type;
if (!value.get_string().get(type) &&
(type == "commonjs" || type == "module")) {
ASSERT_EQUAL(type, "module");
}
}
}
TEST_SUCCEED();
}
bool run() { return thirtysecondsofcode() && baby(); }
} // namespace json_package_tests
int main(int argc, char *argv[]) {
return test_main(argc, argv, json_package_tests::run);
}