admin/simdjson-simdjson
1
0
Fork 0
mirror of https://github.com/simdjson/simdjson synced 2026-06-08 17:27:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into dlemire/improving_documentation

Browse Source
This commit is contained in:
Daniel Lemire
2020-06-23 13:07:29 -04:00
committed by GitHub
parent 1547f2ec80 257089884f
commit 8cc9f496ee
40 changed files with 1866 additions and 1146 deletions
Download Patch File Download Diff File Expand all files Collapse all files
benchmark/bench_dom_api.cpp
+6 -5
View File
@@ -407,7 +407,7 @@ static void iterator_twitter_default_profile(State& state) {
set<string_view> default_users;
ParsedJson::Iterator iter(pj);
// for (dom::object tweet : doc["statuses"].get<dom::array>()) {
// for (dom::object tweet : doc["statuses"]) {
if (!(iter.move_to_key("statuses") && iter.is_array())) { return; }
if (iter.down()) { // first status
do {
@@ -480,23 +480,24 @@ static void iterator_twitter_image_sizes(State& state) {
set<tuple<uint64_t, uint64_t>> image_sizes;
ParsedJson::Iterator iter(pj);
// for (dom::object tweet : doc["statuses"].get<dom::array>()) {
// for (dom::object tweet : doc["statuses"]) {
if (!(iter.move_to_key("statuses") && iter.is_array())) { return; }
if (iter.down()) { // first status
do {
// auto [media, not_found] = tweet["entities"]["media"];
// dom::object media;
// not_found = tweet["entities"]["media"].get(media);
// if (!not_found) {
if (iter.move_to_key("entities")) {
if (!iter.is_object()) { return; }
if (iter.move_to_key("media")) {
if (!iter.is_array()) { return; }
// for (dom::object image : media.get<dom::array>()) {
// for (dom::object image : media) {
if (iter.down()) { // first media
do {
// for (auto [key, size] : image["sizes"].get<dom::object>()) {
// for (auto [key, size] : dom::object(image["sizes"])) {
if (!(iter.move_to_key("sizes") && iter.is_object())) { return; }
if (iter.down()) { // first size
do {
benchmark/distinctuseridcompetition.cpp
+8 -6
View File
@@ -40,17 +40,18 @@ void print_vec(const std::vector<int64_t> &v) {
// simdjson_recurse below come be implemented like so but it is slow:
/*void simdjson_recurse(std::vector<int64_t> & v, simdjson::dom::element element) {
if (element.is<simdjson::dom::array>()) {
auto [array, array_error] = element.get<simdjson::dom::array>();
error_code error;
if (element.is_array()) {
dom::array array;
error = element.get(array);
for (auto child : array) {
if (child.is<simdjson::dom::array>() || child.is<simdjson::dom::object>()) {
simdjson_recurse(v, child);
}
}
} else if (element.is<simdjson::dom::object>()) {
auto [object, error] = element.get<simdjson::dom::object>();
} else if (element.is_object()) {
int64_t id;
error = object["user"]["id"].get(id);
error = element["user"]["id"].get(id);
if(!error) {
v.push_back(id);
}
@@ -330,7 +331,8 @@ int main(int argc, char *argv[]) {
std::cerr << "warning: ignoring everything after " << argv[optind + 1]
<< std::endl;
}
auto [p, error] = simdjson::padded_string::load(filename);
simdjson::padded_string p;
auto error = simdjson::padded_string::load(filename).get(p);
if (error) {
std::cerr << "Could not load the file " << filename << std::endl;
return EXIT_FAILURE;
benchmark/minifiercompetition.cpp
+2 -1
View File
@@ -75,7 +75,8 @@ int main(int argc, char *argv[]) {
exit(1);
}
const char *filename = argv[optind];
auto [p, error] = simdjson::padded_string::load(filename);
simdjson::padded_string p;
auto error = simdjson::padded_string::load(filename).get(p);
if (error) {
std::cerr << "Could not load the file " << filename << std::endl;
return EXIT_FAILURE;
benchmark/parse_stream.cpp
+16 -6
View File
@@ -87,10 +87,15 @@ int main(int argc, char *argv[]) {
auto start = std::chrono::steady_clock::now();
count = 0;
for (auto result : parser.parse_many(p, i)) {
simdjson::dom::document_stream docs;
if ((error = parser.parse_many(p, i).get(docs))) {
std::wcerr << "Parsing failed with: " << error << std::endl;
exit(1);
}
for (auto result : docs) {
error = result.error();
if (error != simdjson::SUCCESS) {
std::wcerr << "Parsing failed with: " << error_message(error) << std::endl;
if (error) {
std::wcerr << "Parsing failed with: " << error << std::endl;
exit(1);
}
count++;
@@ -134,10 +139,15 @@ int main(int argc, char *argv[]) {
auto start = std::chrono::steady_clock::now();
// This includes allocation of the parser
for (auto result : parser.parse_many(p, optimal_batch_size)) {
simdjson::dom::document_stream docs;
if ((error = parser.parse_many(p, optimal_batch_size).get(docs))) {
std::wcerr << "Parsing failed with: " << error << std::endl;
exit(1);
}
for (auto result : docs) {
error = result.error();
if (error != simdjson::SUCCESS) {
std::wcerr << "Parsing failed with: " << error_message(error) << std::endl;
if (error) {
std::wcerr << "Parsing failed with: " << error << std::endl;
exit(1);
}
}
benchmark/parseandstatcompetition.cpp
+10 -7
View File
@@ -105,7 +105,8 @@ void simdjson_recurse(stat_t &s, simdjson::dom::element element) {
never_inline stat_t simdjson_compute_stats(const simdjson::padded_string &p) {
stat_t s{};
simdjson::dom::parser parser;
auto [doc, error] = parser.parse(p);
simdjson::dom::element doc;
auto error = parser.parse(p).get(doc);
if (error) {
s.valid = false;
return s;
@@ -154,11 +155,11 @@ static void GenStatPlus(Stat &stat, const dom::element &v) {
break;
case dom::element_type::STRING: {
stat.stringCount++;
std::string_view sv = v.get<std::string_view>();
auto sv = std::string_view(v);
stat.stringLength += sv.size();
} break;
case dom::element_type::BOOL:
if (v.get<bool>()) {
if (bool(v)) {
stat.trueCount++;
} else {
stat.falseCount++;
@@ -409,7 +410,8 @@ int main(int argc, char *argv[]) {
std::cerr << "warning: ignoring everything after " << argv[optind + 1]
<< std::endl;
}
auto [p, error] = simdjson::padded_string::load(filename);
simdjson::padded_string p;
auto error = simdjson::padded_string::load(filename).get(p);
if (error) {
std::cerr << "Could not load the file " << filename << std::endl;
return EXIT_FAILURE;
@@ -464,9 +466,10 @@ int main(int argc, char *argv[]) {
printf("API traversal tests\n");
printf("Based on https://github.com/miloyip/nativejson-benchmark\n");
simdjson::dom::parser parser;
auto [doc, err] = parser.parse(p);
if (err) {
std::cerr << err << std::endl;
simdjson::dom::element doc;
auto error = parser.parse(p).get(doc);
if (error) {
std::cerr << error << std::endl;
}
size_t refval = simdjson_compute_stats_refplus(doc).objectCount;
benchmark/parsingcompetition.cpp
+4 -3
View File
@@ -82,9 +82,10 @@ inline void reset_stream(std::stringstream & is) {
bool bench(const char *filename, bool verbose, bool just_data, double repeat_multiplier) {
auto [p, err] = simdjson::padded_string::load(filename);
if (err) {
std::cerr << "Could not load the file " << filename << std::endl;
simdjson::padded_string p;
auto error = simdjson::padded_string::load(filename).get(p);
if (error) {
std::cerr << "Could not load the file " << filename << ": " << error << std::endl;
return false;
}
benchmark/statisticalmodel.cpp
+4 -2
View File
@@ -96,7 +96,8 @@ void simdjson_recurse(stat_t &s, simdjson::dom::element element) {
stat_t simdjson_compute_stats(const simdjson::padded_string &p) {
stat_t answer{};
simdjson::dom::parser parser;
auto [doc, error] = parser.parse(p);
simdjson::dom::element doc;
auto error = parser.parse(p).get(doc);
if (error) {
answer.valid = false;
return answer;
@@ -136,7 +137,8 @@ int main(int argc, char *argv[]) {
std::cerr << "warning: ignoring everything after " << argv[optind + 1]
<< std::endl;
}
auto [p, error] = simdjson::padded_string::load(filename);
simdjson::padded_string p;
auto error = simdjson::padded_string::load(filename).get(p);
if (error) {
std::cerr << "Could not load the file " << filename << std::endl;
return EXIT_FAILURE;
doc/basics.md
+15 -28
View File
@@ -187,21 +187,20 @@ And another one:
auto abstract_json = R"(
{ "str" : { "123" : {"abc" : 3.14 } } } )"_padded;
dom::parser parser;
double v = parser.parse(abstract_json)["str"]["123"]["abc"].get<double>();
double v = parser.parse(abstract_json)["str"]["123"]["abc"];
cout << "number: " << v << endl;
```
C++17 Support
-------------
While the simdjson library can be used in any project using C++ 11 and above, it has special support
for C++ 17. The APIs for field iteration and error handling in particular are designed to work
nicely with C++17's destructuring syntax. For example:
While the simdjson library can be used in any project using C++ 11 and above, field iteration has special support C++ 17's destructuring syntax. For example:
```c++
dom::parser parser;
padded_string json = R"( { "foo": 1, "bar": 2 } )"_padded;
auto [object, error] = parser.parse(json).get<dom::object>();
dom::parser parser;
dom::object object;
auto error = parser.parse(json).get(object);
if (error) { cerr << error << endl; return; }
for (auto [key, value] : object) {
cout << key << " = " << value << endl;
@@ -212,11 +211,10 @@ For comparison, here is the C++ 11 version of the same code:
```c++
// C++ 11 version for comparison
dom::parser parser;
padded_string json = R"( { "foo": 1, "bar": 2 } )"_padded;
simdjson::error_code error;
dom::parser parser;
dom::object object;
error = parser.parse(json).get(object);
auto error = parser.parse(json).get(object);
if (!error) { cerr << error << endl; return; }
for (dom::key_value_pair field : object) {
cout << field.key << " = " << field.value << endl;
@@ -259,7 +257,6 @@ The UTF-8 validation function merely checks that the input is valid UTF-8: it wo
Your input string does not need any padding. Any string will do. The `validate_utf8` function does not do any memory allocation on the heap, and it does not throw exceptions.
JSON Pointer
------------
@@ -281,29 +278,18 @@ Error Handling
--------------
All simdjson APIs that can fail return `simdjson_result<T>`, which is a &lt;value, error_code&gt;
pair. The error codes and values can be accessed directly, reading the error like so:
pair. You can retrieve the value with .get(), like so:
```c++
auto [doc, error] = parser.parse(json); // doc is a dom::element
dom::element doc;
auto error = parser.parse(json).get(doc);
if (error) { cerr << error << endl; exit(1); }
// Use document here now that we've checked for the error
```
When you use the code this way, it is your responsibility to check for error before using the
result: if there is an error, the result value will not be valid and using it will caused undefined
behavior.
> Note: because of the way `auto [x, y]` works in C++, you have to define new variables each time you
> use it. If your project treats aliased, this means you can't use the same names in `auto [x, error]`
> without triggering warnings or error (and particularly can't use the word "error" every time). To
> circumvent this, you can use this instead:
>
> ```c++
> dom::element doc;
> auto error = parser.parse(json).get(doc); // <-- Assigns to doc and error just like "auto [doc, error]"
> ```
We can write a "quick start" example where we attempt to parse a file and access some data, without triggering exceptions:
```C++
@@ -311,11 +297,12 @@ We can write a "quick start" example where we attempt to parse a file and access
int main(void) {
simdjson::dom::parser parser;
simdjson::dom::element tweets;
auto error = parser.load("twitter.json").get(tweets);
if (error) { std::cerr << error << std::endl; return EXIT_FAILURE; }
simdjson::dom::element res;
simdjson::dom::element res;
if ((error = tweets["search_metadata"]["count"].get(res))) {
std::cerr << "could not access keys" << std::endl;
return EXIT_FAILURE;
@@ -418,8 +405,7 @@ And another one:
cout << "number: " << v << endl;
```
Notice how we can string several operation (`parser.parse(abstract_json)["str"]["123"]["abc"].get<double>()`) and only check for the error once, a strategy we call *error chaining*.
Notice how we can string several operations (`parser.parse(abstract_json)["str"]["123"]["abc"].get(v)`) and only check for the error once, a strategy we call *error chaining*.
The next two functions will take as input a JSON document containing an array with a single element, either a string or a number. They return true upon success.
@@ -529,7 +515,8 @@ Here is a simple example, given "x.json" with this content:
```c++
dom::parser parser;
for (dom::element doc : parser.load_many(filename)) {
dom::document_stream docs = parser.load_many(filename);
for (dom::element doc : docs) {
cout << doc["foo"] << endl;
}
// Prints 1 2 3
doc/performance.md
+6 -4
View File
@@ -68,7 +68,8 @@ without bound:
```c++
dom::parser parser(1000*1000); // Never grow past documents > 1MB
for (web_request request : listen()) {
auto [doc, error] = parser.parse(request.body);
dom::element doc;
auto error = parser.parse(request.body).get(doc);
// If the document was above our limit, emit 413 = payload too large
if (error == CAPACITY) { request.respond(413); continue; }
// ...
@@ -82,11 +83,12 @@ without bound:
```c++
dom::parser parser(0); // This parser will refuse to automatically grow capacity
simdjson::error_code allocate_error = parser.allocate(1000*1000); // This allocates enough capacity to handle documents <= 1MB
if (allocate_error) { cerr << allocate_error << endl; exit(1); }
auto error = parser.allocate(1000*1000); // This allocates enough capacity to handle documents <= 1MB
if (error) { cerr << error << endl; exit(1); }
for (web_request request : listen()) {
auto [doc, error] = parser.parse(request.body);
dom::element doc;
error = parser.parse(request.body).get(doc);
// If the document was above our limit, emit 413 = payload too large
if (error == CAPACITY) { request.respond(413); continue; }
// ...
include/simdjson/dom/array.h
+1 -1
View File
@@ -68,7 +68,7 @@ public:
* Get the value associated with the given JSON pointer.
*
* dom::parser parser;
* array a = parser.parse(R"([ { "foo": { "a": [ 10, 20, 30 ] }} ])");
* array a = parser.parse(R"([ { "foo": { "a": [ 10, 20, 30 ] }} ])"_padded);
* a.at("0/foo/a/1") == 20
* a.at("0")["foo"]["a"].at(1) == 20
*
include/simdjson/dom/document_stream.h
+44 -23
View File
@@ -72,8 +72,20 @@ private:
*/
class document_stream {
public:
/**
* Construct an uninitialized document_stream.
*
* ```c++
* document_stream docs;
* error = parser.parse_many(json).get(docs);
* ```
*/
really_inline document_stream() noexcept;
/** Move one document_stream to another. */
really_inline document_stream(document_stream && other) noexcept = default;
really_inline document_stream(document_stream &&other) noexcept = default;
/** Move one document_stream to another. */
really_inline document_stream &operator=(document_stream &&other) noexcept = default;
really_inline ~document_stream() noexcept;
/**
@@ -99,9 +111,8 @@ public:
*
* Gives the current index in the input document in bytes.
*
* auto stream = parser.parse_many(json,window);
* auto i = stream.begin();
* for(; i != stream.end(); ++i) {
* document_stream stream = parser.parse_many(json,window);
* for(auto i = stream.begin(); i != stream.end(); ++i) {
* auto doc = *i;
* size_t index = i.current_index();
* }
@@ -132,8 +143,7 @@ public:
private:
document_stream &operator=(const document_stream &) = delete; // Disallow copying
document_stream(document_stream &other) = delete; // Disallow copying
document_stream(const document_stream &other) = delete; // Disallow copying
/**
* Construct a document_stream. Does not allocate or parse anything until the iterator is
@@ -141,18 +151,9 @@ private:
*/
really_inline document_stream(
dom::parser &parser,
size_t batch_size,
const uint8_t *buf,
size_t len
) noexcept;
/**
* Construct a document_stream with an initial error.
*/
really_inline document_stream(
dom::parser &parser,
size_t batch_size,
error_code error
size_t len,
size_t batch_size
) noexcept;
/**
@@ -199,13 +200,14 @@ private:
/** Pass the next batch through stage 1 with the given parser. */
inline error_code run_stage1(dom::parser &p, size_t batch_start) noexcept;
dom::parser &parser;
dom::parser *parser;
const uint8_t *buf;
const size_t len;
const size_t batch_size;
size_t batch_start{0};
size_t len;
size_t batch_size;
/** The error (or lack thereof) from the current document. */
error_code error;
size_t batch_start{0};
size_t doc_index{};
#ifdef SIMDJSON_THREADS_ENABLED
inline void load_from_stage1_thread() noexcept;
@@ -229,12 +231,31 @@ private:
#endif // SIMDJSON_THREADS_ENABLED
friend class dom::parser;
size_t doc_index{};
friend struct simdjson_result<dom::document_stream>;
friend struct internal::simdjson_result_base<dom::document_stream>;
}; // class document_stream
} // namespace dom
template<>
struct simdjson_result<dom::document_stream> : public internal::simdjson_result_base<dom::document_stream> {
public:
really_inline simdjson_result() noexcept; ///< @private
really_inline simdjson_result(error_code error) noexcept; ///< @private
really_inline simdjson_result(dom::document_stream &&value) noexcept; ///< @private
#if SIMDJSON_EXCEPTIONS
really_inline dom::document_stream::iterator begin() noexcept(false);
really_inline dom::document_stream::iterator end() noexcept(false);
#else // SIMDJSON_EXCEPTIONS
[[deprecated("parse_many() and load_many() may return errors. Use document_stream stream; error = parser.parse_many().get(doc); instead.")]]
really_inline dom::document_stream::iterator begin() noexcept;
[[deprecated("parse_many() and load_many() may return errors. Use document_stream stream; error = parser.parse_many().get(doc); instead.")]]
really_inline dom::document_stream::iterator end() noexcept;
#endif // SIMDJSON_EXCEPTIONS
}; // struct simdjson_result<dom::document_stream>
} // namespace simdjson
#endif // SIMDJSON_DOCUMENT_STREAM_H
include/simdjson/dom/element.h
+16 -35
View File
@@ -243,25 +243,6 @@ public:
template<typename T>
inline void tie(T &value, error_code &error) && noexcept;
/**
* Get the value as the provided type (T).
*
* Supported types:
* - Boolean: bool
* - Number: double, uint64_t, int64_t
* - String: std::string_view, const char *
* - Array: dom::array
* - Object: dom::object
*
* @tparam T bool, double, uint64_t, int64_t, std::string_view, const char *, dom::array, dom::object
*
* @param value The variable to set to the given type. value is undefined if there is an error.
*
* @returns true if the value was able to be set, false if there was an error.
*/
template<typename T>
WARN_UNUSED inline bool tie(T &value) && noexcept;
#if SIMDJSON_EXCEPTIONS
/**
* Read this element as a boolean.
@@ -355,8 +336,8 @@ public:
* The key will be matched against **unescaped** JSON:
*
* dom::parser parser;
* parser.parse(R"({ "a\n": 1 })")["a\n"].get<uint64_t>().value == 1
* parser.parse(R"({ "a\n": 1 })")["a\\n"].get<uint64_t>().error == NO_SUCH_FIELD
* parser.parse(R"({ "a\n": 1 })"_padded)["a\n"].get<uint64_t>().first == 1
* parser.parse(R"({ "a\n": 1 })"_padded)["a\\n"].get<uint64_t>().error() == NO_SUCH_FIELD
*
* @return The value associated with this field, or:
* - NO_SUCH_FIELD if the field does not exist in the object
@@ -370,8 +351,8 @@ public:
* The key will be matched against **unescaped** JSON:
*
* dom::parser parser;
* parser.parse(R"({ "a\n": 1 })")["a\n"].get<uint64_t>().value == 1
* parser.parse(R"({ "a\n": 1 })")["a\\n"].get<uint64_t>().error == NO_SUCH_FIELD
* parser.parse(R"({ "a\n": 1 })"_padded)["a\n"].get<uint64_t>().first == 1
* parser.parse(R"({ "a\n": 1 })"_padded)["a\\n"].get<uint64_t>().error() == NO_SUCH_FIELD
*
* @return The value associated with this field, or:
* - NO_SUCH_FIELD if the field does not exist in the object
@@ -383,7 +364,7 @@ public:
* Get the value associated with the given JSON pointer.
*
* dom::parser parser;
* element doc = parser.parse(R"({ "foo": { "a": [ 10, 20, 30 ] }})");
* element doc = parser.parse(R"({ "foo": { "a": [ 10, 20, 30 ] }})"_padded);
* doc.at("/foo/a/1") == 20
* doc.at("/")["foo"]["a"].at(1) == 20
* doc.at("")["foo"]["a"].at(1) == 20
@@ -410,8 +391,8 @@ public:
* The key will be matched against **unescaped** JSON:
*
* dom::parser parser;
* parser.parse(R"({ "a\n": 1 })")["a\n"].get<uint64_t>().value == 1
* parser.parse(R"({ "a\n": 1 })")["a\\n"].get<uint64_t>().error == NO_SUCH_FIELD
* parser.parse(R"({ "a\n": 1 })"_padded)["a\n"].get<uint64_t>().first == 1
* parser.parse(R"({ "a\n": 1 })"_padded)["a\\n"].get<uint64_t>().error() == NO_SUCH_FIELD
*
* @return The value associated with this field, or:
* - NO_SUCH_FIELD if the field does not exist in the object
@@ -474,7 +455,7 @@ public:
really_inline simdjson_result<dom::element_type> type() const noexcept;
template<typename T>
really_inline simdjson_result<bool> is() const noexcept;
really_inline bool is() const noexcept;
template<typename T>
really_inline simdjson_result<T> get() const noexcept;
template<typename T>
@@ -489,14 +470,14 @@ public:
really_inline simdjson_result<double> get_double() const noexcept;
really_inline simdjson_result<bool> get_bool() const noexcept;
really_inline simdjson_result<bool> is_array() const noexcept;
really_inline simdjson_result<bool> is_object() const noexcept;
really_inline simdjson_result<bool> is_string() const noexcept;
really_inline simdjson_result<bool> is_int64_t() const noexcept;
really_inline simdjson_result<bool> is_uint64_t() const noexcept;
really_inline simdjson_result<bool> is_double() const noexcept;
really_inline simdjson_result<bool> is_bool() const noexcept;
really_inline simdjson_result<bool> is_null() const noexcept;
really_inline bool is_array() const noexcept;
really_inline bool is_object() const noexcept;
really_inline bool is_string() const noexcept;
really_inline bool is_int64_t() const noexcept;
really_inline bool is_uint64_t() const noexcept;
really_inline bool is_double() const noexcept;
really_inline bool is_bool() const noexcept;
really_inline bool is_null() const noexcept;
really_inline simdjson_result<dom::element> operator[](const std::string_view &key) const noexcept;
really_inline simdjson_result<dom::element> operator[](const char *key) const noexcept;
include/simdjson/dom/object.h
+7 -7
View File
@@ -101,8 +101,8 @@ public:
* The key will be matched against **unescaped** JSON:
*
* dom::parser parser;
* parser.parse(R"({ "a\n": 1 })")["a\n"].get<uint64_t>().value == 1
* parser.parse(R"({ "a\n": 1 })")["a\\n"].get<uint64_t>().error == NO_SUCH_FIELD
* parser.parse(R"({ "a\n": 1 })"_padded)["a\n"].get<uint64_t>().first == 1
* parser.parse(R"({ "a\n": 1 })"_padded)["a\\n"].get<uint64_t>().error() == NO_SUCH_FIELD
*
* This function has linear-time complexity: the keys are checked one by one.
*
@@ -118,8 +118,8 @@ public:
* The key will be matched against **unescaped** JSON:
*
* dom::parser parser;
* parser.parse(R"({ "a\n": 1 })")["a\n"].get<uint64_t>().value == 1
* parser.parse(R"({ "a\n": 1 })")["a\\n"].get<uint64_t>().error == NO_SUCH_FIELD
* parser.parse(R"({ "a\n": 1 })"_padded)["a\n"].get<uint64_t>().first == 1
* parser.parse(R"({ "a\n": 1 })"_padded)["a\\n"].get<uint64_t>().error() == NO_SUCH_FIELD
*
* This function has linear-time complexity: the keys are checked one by one.
*
@@ -133,7 +133,7 @@ public:
* Get the value associated with the given JSON pointer.
*
* dom::parser parser;
* object obj = parser.parse(R"({ "foo": { "a": [ 10, 20, 30 ] }})");
* object obj = parser.parse(R"({ "foo": { "a": [ 10, 20, 30 ] }})"_padded);
* obj.at("foo/a/1") == 20
* obj.at("foo")["a"].at(1) == 20
*
@@ -151,8 +151,8 @@ public:
* The key will be matched against **unescaped** JSON:
*
* dom::parser parser;
* parser.parse(R"({ "a\n": 1 })")["a\n"].get<uint64_t>().value == 1
* parser.parse(R"({ "a\n": 1 })")["a\\n"].get<uint64_t>().error == NO_SUCH_FIELD
* parser.parse(R"({ "a\n": 1 })"_padded)["a\n"].get<uint64_t>().first == 1
* parser.parse(R"({ "a\n": 1 })"_padded)["a\\n"].get<uint64_t>().error() == NO_SUCH_FIELD
*
* This function has linear-time complexity: the keys are checked one by one.
*
include/simdjson/dom/parser.h
+23 -17
View File
@@ -173,9 +173,13 @@ public:
* the same interface, requiring you to check the error before using the document:
*
* dom::parser parser;
* for (auto [doc, error] : parser.load_many(path)) {
* if (error) { cerr << error << endl; exit(1); }
* cout << std::string(doc["title"]) << endl;
* dom::document_stream docs;
* auto error = parser.load_many(path).get(docs);
* if (error) { cerr << error << endl; exit(1); }
* for (auto doc : docs) {
* std::string_view title;
* if ((error = doc["title"].get(title)) { cerr << error << endl; exit(1); }
* cout << title << endl;
* }
*
* ### Threads
@@ -193,20 +197,19 @@ public:
* spot is cache-related: small enough to fit in cache, yet big enough to
* parse as many documents as possible in one tight loop.
* Defaults to 10MB, which has been a reasonable sweet spot in our tests.
* @return The stream. If there is an error, it will be returned during iteration. An empty input
* will yield 0 documents rather than an EMPTY error. Errors:
* @return The stream, or an error. An empty input will yield 0 documents rather than an EMPTY error. Errors:
* - IO_ERROR if there was an error opening or reading the file.
* - MEMALLOC if the parser does not have enough capacity and memory allocation fails.
* - CAPACITY if the parser does not have enough capacity and batch_size > max_capacity.
* - other json errors if parsing fails.
*/
inline document_stream load_many(const std::string &path, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept;
inline simdjson_result<document_stream> load_many(const std::string &path, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept;
/**
* Parse a buffer containing many JSON documents.
*
* dom::parser parser;
* for (const element doc : parser.parse_many(buf, len)) {
* for (element doc : parser.parse_many(buf, len)) {
* cout << std::string(doc["title"]) << endl;
* }
*
@@ -234,9 +237,13 @@ public:
* the same interface, requiring you to check the error before using the document:
*
* dom::parser parser;
* for (auto [doc, error] : parser.parse_many(buf, len)) {
* if (error) { cerr << error << endl; exit(1); }
* cout << std::string(doc["title"]) << endl;
* dom::document_stream docs;
* auto error = parser.load_many(path).get(docs);
* if (error) { cerr << error << endl; exit(1); }
* for (auto doc : docs) {
* std::string_view title;
* if ((error = doc["title"].get(title)) { cerr << error << endl; exit(1); }
* cout << title << endl;
* }
*
* ### REQUIRED: Buffer Padding
@@ -260,22 +267,21 @@ public:
* spot is cache-related: small enough to fit in cache, yet big enough to
* parse as many documents as possible in one tight loop.
* Defaults to 10MB, which has been a reasonable sweet spot in our tests.
* @return The stream. If there is an error, it will be returned during iteration. An empty input
* will yield 0 documents rather than an EMPTY error. Errors:
* @return The stream, or an error. An empty input will yield 0 documents rather than an EMPTY error. Errors:
* - MEMALLOC if the parser does not have enough capacity and memory allocation fails
* - CAPACITY if the parser does not have enough capacity and batch_size > max_capacity.
* - other json errors if parsing fails.
*/
inline document_stream parse_many(const uint8_t *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept;
inline simdjson_result<document_stream> parse_many(const uint8_t *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept;
/** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */
inline document_stream parse_many(const char *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept;
inline simdjson_result<document_stream> parse_many(const char *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept;
/** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */
inline document_stream parse_many(const std::string &s, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept;
inline simdjson_result<document_stream> parse_many(const std::string &s, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept;
/** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */
inline document_stream parse_many(const padded_string &s, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept;
inline simdjson_result<document_stream> parse_many(const padded_string &s, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept;
/** @private We do not want to allow implicit conversion from C string to std::string. */
really_inline simdjson_result<element> parse_many(const char *buf, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept = delete;
simdjson_result<document_stream> parse_many(const char *buf, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept = delete;
/**
* Ensure this parser has enough memory to process JSON documents up to `capacity` bytes in length
include/simdjson/error.h
+2 -1
View File
@@ -42,7 +42,8 @@ enum error_code {
* Get the error message for the given error code.
*
* dom::parser parser;
* auto [doc, error] = parser.parse("foo");
* dom::element doc;
* auto error = parser.parse("foo").get(doc);
* if (error) { printf("Error: %s\n", error_message(error)); }
*
* @return The error message.
include/simdjson/inline/document_stream.h
+49 -24
View File
@@ -66,11 +66,11 @@ inline void stage1_worker::run(document_stream * ds, dom::parser * stage1, size_
really_inline document_stream::document_stream(
dom::parser &_parser,
size_t _batch_size,
const uint8_t *_buf,
size_t _len
size_t _len,
size_t _batch_size
) noexcept
: parser{_parser},
: parser{&_parser},
buf{_buf},
len{_len},
batch_size{_batch_size},
@@ -83,21 +83,15 @@ really_inline document_stream::document_stream(
#endif
}
really_inline document_stream::document_stream(
dom::parser &_parser,
size_t _batch_size,
error_code _error
) noexcept
: parser{_parser},
really_inline document_stream::document_stream() noexcept
: parser{nullptr},
buf{nullptr},
len{0},
batch_size{_batch_size},
error{_error}
{
assert(_error);
batch_size{0},
error{UNINITIALIZED} {
}
inline document_stream::~document_stream() noexcept {
really_inline document_stream::~document_stream() noexcept {
}
really_inline document_stream::iterator document_stream::begin() noexcept {
@@ -117,7 +111,7 @@ really_inline document_stream::iterator::iterator(document_stream& _stream, bool
really_inline simdjson_result<element> document_stream::iterator::operator*() noexcept {
// Once we have yielded any errors, we're finished.
if (stream.error) { finished = true; return stream.error; }
return stream.parser.doc.root();
return stream.parser->doc.root();
}
really_inline document_stream::iterator& document_stream::iterator::operator++() noexcept {
@@ -134,12 +128,12 @@ really_inline bool document_stream::iterator::operator!=(const document_stream::
inline void document_stream::start() noexcept {
if (error) { return; }
error = parser.ensure_capacity(batch_size);
error = parser->ensure_capacity(batch_size);
if (error) { return; }
// Always run the first stage 1 parse immediately
batch_start = 0;
error = run_stage1(parser, batch_start);
error = run_stage1(*parser, batch_start);
if (error) { return; }
#ifdef SIMDJSON_THREADS_ENABLED
@@ -163,8 +157,8 @@ inline void document_stream::next() noexcept {
if (error) { return; }
// Load the next document from the batch
doc_index = batch_start + parser.implementation->structural_indexes[parser.implementation->next_structural_index];
error = parser.implementation->stage2_next(parser.doc);
doc_index = batch_start + parser->implementation->structural_indexes[parser->implementation->next_structural_index];
error = parser->implementation->stage2_next(parser->doc);
// If that was the last document in the batch, load another batch (if available)
while (error == EMPTY) {
batch_start = next_batch_start();
@@ -173,17 +167,17 @@ inline void document_stream::next() noexcept {
#ifdef SIMDJSON_THREADS_ENABLED
load_from_stage1_thread();
#else
error = run_stage1(parser, batch_start);
error = run_stage1(*parser, batch_start);
#endif
if (error) { continue; } // If the error was EMPTY, we may want to load another batch.
// Run stage 2 on the first document in the batch
doc_index = batch_start + parser.implementation->structural_indexes[parser.implementation->next_structural_index];
error = parser.implementation->stage2_next(parser.doc);
doc_index = batch_start + parser->implementation->structural_indexes[parser->implementation->next_structural_index];
error = parser->implementation->stage2_next(parser->doc);
}
}
inline size_t document_stream::next_batch_start() const noexcept {
return batch_start + parser.implementation->structural_indexes[parser.implementation->n_structural_indexes];
return batch_start + parser->implementation->structural_indexes[parser->implementation->n_structural_indexes];
}
inline error_code document_stream::run_stage1(dom::parser &p, size_t _batch_start) noexcept {
@@ -202,7 +196,7 @@ inline void document_stream::load_from_stage1_thread() noexcept {
worker->finish();
// Swap to the parser that was loaded up in the thread. Make sure the parser has
// enough memory to swap to, as well.
std::swap(parser, stage1_thread_parser);
std::swap(*parser, stage1_thread_parser);
error = stage1_thread_error;
if (error) { return; }
@@ -226,5 +220,36 @@ inline void document_stream::start_stage1_thread() noexcept {
#endif // SIMDJSON_THREADS_ENABLED
} // namespace dom
really_inline simdjson_result<dom::document_stream>::simdjson_result() noexcept
: simdjson_result_base() {
}
really_inline simdjson_result<dom::document_stream>::simdjson_result(error_code error) noexcept
: simdjson_result_base(error) {
}
really_inline simdjson_result<dom::document_stream>::simdjson_result(dom::document_stream &&value) noexcept
: simdjson_result_base(std::forward<dom::document_stream>(value)) {
}
#if SIMDJSON_EXCEPTIONS
really_inline dom::document_stream::iterator simdjson_result<dom::document_stream>::begin() noexcept(false) {
if (error()) { throw simdjson_error(error()); }
return first.begin();
}
really_inline dom::document_stream::iterator simdjson_result<dom::document_stream>::end() noexcept(false) {
if (error()) { throw simdjson_error(error()); }
return first.end();
}
#else // SIMDJSON_EXCEPTIONS
really_inline dom::document_stream::iterator simdjson_result<dom::document_stream>::begin() noexcept {
first.error = error();
return first.begin();
}
really_inline dom::document_stream::iterator simdjson_result<dom::document_stream>::end() noexcept {
first.error = error();
return first.end();
}
#endif // SIMDJSON_EXCEPTIONS
} // namespace simdjson
#endif // SIMDJSON_INLINE_DOCUMENT_STREAM_H
include/simdjson/inline/element.h
+18 -27
View File
@@ -24,9 +24,8 @@ inline simdjson_result<dom::element_type> simdjson_result<dom::element>::type()
}
template<typename T>
really_inline simdjson_result<bool> simdjson_result<dom::element>::is() const noexcept {
if (error()) { return error(); }
return first.is<T>();
really_inline bool simdjson_result<dom::element>::is() const noexcept {
return !error() && first.is<T>();
}
template<typename T>
really_inline simdjson_result<T> simdjson_result<dom::element>::get() const noexcept {
@@ -72,38 +71,30 @@ really_inline simdjson_result<bool> simdjson_result<dom::element>::get_bool() co
return first.get_bool();
}
really_inline simdjson_result<bool> simdjson_result<dom::element>::is_array() const noexcept {
if (error()) { return error(); }
return first.is_array();
really_inline bool simdjson_result<dom::element>::is_array() const noexcept {
return !error() && first.is_array();
}
really_inline simdjson_result<bool> simdjson_result<dom::element>::is_object() const noexcept {
if (error()) { return error(); }
return first.is_object();
really_inline bool simdjson_result<dom::element>::is_object() const noexcept {
return !error() && first.is_object();
}
really_inline simdjson_result<bool> simdjson_result<dom::element>::is_string() const noexcept {
if (error()) { return error(); }
return first.is_string();
really_inline bool simdjson_result<dom::element>::is_string() const noexcept {
return !error() && first.is_string();
}
really_inline simdjson_result<bool> simdjson_result<dom::element>::is_int64_t() const noexcept {
if (error()) { return error(); }
return first.is_int64_t();
really_inline bool simdjson_result<dom::element>::is_int64_t() const noexcept {
return !error() && first.is_int64_t();
}
really_inline simdjson_result<bool> simdjson_result<dom::element>::is_uint64_t() const noexcept {
if (error()) { return error(); }
return first.is_uint64_t();
really_inline bool simdjson_result<dom::element>::is_uint64_t() const noexcept {
return !error() && first.is_uint64_t();
}
really_inline simdjson_result<bool> simdjson_result<dom::element>::is_double() const noexcept {
if (error()) { return error(); }
return first.is_double();
really_inline bool simdjson_result<dom::element>::is_double() const noexcept {
return !error() && first.is_double();
}
really_inline simdjson_result<bool> simdjson_result<dom::element>::is_bool() const noexcept {
if (error()) { return error(); }
return first.is_bool();
really_inline bool simdjson_result<dom::element>::is_bool() const noexcept {
return !error() && first.is_bool();
}
really_inline simdjson_result<bool> simdjson_result<dom::element>::is_null() const noexcept {
if (error()) { return error(); }
return first.is_null();
really_inline bool simdjson_result<dom::element>::is_null() const noexcept {
return !error() && first.is_null();
}
really_inline simdjson_result<dom::element> simdjson_result<dom::element>::operator[](const std::string_view &key) const noexcept {
include/simdjson/inline/parser.h
+8 -11
View File
@@ -80,17 +80,14 @@ inline simdjson_result<element> parser::load(const std::string &path) & noexcept
size_t len;
auto _error = read_file(path).get(len);
if (_error) { return _error; }
return parse(loaded_bytes.get(), len, false);
}
inline document_stream parser::load_many(const std::string &path, size_t batch_size) noexcept {
inline simdjson_result<document_stream> parser::load_many(const std::string &path, size_t batch_size) noexcept {
size_t len;
auto _error = read_file(path).get(len);
if (_error) {
return document_stream(*this, batch_size, _error);
}
return document_stream(*this, batch_size, (const uint8_t*)loaded_bytes.get(), len);
if (_error) { return _error; }
return document_stream(*this, (const uint8_t*)loaded_bytes.get(), len, batch_size);
}
inline simdjson_result<element> parser::parse(const uint8_t *buf, size_t len, bool realloc_if_needed) & noexcept {
@@ -123,16 +120,16 @@ really_inline simdjson_result<element> parser::parse(const padded_string &s) & n
return parse(s.data(), s.length(), false);
}
inline document_stream parser::parse_many(const uint8_t *buf, size_t len, size_t batch_size) noexcept {
return document_stream(*this, batch_size, buf, len);
inline simdjson_result<document_stream> parser::parse_many(const uint8_t *buf, size_t len, size_t batch_size) noexcept {
return document_stream(*this, buf, len, batch_size);
}
inline document_stream parser::parse_many(const char *buf, size_t len, size_t batch_size) noexcept {
inline simdjson_result<document_stream> parser::parse_many(const char *buf, size_t len, size_t batch_size) noexcept {
return parse_many((const uint8_t *)buf, len, batch_size);
}
inline document_stream parser::parse_many(const std::string &s, size_t batch_size) noexcept {
inline simdjson_result<document_stream> parser::parse_many(const std::string &s, size_t batch_size) noexcept {
return parse_many(s.data(), s.length(), batch_size);
}
inline document_stream parser::parse_many(const padded_string &s, size_t batch_size) noexcept {
inline simdjson_result<document_stream> parser::parse_many(const padded_string &s, size_t batch_size) noexcept {
return parse_many(s.data(), s.length(), batch_size);
}
singleheader/amalgamate.sh
+7 -4
View File
@@ -135,9 +135,8 @@ int main(int argc, char *argv[]) {
}
const char * filename = argv[1];
simdjson::dom::parser parser;
simdjson::error_code error;
UNUSED simdjson::dom::element elem;
parser.load(filename).tie(elem, error); // do the parsing
auto error = parser.load(filename).get(elem); // do the parsing
if (error) {
std::cout << "parse failed" << std::endl;
std::cout << "error code: " << error << std::endl;
@@ -152,8 +151,12 @@ int main(int argc, char *argv[]) {
// parse_many
const char * filename2 = argv[2];
for (auto result : parser.load_many(filename2)) {
error = result.error();
simdjson::dom::document_stream stream;
error = parser.load_many(filename2).get(stream);
if (!error) {
for (auto result : stream) {
error = result.error();
}
}
if (error) {
std::cout << "parse_many failed" << std::endl;
singleheader/amalgamate_demo.cpp
+8 -5
View File
@@ -1,4 +1,4 @@
/* auto-generated on Fri 12 Jun 2020 13:09:36 EDT. Do not edit! */
/* auto-generated on Sun Jun 21 11:49:12 PDT 2020. Do not edit! */
#include <iostream>
#include "simdjson.h"
@@ -9,9 +9,8 @@ int main(int argc, char *argv[]) {
}
const char * filename = argv[1];
simdjson::dom::parser parser;
simdjson::error_code error;
UNUSED simdjson::dom::element elem;
parser.load(filename).tie(elem, error); // do the parsing
auto error = parser.load(filename).get(elem); // do the parsing
if (error) {
std::cout << "parse failed" << std::endl;
std::cout << "error code: " << error << std::endl;
@@ -26,8 +25,12 @@ int main(int argc, char *argv[]) {
// parse_many
const char * filename2 = argv[2];
for (auto result : parser.load_many(filename2)) {
error = result.error();
simdjson::dom::document_stream stream;
error = parser.load_many(filename2).get(stream);
if (!error) {
for (auto result : stream) {
error = result.error();
}
}
if (error) {
std::cout << "parse_many failed" << std::endl;
singleheader/simdjson.cpp
+240 -118
View File
@@ -1,4 +1,4 @@
/* auto-generated on Fri 12 Jun 2020 13:09:36 EDT. Do not edit! */
/* auto-generated on Sun Jun 21 11:49:12 PDT 2020. Do not edit! */
/* begin file src/simdjson.cpp */
#include "simdjson.h"
@@ -586,6 +586,11 @@ const implementation *detect_best_supported_implementation_on_first_use::set_bes
SIMDJSON_DLLIMPORTEXPORT const internal::available_implementation_list available_implementations{};
SIMDJSON_DLLIMPORTEXPORT internal::atomic_ptr<const implementation> active_implementation{&internal::detect_best_supported_implementation_on_first_use_singleton};
WARN_UNUSED error_code minify(const char *buf, size_t len, char *dst, size_t &dst_len) noexcept {
return active_implementation->minify((const uint8_t *)buf, len, (uint8_t *)dst, dst_len);
}
} // namespace simdjson
/* end file src/fallback/implementation.h */
@@ -2794,6 +2799,12 @@ really_inline simd8<bool> must_be_continuation(simd8<uint8_t> prev1, simd8<uint8
return is_second_byte ^ is_third_byte ^ is_fourth_byte;
}
really_inline simd8<bool> must_be_2_3_continuation(simd8<uint8_t> prev2, simd8<uint8_t> prev3) {
simd8<bool> is_third_byte = prev2 >= uint8_t(0b11100000u);
simd8<bool> is_fourth_byte = prev3 >= uint8_t(0b11110000u);
return is_third_byte ^ is_fourth_byte;
}
/* begin file src/generic/stage1/buf_block_reader.h */
// Walks through a buffer in block-sized increments, loading the last part with spaces
template<size_t STEP_SIZE>
@@ -2921,7 +2932,9 @@ public:
really_inline error_code finish(bool streaming);
private:
// Intended to be defined by the implementation
really_inline uint64_t find_escaped(uint64_t escape);
really_inline uint64_t find_escaped_branchless(uint64_t escape);
// Whether the last iteration was still inside a string (all 1's = true, all 0's = false).
uint64_t prev_in_string = 0ULL;
@@ -2956,7 +2969,7 @@ private:
// desired | x | x x x x x x x x |
// text | \\\ | \\\"\\\" \\\" \\"\\" |
//
really_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) {
really_inline uint64_t json_string_scanner::find_escaped_branchless(uint64_t backslash) {
// If there was overflow, pretend the first character isn't a backslash
backslash &= ~prev_escaped;
uint64_t follows_escape = backslash << 1 | prev_escaped;
@@ -2985,13 +2998,23 @@ really_inline json_string_block json_string_scanner::next(const simd::simd8x64<u
const uint64_t backslash = in.eq('\\');
const uint64_t escaped = find_escaped(backslash);
const uint64_t quote = in.eq('"') & ~escaped;
//
// prefix_xor flips on bits inside the string (and flips off the end quote).
//
// Then we xor with prev_in_string: if we were in a string already, its effect is flipped
// (characters inside strings are outside, and characters outside strings are inside).
//
const uint64_t in_string = prefix_xor(quote) ^ prev_in_string;
//
// Check if we're still in a string at the end of the box so the next block will know
//
// right shift of a signed value expected to be well-defined and standard
// compliant as of C++20, John Regher from Utah U. says this is fine code
//
prev_in_string = uint64_t(static_cast<int64_t>(in_string) >> 63);
// Use ^ to turn the beginning quote off, and the end quote on.
return {
backslash,
@@ -3117,6 +3140,15 @@ really_inline error_code json_scanner::finish(bool streaming) {
} // namespace stage1
/* end file src/generic/stage1/json_scanner.h */
namespace stage1 {
really_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) {
// On ARM, we don't short-circuit this if there are no backslashes, because the branch gives us no
// benefit and therefore makes things worse.
// if (!backslash) { uint64_t escaped = prev_escaped; prev_escaped = 0; return escaped; }
return find_escaped_branchless(backslash);
}
}
/* begin file src/generic/stage1/json_minifier.h */
// This file contains the common code every implementation uses in stage1
// It is intended to be included multiple times and compiled multiple times
@@ -3288,7 +3320,7 @@ really_inline static size_t trim_partial_utf8(const uint8_t *buf, size_t len) {
return len;
}
/* end file src/generic/stage1/find_next_document_index.h */
/* begin file src/generic/stage1/utf8_lookup2_algorithm.h */
/* begin file src/generic/stage1/utf8_lookup3_algorithm.h */
//
// Detect Unicode errors.
//
@@ -3380,67 +3412,79 @@ namespace utf8_validation {
static const int TOO_LARGE = 0x10; // 11110100 (1001|101_)____
static const int TOO_LARGE_2 = 0x20; // 1111(1___|011_|0101) 10______
// New with lookup3. We want to catch the case where an non-continuation
// follows a leading byte
static const int TOO_SHORT_2_3_4 = 0x40; // (110_|1110|1111) ____ (0___|110_|1111) ____
// We also want to catch a continuation that is preceded by an ASCII byte
static const int LONELY_CONTINUATION = 0x80; // 0___ ____ 01__ ____
// After processing the rest of byte 1 (the low bits), we're still not done--we have to check
// byte 2 to be sure which things are errors and which aren't.
// Since high_bits is byte 5, byte 2 is high_bits.prev<3>
static const int CARRY = OVERLONG_2 | TOO_LARGE_2;
const simd8<uint8_t> byte_2_high = input.shr<4>().lookup_16<uint8_t>(
// ASCII: ________ [0___]____
CARRY, CARRY, CARRY, CARRY,
CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4,
CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4,
// ASCII: ________ [0___]____
CARRY, CARRY, CARRY, CARRY,
CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4,
CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4,
// Continuations: ________ [10__]____
CARRY | OVERLONG_3 | OVERLONG_4, // ________ [1000]____
CARRY | OVERLONG_3 | TOO_LARGE, // ________ [1001]____
CARRY | TOO_LARGE | SURROGATE, // ________ [1010]____
CARRY | TOO_LARGE | SURROGATE, // ________ [1011]____
CARRY | OVERLONG_3 | OVERLONG_4 | LONELY_CONTINUATION, // ________ [1000]____
CARRY | OVERLONG_3 | TOO_LARGE | LONELY_CONTINUATION, // ________ [1001]____
CARRY | TOO_LARGE | SURROGATE | LONELY_CONTINUATION, // ________ [1010]____
CARRY | TOO_LARGE | SURROGATE | LONELY_CONTINUATION, // ________ [1011]____
// Multibyte Leads: ________ [11__]____
CARRY, CARRY, CARRY, CARRY
CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4, // 110_
CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4
);
const simd8<uint8_t> byte_1_high = prev1.shr<4>().lookup_16<uint8_t>(
// [0___]____ (ASCII)
0, 0, 0, 0,
0, 0, 0, 0,
LONELY_CONTINUATION, LONELY_CONTINUATION, LONELY_CONTINUATION, LONELY_CONTINUATION,
LONELY_CONTINUATION, LONELY_CONTINUATION, LONELY_CONTINUATION, LONELY_CONTINUATION,
// [10__]____ (continuation)
0, 0, 0, 0,
// [11__]____ (2+-byte leads)
OVERLONG_2, 0, // [110_]____ (2-byte lead)
OVERLONG_3 | SURROGATE, // [1110]____ (3-byte lead)
OVERLONG_4 | TOO_LARGE | TOO_LARGE_2 // [1111]____ (4+-byte lead)
OVERLONG_2 | TOO_SHORT_2_3_4, TOO_SHORT_2_3_4, // [110_]____ (2-byte lead)
OVERLONG_3 | SURROGATE | TOO_SHORT_2_3_4, // [1110]____ (3-byte lead)
OVERLONG_4 | TOO_LARGE | TOO_LARGE_2 | TOO_SHORT_2_3_4 // [1111]____ (4+-byte lead)
);
const simd8<uint8_t> byte_1_low = (prev1 & 0x0F).lookup_16<uint8_t>(
// ____[00__] ________
OVERLONG_2 | OVERLONG_3 | OVERLONG_4, // ____[0000] ________
OVERLONG_2, // ____[0001] ________
0, 0,
OVERLONG_2 | OVERLONG_3 | OVERLONG_4 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[0000] ________
OVERLONG_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[0001] ________
TOO_SHORT_2_3_4 | LONELY_CONTINUATION,
TOO_SHORT_2_3_4 | LONELY_CONTINUATION,
// ____[01__] ________
TOO_LARGE, // ____[0100] ________
TOO_LARGE_2,
TOO_LARGE_2,
TOO_LARGE_2,
TOO_LARGE | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[0100] ________
TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION,
TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION,
TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION,
// ____[10__] ________
TOO_LARGE_2, TOO_LARGE_2, TOO_LARGE_2, TOO_LARGE_2,
TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION,
TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION,
TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION,
TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION,
// ____[11__] ________
TOO_LARGE_2,
TOO_LARGE_2 | SURROGATE, // ____[1101] ________
TOO_LARGE_2, TOO_LARGE_2
TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION,
TOO_LARGE_2 | SURROGATE | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[1101] ________
TOO_LARGE_2 | TOO_SHORT_2_3_4| LONELY_CONTINUATION,
TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION
);
return byte_1_high & byte_1_low & byte_2_high;
}
really_inline simd8<uint8_t> check_multibyte_lengths(simd8<uint8_t> input, simd8<uint8_t> prev_input, simd8<uint8_t> prev1) {
really_inline simd8<uint8_t> check_multibyte_lengths(simd8<uint8_t> input, simd8<uint8_t> prev_input,
simd8<uint8_t> prev1) {
simd8<uint8_t> prev2 = input.prev<2>(prev_input);
simd8<uint8_t> prev3 = input.prev<3>(prev_input);
// Cont is 10000000-101111111 (-65...-128)
simd8<bool> is_continuation = simd8<int8_t>(input) < int8_t(-64);
// must_be_continuation is architecture-specific because Intel doesn't have unsigned comparisons
return simd8<uint8_t>(must_be_continuation(prev1, prev2, prev3) ^ is_continuation);
// is_2_3_continuation uses one more instruction than lookup2
simd8<bool> is_2_3_continuation = (simd8<int8_t>(input).max(simd8<int8_t>(prev1))) < int8_t(-64);
// must_be_2_3_continuation has two fewer instructions than lookup 2
return simd8<uint8_t>(must_be_2_3_continuation(prev2, prev3) ^ is_2_3_continuation);
}
//
// Return nonzero if there are incomplete multibyte characters at the end of the block:
// e.g. if there is a 4-byte character, but it's 3 bytes from the end.
@@ -3507,7 +3551,7 @@ namespace utf8_validation {
}
using utf8_validation::utf8_checker;
/* end file src/generic/stage1/utf8_lookup2_algorithm.h */
/* end file src/generic/stage1/utf8_lookup3_algorithm.h */
/* begin file src/generic/stage1/json_structural_indexer.h */
// This file contains the common code every implementation uses in stage1
// It is intended to be included multiple times and compiled multiple times
@@ -4432,7 +4476,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const src,
}
// we over-decrement by one when there is a '.'
digit_count -= int(start - start_digits);
if (unlikely(digit_count >= 19)) {
if (digit_count >= 19) {
// Ok, chances are good that we had an overflow!
// this is almost never going to get called!!!
// we start anew, going slowly!!!
@@ -4442,7 +4486,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const src,
//
bool success = slow_float_parsing((const char *) src, writer);
// The number was already written, but we made a copy of the writer
// when we passed it to the parse_large_integer() function, so
// when we passed it to the parse_large_integer() function, so
writer.skip_double();
return success;
}
@@ -4481,7 +4525,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const src,
// need to recover: we parse the whole thing again.
bool success = parse_large_integer(src, writer, found_minus);
// The number was already written, but we made a copy of the writer
// when we passed it to the parse_large_integer() function, so
// when we passed it to the parse_large_integer() function, so
writer.skip_large_integer();
return success;
}
@@ -6525,7 +6569,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const src,
}
// we over-decrement by one when there is a '.'
digit_count -= int(start - start_digits);
if (unlikely(digit_count >= 19)) {
if (digit_count >= 19) {
// Ok, chances are good that we had an overflow!
// this is almost never going to get called!!!
// we start anew, going slowly!!!
@@ -6535,7 +6579,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const src,
//
bool success = slow_float_parsing((const char *) src, writer);
// The number was already written, but we made a copy of the writer
// when we passed it to the parse_large_integer() function, so
// when we passed it to the parse_large_integer() function, so
writer.skip_double();
return success;
}
@@ -6574,7 +6618,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const src,
// need to recover: we parse the whole thing again.
bool success = parse_large_integer(src, writer, found_minus);
// The number was already written, but we made a copy of the writer
// when we passed it to the parse_large_integer() function, so
// when we passed it to the parse_large_integer() function, so
writer.skip_large_integer();
return success;
}
@@ -8119,6 +8163,14 @@ really_inline simd8<bool> must_be_continuation(simd8<uint8_t> prev1, simd8<uint8
return simd8<int8_t>(is_second_byte | is_third_byte | is_fourth_byte) > int8_t(0);
}
really_inline simd8<bool> must_be_2_3_continuation(simd8<uint8_t> prev2, simd8<uint8_t> prev3) {
simd8<uint8_t> is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0
simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0
// Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine.
return simd8<int8_t>(is_third_byte | is_fourth_byte) > int8_t(0);
}
/* begin file src/generic/stage1/buf_block_reader.h */
// Walks through a buffer in block-sized increments, loading the last part with spaces
template<size_t STEP_SIZE>
@@ -8246,7 +8298,9 @@ public:
really_inline error_code finish(bool streaming);
private:
// Intended to be defined by the implementation
really_inline uint64_t find_escaped(uint64_t escape);
really_inline uint64_t find_escaped_branchless(uint64_t escape);
// Whether the last iteration was still inside a string (all 1's = true, all 0's = false).
uint64_t prev_in_string = 0ULL;
@@ -8281,7 +8335,7 @@ private:
// desired | x | x x x x x x x x |
// text | \\\ | \\\"\\\" \\\" \\"\\" |
//
really_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) {
really_inline uint64_t json_string_scanner::find_escaped_branchless(uint64_t backslash) {
// If there was overflow, pretend the first character isn't a backslash
backslash &= ~prev_escaped;
uint64_t follows_escape = backslash << 1 | prev_escaped;
@@ -8310,13 +8364,23 @@ really_inline json_string_block json_string_scanner::next(const simd::simd8x64<u
const uint64_t backslash = in.eq('\\');
const uint64_t escaped = find_escaped(backslash);
const uint64_t quote = in.eq('"') & ~escaped;
//
// prefix_xor flips on bits inside the string (and flips off the end quote).
//
// Then we xor with prev_in_string: if we were in a string already, its effect is flipped
// (characters inside strings are outside, and characters outside strings are inside).
//
const uint64_t in_string = prefix_xor(quote) ^ prev_in_string;
//
// Check if we're still in a string at the end of the box so the next block will know
//
// right shift of a signed value expected to be well-defined and standard
// compliant as of C++20, John Regher from Utah U. says this is fine code
//
prev_in_string = uint64_t(static_cast<int64_t>(in_string) >> 63);
// Use ^ to turn the beginning quote off, and the end quote on.
return {
backslash,
@@ -8442,6 +8506,13 @@ really_inline error_code json_scanner::finish(bool streaming) {
} // namespace stage1
/* end file src/generic/stage1/json_scanner.h */
namespace stage1 {
really_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) {
if (!backslash) { uint64_t escaped = prev_escaped; prev_escaped = 0; return escaped; }
return find_escaped_branchless(backslash);
}
}
/* begin file src/generic/stage1/json_minifier.h */
// This file contains the common code every implementation uses in stage1
// It is intended to be included multiple times and compiled multiple times
@@ -8613,7 +8684,7 @@ really_inline static size_t trim_partial_utf8(const uint8_t *buf, size_t len) {
return len;
}
/* end file src/generic/stage1/find_next_document_index.h */
/* begin file src/generic/stage1/utf8_lookup2_algorithm.h */
/* begin file src/generic/stage1/utf8_lookup3_algorithm.h */
//
// Detect Unicode errors.
//
@@ -8705,67 +8776,79 @@ namespace utf8_validation {
static const int TOO_LARGE = 0x10; // 11110100 (1001|101_)____
static const int TOO_LARGE_2 = 0x20; // 1111(1___|011_|0101) 10______
// New with lookup3. We want to catch the case where an non-continuation
// follows a leading byte
static const int TOO_SHORT_2_3_4 = 0x40; // (110_|1110|1111) ____ (0___|110_|1111) ____
// We also want to catch a continuation that is preceded by an ASCII byte
static const int LONELY_CONTINUATION = 0x80; // 0___ ____ 01__ ____
// After processing the rest of byte 1 (the low bits), we're still not done--we have to check
// byte 2 to be sure which things are errors and which aren't.
// Since high_bits is byte 5, byte 2 is high_bits.prev<3>
static const int CARRY = OVERLONG_2 | TOO_LARGE_2;
const simd8<uint8_t> byte_2_high = input.shr<4>().lookup_16<uint8_t>(
// ASCII: ________ [0___]____
CARRY, CARRY, CARRY, CARRY,
CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4,
CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4,
// ASCII: ________ [0___]____
CARRY, CARRY, CARRY, CARRY,
CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4,
CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4,
// Continuations: ________ [10__]____
CARRY | OVERLONG_3 | OVERLONG_4, // ________ [1000]____
CARRY | OVERLONG_3 | TOO_LARGE, // ________ [1001]____
CARRY | TOO_LARGE | SURROGATE, // ________ [1010]____
CARRY | TOO_LARGE | SURROGATE, // ________ [1011]____
CARRY | OVERLONG_3 | OVERLONG_4 | LONELY_CONTINUATION, // ________ [1000]____
CARRY | OVERLONG_3 | TOO_LARGE | LONELY_CONTINUATION, // ________ [1001]____
CARRY | TOO_LARGE | SURROGATE | LONELY_CONTINUATION, // ________ [1010]____
CARRY | TOO_LARGE | SURROGATE | LONELY_CONTINUATION, // ________ [1011]____
// Multibyte Leads: ________ [11__]____
CARRY, CARRY, CARRY, CARRY
CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4, // 110_
CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4
);
const simd8<uint8_t> byte_1_high = prev1.shr<4>().lookup_16<uint8_t>(
// [0___]____ (ASCII)
0, 0, 0, 0,
0, 0, 0, 0,
LONELY_CONTINUATION, LONELY_CONTINUATION, LONELY_CONTINUATION, LONELY_CONTINUATION,
LONELY_CONTINUATION, LONELY_CONTINUATION, LONELY_CONTINUATION, LONELY_CONTINUATION,
// [10__]____ (continuation)
0, 0, 0, 0,
// [11__]____ (2+-byte leads)
OVERLONG_2, 0, // [110_]____ (2-byte lead)
OVERLONG_3 | SURROGATE, // [1110]____ (3-byte lead)
OVERLONG_4 | TOO_LARGE | TOO_LARGE_2 // [1111]____ (4+-byte lead)
OVERLONG_2 | TOO_SHORT_2_3_4, TOO_SHORT_2_3_4, // [110_]____ (2-byte lead)
OVERLONG_3 | SURROGATE | TOO_SHORT_2_3_4, // [1110]____ (3-byte lead)
OVERLONG_4 | TOO_LARGE | TOO_LARGE_2 | TOO_SHORT_2_3_4 // [1111]____ (4+-byte lead)
);
const simd8<uint8_t> byte_1_low = (prev1 & 0x0F).lookup_16<uint8_t>(
// ____[00__] ________
OVERLONG_2 | OVERLONG_3 | OVERLONG_4, // ____[0000] ________
OVERLONG_2, // ____[0001] ________
0, 0,
OVERLONG_2 | OVERLONG_3 | OVERLONG_4 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[0000] ________
OVERLONG_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[0001] ________
TOO_SHORT_2_3_4 | LONELY_CONTINUATION,
TOO_SHORT_2_3_4 | LONELY_CONTINUATION,
// ____[01__] ________
TOO_LARGE, // ____[0100] ________
TOO_LARGE_2,
TOO_LARGE_2,
TOO_LARGE_2,
TOO_LARGE | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[0100] ________
TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION,
TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION,
TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION,
// ____[10__] ________
TOO_LARGE_2, TOO_LARGE_2, TOO_LARGE_2, TOO_LARGE_2,
TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION,
TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION,
TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION,
TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION,
// ____[11__] ________
TOO_LARGE_2,
TOO_LARGE_2 | SURROGATE, // ____[1101] ________
TOO_LARGE_2, TOO_LARGE_2
TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION,
TOO_LARGE_2 | SURROGATE | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[1101] ________
TOO_LARGE_2 | TOO_SHORT_2_3_4| LONELY_CONTINUATION,
TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION
);
return byte_1_high & byte_1_low & byte_2_high;
}
really_inline simd8<uint8_t> check_multibyte_lengths(simd8<uint8_t> input, simd8<uint8_t> prev_input, simd8<uint8_t> prev1) {
really_inline simd8<uint8_t> check_multibyte_lengths(simd8<uint8_t> input, simd8<uint8_t> prev_input,
simd8<uint8_t> prev1) {
simd8<uint8_t> prev2 = input.prev<2>(prev_input);
simd8<uint8_t> prev3 = input.prev<3>(prev_input);
// Cont is 10000000-101111111 (-65...-128)
simd8<bool> is_continuation = simd8<int8_t>(input) < int8_t(-64);
// must_be_continuation is architecture-specific because Intel doesn't have unsigned comparisons
return simd8<uint8_t>(must_be_continuation(prev1, prev2, prev3) ^ is_continuation);
// is_2_3_continuation uses one more instruction than lookup2
simd8<bool> is_2_3_continuation = (simd8<int8_t>(input).max(simd8<int8_t>(prev1))) < int8_t(-64);
// must_be_2_3_continuation has two fewer instructions than lookup 2
return simd8<uint8_t>(must_be_2_3_continuation(prev2, prev3) ^ is_2_3_continuation);
}
//
// Return nonzero if there are incomplete multibyte characters at the end of the block:
// e.g. if there is a 4-byte character, but it's 3 bytes from the end.
@@ -8832,7 +8915,7 @@ namespace utf8_validation {
}
using utf8_validation::utf8_checker;
/* end file src/generic/stage1/utf8_lookup2_algorithm.h */
/* end file src/generic/stage1/utf8_lookup3_algorithm.h */
/* begin file src/generic/stage1/json_structural_indexer.h */
// This file contains the common code every implementation uses in stage1
// It is intended to be included multiple times and compiled multiple times
@@ -9762,7 +9845,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const src,
}
// we over-decrement by one when there is a '.'
digit_count -= int(start - start_digits);
if (unlikely(digit_count >= 19)) {
if (digit_count >= 19) {
// Ok, chances are good that we had an overflow!
// this is almost never going to get called!!!
// we start anew, going slowly!!!
@@ -9772,7 +9855,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const src,
//
bool success = slow_float_parsing((const char *) src, writer);
// The number was already written, but we made a copy of the writer
// when we passed it to the parse_large_integer() function, so
// when we passed it to the parse_large_integer() function, so
writer.skip_double();
return success;
}
@@ -9811,7 +9894,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const src,
// need to recover: we parse the whole thing again.
bool success = parse_large_integer(src, writer, found_minus);
// The number was already written, but we made a copy of the writer
// when we passed it to the parse_large_integer() function, so
// when we passed it to the parse_large_integer() function, so
writer.skip_large_integer();
return success;
}
@@ -11327,6 +11410,14 @@ really_inline simd8<bool> must_be_continuation(simd8<uint8_t> prev1, simd8<uint8
return simd8<int8_t>(is_second_byte | is_third_byte | is_fourth_byte) > int8_t(0);
}
really_inline simd8<bool> must_be_2_3_continuation(simd8<uint8_t> prev2, simd8<uint8_t> prev3) {
simd8<uint8_t> is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0
simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0
// Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine.
return simd8<int8_t>(is_third_byte | is_fourth_byte) > int8_t(0);
}
/* begin file src/generic/stage1/buf_block_reader.h */
// Walks through a buffer in block-sized increments, loading the last part with spaces
template<size_t STEP_SIZE>
@@ -11454,7 +11545,9 @@ public:
really_inline error_code finish(bool streaming);
private:
// Intended to be defined by the implementation
really_inline uint64_t find_escaped(uint64_t escape);
really_inline uint64_t find_escaped_branchless(uint64_t escape);
// Whether the last iteration was still inside a string (all 1's = true, all 0's = false).
uint64_t prev_in_string = 0ULL;
@@ -11489,7 +11582,7 @@ private:
// desired | x | x x x x x x x x |
// text | \\\ | \\\"\\\" \\\" \\"\\" |
//
really_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) {
really_inline uint64_t json_string_scanner::find_escaped_branchless(uint64_t backslash) {
// If there was overflow, pretend the first character isn't a backslash
backslash &= ~prev_escaped;
uint64_t follows_escape = backslash << 1 | prev_escaped;
@@ -11518,13 +11611,23 @@ really_inline json_string_block json_string_scanner::next(const simd::simd8x64<u
const uint64_t backslash = in.eq('\\');
const uint64_t escaped = find_escaped(backslash);
const uint64_t quote = in.eq('"') & ~escaped;
//
// prefix_xor flips on bits inside the string (and flips off the end quote).
//
// Then we xor with prev_in_string: if we were in a string already, its effect is flipped
// (characters inside strings are outside, and characters outside strings are inside).
//
const uint64_t in_string = prefix_xor(quote) ^ prev_in_string;
//
// Check if we're still in a string at the end of the box so the next block will know
//
// right shift of a signed value expected to be well-defined and standard
// compliant as of C++20, John Regher from Utah U. says this is fine code
//
prev_in_string = uint64_t(static_cast<int64_t>(in_string) >> 63);
// Use ^ to turn the beginning quote off, and the end quote on.
return {
backslash,
@@ -11650,6 +11753,13 @@ really_inline error_code json_scanner::finish(bool streaming) {
} // namespace stage1
/* end file src/generic/stage1/json_scanner.h */
namespace stage1 {
really_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) {
if (!backslash) { uint64_t escaped = prev_escaped; prev_escaped = 0; return escaped; }
return find_escaped_branchless(backslash);
}
}
/* begin file src/generic/stage1/json_minifier.h */
// This file contains the common code every implementation uses in stage1
// It is intended to be included multiple times and compiled multiple times
@@ -11821,7 +11931,7 @@ really_inline static size_t trim_partial_utf8(const uint8_t *buf, size_t len) {
return len;
}
/* end file src/generic/stage1/find_next_document_index.h */
/* begin file src/generic/stage1/utf8_lookup2_algorithm.h */
/* begin file src/generic/stage1/utf8_lookup3_algorithm.h */
//
// Detect Unicode errors.
//
@@ -11913,67 +12023,79 @@ namespace utf8_validation {
static const int TOO_LARGE = 0x10; // 11110100 (1001|101_)____
static const int TOO_LARGE_2 = 0x20; // 1111(1___|011_|0101) 10______
// New with lookup3. We want to catch the case where an non-continuation
// follows a leading byte
static const int TOO_SHORT_2_3_4 = 0x40; // (110_|1110|1111) ____ (0___|110_|1111) ____
// We also want to catch a continuation that is preceded by an ASCII byte
static const int LONELY_CONTINUATION = 0x80; // 0___ ____ 01__ ____
// After processing the rest of byte 1 (the low bits), we're still not done--we have to check
// byte 2 to be sure which things are errors and which aren't.
// Since high_bits is byte 5, byte 2 is high_bits.prev<3>
static const int CARRY = OVERLONG_2 | TOO_LARGE_2;
const simd8<uint8_t> byte_2_high = input.shr<4>().lookup_16<uint8_t>(
// ASCII: ________ [0___]____
CARRY, CARRY, CARRY, CARRY,
CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4,
CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4,
// ASCII: ________ [0___]____
CARRY, CARRY, CARRY, CARRY,
CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4,
CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4,
// Continuations: ________ [10__]____
CARRY | OVERLONG_3 | OVERLONG_4, // ________ [1000]____
CARRY | OVERLONG_3 | TOO_LARGE, // ________ [1001]____
CARRY | TOO_LARGE | SURROGATE, // ________ [1010]____
CARRY | TOO_LARGE | SURROGATE, // ________ [1011]____
CARRY | OVERLONG_3 | OVERLONG_4 | LONELY_CONTINUATION, // ________ [1000]____
CARRY | OVERLONG_3 | TOO_LARGE | LONELY_CONTINUATION, // ________ [1001]____
CARRY | TOO_LARGE | SURROGATE | LONELY_CONTINUATION, // ________ [1010]____
CARRY | TOO_LARGE | SURROGATE | LONELY_CONTINUATION, // ________ [1011]____
// Multibyte Leads: ________ [11__]____
CARRY, CARRY, CARRY, CARRY
CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4, // 110_
CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4
);
const simd8<uint8_t> byte_1_high = prev1.shr<4>().lookup_16<uint8_t>(
// [0___]____ (ASCII)
0, 0, 0, 0,
0, 0, 0, 0,
LONELY_CONTINUATION, LONELY_CONTINUATION, LONELY_CONTINUATION, LONELY_CONTINUATION,
LONELY_CONTINUATION, LONELY_CONTINUATION, LONELY_CONTINUATION, LONELY_CONTINUATION,
// [10__]____ (continuation)
0, 0, 0, 0,
// [11__]____ (2+-byte leads)
OVERLONG_2, 0, // [110_]____ (2-byte lead)
OVERLONG_3 | SURROGATE, // [1110]____ (3-byte lead)
OVERLONG_4 | TOO_LARGE | TOO_LARGE_2 // [1111]____ (4+-byte lead)
OVERLONG_2 | TOO_SHORT_2_3_4, TOO_SHORT_2_3_4, // [110_]____ (2-byte lead)
OVERLONG_3 | SURROGATE | TOO_SHORT_2_3_4, // [1110]____ (3-byte lead)
OVERLONG_4 | TOO_LARGE | TOO_LARGE_2 | TOO_SHORT_2_3_4 // [1111]____ (4+-byte lead)
);
const simd8<uint8_t> byte_1_low = (prev1 & 0x0F).lookup_16<uint8_t>(
// ____[00__] ________
OVERLONG_2 | OVERLONG_3 | OVERLONG_4, // ____[0000] ________
OVERLONG_2, // ____[0001] ________
0, 0,
OVERLONG_2 | OVERLONG_3 | OVERLONG_4 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[0000] ________
OVERLONG_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[0001] ________
TOO_SHORT_2_3_4 | LONELY_CONTINUATION,
TOO_SHORT_2_3_4 | LONELY_CONTINUATION,
// ____[01__] ________
TOO_LARGE, // ____[0100] ________
TOO_LARGE_2,
TOO_LARGE_2,
TOO_LARGE_2,
TOO_LARGE | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[0100] ________
TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION,
TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION,
TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION,
// ____[10__] ________
TOO_LARGE_2, TOO_LARGE_2, TOO_LARGE_2, TOO_LARGE_2,
TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION,
TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION,
TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION,
TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION,
// ____[11__] ________
TOO_LARGE_2,
TOO_LARGE_2 | SURROGATE, // ____[1101] ________
TOO_LARGE_2, TOO_LARGE_2
TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION,
TOO_LARGE_2 | SURROGATE | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[1101] ________
TOO_LARGE_2 | TOO_SHORT_2_3_4| LONELY_CONTINUATION,
TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION
);
return byte_1_high & byte_1_low & byte_2_high;
}
really_inline simd8<uint8_t> check_multibyte_lengths(simd8<uint8_t> input, simd8<uint8_t> prev_input, simd8<uint8_t> prev1) {
really_inline simd8<uint8_t> check_multibyte_lengths(simd8<uint8_t> input, simd8<uint8_t> prev_input,
simd8<uint8_t> prev1) {
simd8<uint8_t> prev2 = input.prev<2>(prev_input);
simd8<uint8_t> prev3 = input.prev<3>(prev_input);
// Cont is 10000000-101111111 (-65...-128)
simd8<bool> is_continuation = simd8<int8_t>(input) < int8_t(-64);
// must_be_continuation is architecture-specific because Intel doesn't have unsigned comparisons
return simd8<uint8_t>(must_be_continuation(prev1, prev2, prev3) ^ is_continuation);
// is_2_3_continuation uses one more instruction than lookup2
simd8<bool> is_2_3_continuation = (simd8<int8_t>(input).max(simd8<int8_t>(prev1))) < int8_t(-64);
// must_be_2_3_continuation has two fewer instructions than lookup 2
return simd8<uint8_t>(must_be_2_3_continuation(prev2, prev3) ^ is_2_3_continuation);
}
//
// Return nonzero if there are incomplete multibyte characters at the end of the block:
// e.g. if there is a 4-byte character, but it's 3 bytes from the end.
@@ -12040,7 +12162,7 @@ namespace utf8_validation {
}
using utf8_validation::utf8_checker;
/* end file src/generic/stage1/utf8_lookup2_algorithm.h */
/* end file src/generic/stage1/utf8_lookup3_algorithm.h */
/* begin file src/generic/stage1/json_structural_indexer.h */
// This file contains the common code every implementation uses in stage1
// It is intended to be included multiple times and compiled multiple times
@@ -12973,7 +13095,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const src,
}
// we over-decrement by one when there is a '.'
digit_count -= int(start - start_digits);
if (unlikely(digit_count >= 19)) {
if (digit_count >= 19) {
// Ok, chances are good that we had an overflow!
// this is almost never going to get called!!!
// we start anew, going slowly!!!
@@ -12983,7 +13105,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const src,
//
bool success = slow_float_parsing((const char *) src, writer);
// The number was already written, but we made a copy of the writer
// when we passed it to the parse_large_integer() function, so
// when we passed it to the parse_large_integer() function, so
writer.skip_double();
return success;
}
@@ -13022,7 +13144,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const src,
// need to recover: we parse the whole thing again.
bool success = parse_large_integer(src, writer, found_minus);
// The number was already written, but we made a copy of the writer
// when we passed it to the parse_large_integer() function, so
// when we passed it to the parse_large_integer() function, so
writer.skip_large_integer();
return success;
}
singleheader/simdjson.h
+807 -226
View File
File diff suppressed because it is too large Load Diff
tests/allparserscheckfile.cpp
+9 -8
View File
@@ -63,9 +63,10 @@ int main(int argc, char *argv[]) {
exit(1);
}
const char *filename = argv[optind];
auto [p, loaderr] = simdjson::padded_string::load(filename);
if (loaderr) {
std::cerr << "Could not load the file " << filename << ": " << loaderr << std::endl;
simdjson::padded_string p;
auto error = simdjson::padded_string::load(filename).get(p);
if (error) {
std::cerr << "Could not load the file " << filename << ": " << error << std::endl;
return EXIT_FAILURE;
}
if (verbose) {
@@ -79,7 +80,7 @@ int main(int argc, char *argv[]) {
std::cout << std::endl;
}
simdjson::dom::parser parser;
auto err = parser.parse(p).error();
error = parser.parse(p).error();
rapidjson::Document d;
@@ -95,19 +96,19 @@ int main(int argc, char *argv[]) {
.is_valid();
if (just_favorites) {
printf("our parser : %s \n",
(err == simdjson::error_code::SUCCESS) ? "correct" : "invalid");
(error == simdjson::error_code::SUCCESS) ? "correct" : "invalid");
printf("rapid (check encoding) : %s \n",
rapid_correct_checkencoding ? "correct" : "invalid");
printf("sajson : %s \n",
sajson_correct ? "correct" : "invalid");
if (err == simdjson::DEPTH_ERROR) {
if (error == simdjson::DEPTH_ERROR) {
printf("simdjson encountered a DEPTH_ERROR, it was parametrized to "
"reject documents with depth exceeding %zu.\n",
parser.max_depth());
}
if (((err == simdjson::error_code::SUCCESS) != rapid_correct_checkencoding) ||
if (((error == simdjson::error_code::SUCCESS) != rapid_correct_checkencoding) ||
(rapid_correct_checkencoding != sajson_correct) ||
((err == simdjson::SUCCESS) != sajson_correct)) {
((error == simdjson::SUCCESS) != sajson_correct)) {
printf("WARNING: THEY DISAGREE\n\n");
return EXIT_FAILURE;
}
tests/basictests.cpp
+333 -368
View File
File diff suppressed because it is too large Load Diff
tests/cast_tester.h
+50 -55
View File
@@ -26,6 +26,11 @@ public:
bool test_get_error(element element, error_code expected_error);
bool test_get_error(simdjson_result<element> element, error_code expected_error);
bool test_get_t(element element, T expected = {});
bool test_get_t(simdjson_result<element> element, T expected = {});
bool test_get_t_error(element element, error_code expected_error);
bool test_get_t_error(simdjson_result<element> element, error_code expected_error);
#if SIMDJSON_EXCEPTIONS
bool test_implicit_cast(element element, T expected = {});
bool test_implicit_cast(simdjson_result<element> element, T expected = {});
@@ -35,7 +40,6 @@ public:
bool test_is(element element, bool expected);
bool test_is(simdjson_result<element> element, bool expected);
bool test_is_error(simdjson_result<element> element, error_code expected_error);
bool test_named_get(element element, T expected = {});
bool test_named_get(simdjson_result<element> element, T expected = {});
@@ -44,81 +48,94 @@ public:
bool test_named_is(element element, bool expected);
bool test_named_is(simdjson_result<element> element, bool expected);
bool test_named_is_error(simdjson_result<element> element, error_code expected_error);
private:
simdjson_result<T> named_get(element element);
simdjson_result<T> named_get(simdjson_result<element> element);
bool named_is(element element);
simdjson_result<bool> named_is(simdjson_result<element> element);
bool named_is(simdjson_result<element> element);
bool assert_equal(const T& expected, const T& actual);
};
template<typename T>
bool cast_tester<T>::test_get(element element, T expected) {
T actual;
error_code error;
error = element.get(actual);
ASSERT_SUCCESS(error);
ASSERT_SUCCESS(element.get(actual));
return assert_equal(actual, expected);
}
template<typename T>
bool cast_tester<T>::test_get(simdjson_result<element> element, T expected) {
T actual;
error_code error;
error = element.get(actual);
ASSERT_SUCCESS(error);
ASSERT_SUCCESS(element.get(actual));
return assert_equal(actual, expected);
}
template<typename T>
bool cast_tester<T>::test_get_error(element element, error_code expected_error) {
T actual;
error_code error;
error = element.get(actual);
ASSERT_EQUAL(error, expected_error);
ASSERT_EQUAL(element.get(actual), expected_error);
return true;
}
template<typename T>
bool cast_tester<T>::test_get_error(simdjson_result<element> element, error_code expected_error) {
T actual;
error_code error;
error = element.get(actual);
ASSERT_EQUAL(error, expected_error);
ASSERT_EQUAL(element.get(actual), expected_error);
return true;
}
template<typename T>
bool cast_tester<T>::test_get_t(element element, T expected) {
auto actual = element.get<T>();
ASSERT_SUCCESS(actual.error());
return assert_equal(actual.first, expected);
}
template<typename T>
bool cast_tester<T>::test_get_t(simdjson_result<element> element, T expected) {
auto actual = element.get<T>();
ASSERT_SUCCESS(actual.error());
return assert_equal(actual.first, expected);
}
template<typename T>
bool cast_tester<T>::test_get_t_error(element element, error_code expected_error) {
ASSERT_EQUAL(element.get<T>().error(), expected_error);
return true;
}
template<typename T>
bool cast_tester<T>::test_get_t_error(simdjson_result<element> element, error_code expected_error) {
ASSERT_EQUAL(element.get<T>().error(), expected_error);
return true;
}
template<typename T>
bool cast_tester<T>::test_named_get(element element, T expected) {
T actual;
auto error = named_get(element).get(actual);
ASSERT_SUCCESS(error);
ASSERT_SUCCESS(named_get(element).get(actual));
return assert_equal(actual, expected);
}
template<typename T>
bool cast_tester<T>::test_named_get(simdjson_result<element> element, T expected) {
T actual;
auto error = named_get(element).get(actual);
ASSERT_SUCCESS(error);
ASSERT_SUCCESS(named_get(element).get(actual));
return assert_equal(actual, expected);
}
template<typename T>
bool cast_tester<T>::test_named_get_error(element element, error_code expected_error) {
T actual;
auto error = named_get(element).get(actual);
ASSERT_EQUAL(error, expected_error);
ASSERT_EQUAL(named_get(element).get(actual), expected_error);
return true;
}
template<typename T>
bool cast_tester<T>::test_named_get_error(simdjson_result<element> element, error_code expected_error) {
T actual;
auto error = named_get(element).get(actual);
ASSERT_EQUAL(error, expected_error);
ASSERT_EQUAL(named_get(element).get(actual), expected_error);
return true;
}
@@ -187,18 +204,7 @@ bool cast_tester<T>::test_is(element element, bool expected) {
template<typename T>
bool cast_tester<T>::test_is(simdjson_result<element> element, bool expected) {
bool actual;
auto error = element.is<T>().get(actual);
ASSERT_SUCCESS(error);
ASSERT_EQUAL(actual, expected);
return true;
}
template<typename T>
bool cast_tester<T>::test_is_error(simdjson_result<element> element, error_code expected_error) {
UNUSED bool actual;
auto error = element.is<T>().get(actual);
ASSERT_EQUAL(error, expected_error);
ASSERT_EQUAL(element.is<T>(), expected);
return true;
}
@@ -210,18 +216,7 @@ bool cast_tester<T>::test_named_is(element element, bool expected) {
template<typename T>
bool cast_tester<T>::test_named_is(simdjson_result<element> element, bool expected) {
bool actual;
auto error = named_is(element).get(actual);
ASSERT_SUCCESS(error);
ASSERT_EQUAL(actual, expected);
return true;
}
template<typename T>
bool cast_tester<T>::test_named_is_error(simdjson_result<element> element, error_code expected_error) {
bool actual;
auto error = named_is(element).get(actual);
ASSERT_EQUAL(error, expected_error);
ASSERT_EQUAL(named_is(element), expected);
return true;
}
@@ -252,14 +247,14 @@ template<> bool cast_tester<int64_t>::named_is(element element) { return element
template<> bool cast_tester<double>::named_is(element element) { return element.is_double(); }
template<> bool cast_tester<bool>::named_is(element element) { return element.is_bool(); }
template<> simdjson_result<bool> cast_tester<array>::named_is(simdjson_result<element> element) { return element.is_array(); }
template<> simdjson_result<bool> cast_tester<object>::named_is(simdjson_result<element> element) { return element.is_object(); }
template<> simdjson_result<bool> cast_tester<const char *>::named_is(simdjson_result<element> element) { return element.is_string(); }
template<> simdjson_result<bool> cast_tester<std::string_view>::named_is(simdjson_result<element> element) { return element.is_string(); }
template<> simdjson_result<bool> cast_tester<uint64_t>::named_is(simdjson_result<element> element) { return element.is_uint64_t(); }
template<> simdjson_result<bool> cast_tester<int64_t>::named_is(simdjson_result<element> element) { return element.is_int64_t(); }
template<> simdjson_result<bool> cast_tester<double>::named_is(simdjson_result<element> element) { return element.is_double(); }
template<> simdjson_result<bool> cast_tester<bool>::named_is(simdjson_result<element> element) { return element.is_bool(); }
template<> bool cast_tester<array>::named_is(simdjson_result<element> element) { return element.is_array(); }
template<> bool cast_tester<object>::named_is(simdjson_result<element> element) { return element.is_object(); }
template<> bool cast_tester<const char *>::named_is(simdjson_result<element> element) { return element.is_string(); }
template<> bool cast_tester<std::string_view>::named_is(simdjson_result<element> element) { return element.is_string(); }
template<> bool cast_tester<uint64_t>::named_is(simdjson_result<element> element) { return element.is_uint64_t(); }
template<> bool cast_tester<int64_t>::named_is(simdjson_result<element> element) { return element.is_int64_t(); }
template<> bool cast_tester<double>::named_is(simdjson_result<element> element) { return element.is_double(); }
template<> bool cast_tester<bool>::named_is(simdjson_result<element> element) { return element.is_bool(); }
template<typename T> bool cast_tester<T>::assert_equal(const T& expected, const T& actual) {
ASSERT_EQUAL(expected, actual);
tests/errortests.cpp
+34 -38
View File
@@ -14,15 +14,8 @@
using namespace simdjson;
using namespace std;
#ifndef SIMDJSON_BENCHMARK_DATA_DIR
#define SIMDJSON_BENCHMARK_DATA_DIR "jsonexamples/"
#endif
const char *TWITTER_JSON = SIMDJSON_BENCHMARK_DATA_DIR "twitter.json";
#include "test_macros.h"
#define TEST_START() { cout << "Running " << __func__ << " ..." << endl; }
#define ASSERT_ERROR(ACTUAL, EXPECTED) if ((ACTUAL) != (EXPECTED)) { cerr << "FAIL: Unexpected error \"" << (ACTUAL) << "\" (expected \"" << (EXPECTED) << "\")" << endl; return false; }
#define TEST_FAIL(MESSAGE) { cerr << "FAIL: " << (MESSAGE) << endl; return false; }
#define TEST_SUCCEED() { return true; }
namespace parser_load {
const char * NONEXISTENT_FILE = "this_file_does_not_exist.json";
bool parser_load_capacity() {
@@ -35,7 +28,9 @@ namespace parser_load {
bool parser_load_many_capacity() {
TEST_START();
dom::parser parser(1); // 1 byte max capacity
for (auto doc : parser.load_many(TWITTER_JSON)) {
dom::document_stream docs;
ASSERT_SUCCESS(parser.load_many(TWITTER_JSON).get(docs));
for (auto doc : docs) {
ASSERT_ERROR(doc.error(), CAPACITY);
TEST_SUCCEED();
}
@@ -47,17 +42,20 @@ namespace parser_load {
const padded_string DOC = "1 2 [} 3"_padded;
size_t count = 0;
dom::parser parser;
for (auto doc : parser.parse_many(DOC)) {
dom::document_stream docs;
ASSERT_SUCCESS(parser.parse_many(DOC).get(docs));
for (auto doc : docs) {
count++;
auto [val, error] = doc.get<uint64_t>();
uint64_t val;
auto error = doc.get(val);
if (count == 3) {
ASSERT_ERROR(error, TAPE_ERROR);
} else {
if (error) { TEST_FAIL(error); }
if (val != count) { cerr << "FAIL: expected " << count << ", got " << val << endl; return false; }
ASSERT_SUCCESS(error);
ASSERT_EQUAL(val, count);
}
}
if (count != 3) { cerr << "FAIL: expected 2 documents and 1 error, got " << count << " total things" << endl; return false; }
ASSERT_EQUAL(count, 3);
TEST_SUCCEED();
}
@@ -66,11 +64,13 @@ namespace parser_load {
const padded_string DOC = "["_padded;
size_t count = 0;
dom::parser parser;
for (auto doc : parser.parse_many(DOC)) {
dom::document_stream docs;
ASSERT_SUCCESS(parser.parse_many(DOC).get(docs));
for (auto doc : docs) {
count++;
ASSERT_ERROR(doc.error(), TAPE_ERROR);
}
if (count != 1) { cerr << "FAIL: expected no documents and 1 error, got " << count << " total things" << endl; return false; }
ASSERT_EQUAL(count, 1);
TEST_SUCCEED();
}
@@ -79,60 +79,56 @@ namespace parser_load {
const padded_string DOC = "1 2 ["_padded;
size_t count = 0;
dom::parser parser;
for (auto doc : parser.parse_many(DOC)) {
dom::document_stream docs;
ASSERT_SUCCESS(parser.parse_many(DOC).get(docs));
for (auto doc : docs) {
count++;
auto [val, error] = doc.get<uint64_t>();
uint64_t val;
auto error = doc.get(val);
if (count == 3) {
ASSERT_ERROR(error, TAPE_ERROR);
} else {
if (error) { TEST_FAIL(error); }
if (val != count) { cerr << "FAIL: expected " << count << ", got " << val << endl; return false; }
ASSERT_SUCCESS(error);
ASSERT_EQUAL(val, count);
}
}
if (count != 3) { cerr << "FAIL: expected 2 documents and 1 error, got " << count << " total things" << endl; return false; }
ASSERT_EQUAL(count, 3);
TEST_SUCCEED();
}
bool parser_load_nonexistent() {
TEST_START();
dom::parser parser;
auto error = parser.load(NONEXISTENT_FILE).error();
ASSERT_ERROR(error, IO_ERROR);
ASSERT_ERROR( parser.load(NONEXISTENT_FILE).error(), IO_ERROR );
TEST_SUCCEED();
}
bool parser_load_many_nonexistent() {
TEST_START();
dom::parser parser;
for (auto doc : parser.load_many(NONEXISTENT_FILE)) {
ASSERT_ERROR(doc.error(), IO_ERROR);
TEST_SUCCEED();
}
TEST_FAIL("No documents returned");
ASSERT_ERROR( parser.load_many(NONEXISTENT_FILE).error(), IO_ERROR );
TEST_SUCCEED();
}
bool padded_string_load_nonexistent() {
TEST_START();
auto error = padded_string::load(NONEXISTENT_FILE).error();
ASSERT_ERROR(error, IO_ERROR);
ASSERT_ERROR(padded_string::load(NONEXISTENT_FILE).error(), IO_ERROR);
TEST_SUCCEED();
}
bool parser_load_chain() {
TEST_START();
dom::parser parser;
auto error = parser.load(NONEXISTENT_FILE)["foo"].get<uint64_t>().error();
ASSERT_ERROR(error, IO_ERROR);
UNUSED uint64_t foo;
ASSERT_ERROR( parser.load(NONEXISTENT_FILE)["foo"].get(foo), IO_ERROR);
TEST_SUCCEED();
}
bool parser_load_many_chain() {
TEST_START();
dom::parser parser;
for (auto doc : parser.load_many(NONEXISTENT_FILE)) {
auto error = doc["foo"].get<uint64_t>().error();
ASSERT_ERROR(error, IO_ERROR);
TEST_SUCCEED();
}
TEST_FAIL("No documents returned");
UNUSED dom::document_stream stream;
ASSERT_ERROR( parser.load_many(NONEXISTENT_FILE).get(stream), IO_ERROR );
TEST_SUCCEED();
}
bool run() {
return true
&& parser_load_capacity()
tests/integer_tests.cpp
+33 -33
View File
@@ -3,6 +3,7 @@
#include <limits>
#include "simdjson.h"
#include "test_macros.h"
// we define our own asserts to get around NDEBUG
#ifndef ASSERT
@@ -29,44 +30,43 @@ template <typename T> static const std::string make_json(T value) {
}
template <typename T>
static void parse_and_validate(const std::string src, T expected) {
static bool parse_and_validate(const std::string src, T expected) {
std::cout << "src: " << src << ", ";
const padded_string pstr{src};
simdjson::dom::parser parser;
bool result;
if constexpr (std::is_same<int64_t, T>::value) {
auto [actual, error] = parser.parse(pstr).get<dom::object>()["key"].get<int64_t>();
if (error) { std::cerr << error << std::endl; abort(); }
result = (expected == actual);
int64_t actual;
ASSERT_SUCCESS( parser.parse(pstr)["key"].get(actual) );
std::cout << std::boolalpha << "test: " << (expected == actual) << std::endl;
ASSERT_EQUAL( expected, actual );
} else {
auto [actual, error] = parser.parse(pstr).get<dom::object>()["key"].get<uint64_t>();
if (error) { std::cerr << error << std::endl; abort(); }
result = (expected == actual);
}
std::cout << std::boolalpha << "test: " << result << std::endl;
if(!result) {
std::cerr << "bug detected" << std::endl;
exit(EXIT_FAILURE);
uint64_t actual;
ASSERT_SUCCESS( parser.parse(pstr)["key"].get(actual) );
std::cout << std::boolalpha << "test: " << (expected == actual) << std::endl;
ASSERT_EQUAL( expected, actual );
}
return true;
}
static bool parse_and_check_signed(const std::string src) {
std::cout << "src: " << src << ", expecting signed" << std::endl;
const padded_string pstr{src};
simdjson::dom::parser parser;
auto [value, error] = parser.parse(pstr).get<dom::object>()["key"];
if (error) { std::cerr << error << std::endl; abort(); }
return value.is<int64_t>();
simdjson::dom::element value;
ASSERT_SUCCESS( parser.parse(pstr).get<dom::object>()["key"].get(value) );
ASSERT_EQUAL( value.is<int64_t>(), true );
return true;
}
static bool parse_and_check_unsigned(const std::string src) {
std::cout << "src: " << src << ", expecting signed" << std::endl;
const padded_string pstr{src};
simdjson::dom::parser parser;
auto [value, error] = parser.parse(pstr).get<dom::object>()["key"];
if (error) { std::cerr << error << std::endl; abort(); }
return value.is<uint64_t>();
simdjson::dom::element value;
ASSERT_SUCCESS( parser.parse(pstr).get<dom::object>()["key"].get(value) );
ASSERT_EQUAL( value.is<uint64_t>(), true );
return true;
}
int main() {
@@ -75,21 +75,21 @@ int main() {
constexpr auto int64_min = numeric_limits<int64_t>::lowest();
constexpr auto uint64_max = numeric_limits<uint64_t>::max();
constexpr auto uint64_min = numeric_limits<uint64_t>::lowest();
parse_and_validate(make_json(int64_max), int64_max);
parse_and_validate(make_json(int64_min), int64_min);
parse_and_validate(make_json(uint64_max), uint64_max);
parse_and_validate(make_json(uint64_min), uint64_min);
constexpr auto int64_max_plus1 = static_cast<uint64_t>(int64_max) + 1;
parse_and_validate(make_json(int64_max_plus1), int64_max_plus1);
if(!parse_and_check_signed(make_json(int64_max))) {
std::cerr << "bug: large signed integers should be represented as signed integers" << std::endl;
return EXIT_FAILURE;
if (true
&& parse_and_validate(make_json(int64_max), int64_max)
&& parse_and_validate(make_json(uint64_max), uint64_max)
&& parse_and_validate(make_json(uint64_min), uint64_min)
&& parse_and_validate(make_json(int64_min), int64_min)
&& parse_and_validate(make_json(uint64_max), uint64_max)
&& parse_and_validate(make_json(uint64_min), uint64_min)
&& parse_and_validate(make_json(int64_max_plus1), int64_max_plus1)
&& parse_and_check_signed(make_json(int64_max))
&& parse_and_check_unsigned(make_json(uint64_max))
) {
std::cout << "All ok." << std::endl;
return EXIT_SUCCESS;
}
if(!parse_and_check_unsigned(make_json(uint64_max))) {
std::cerr << "bug: a large unsigned integers is not represented as an unsigned integer" << std::endl;
return EXIT_FAILURE;
}
std::cout << "All ok." << std::endl;
return EXIT_SUCCESS;
return EXIT_FAILURE;
}
tests/jsoncheck.cpp
+2 -1
View File
@@ -60,7 +60,8 @@ bool validate(const char *dirname) {
char *fullpath = static_cast<char *>(malloc(fullpathlen));
snprintf(fullpath, fullpathlen, "%s%s%s", dirname, needsep ? "/" : "", name);
auto [p, error] = simdjson::padded_string::load(fullpath);
simdjson::padded_string p;
auto error = simdjson::padded_string::load(fullpath).get(p);
if (error) {
std::cerr << "Could not load the file " << fullpath << std::endl;
return EXIT_FAILURE;
tests/numberparsingcheck.cpp
+2 -1
View File
@@ -172,7 +172,8 @@ bool validate(const char *dirname) {
} else {
strcpy(fullpath + dirlen, name);
}
auto [p, error] = simdjson::padded_string::load(fullpath);
simdjson::padded_string p;
auto error = simdjson::padded_string::load(fullpath).get(p);
if (error) {
std::cerr << "Could not load the file " << fullpath << std::endl;
return EXIT_FAILURE;
tests/parse_many_test.cpp
+6 -3
View File
@@ -69,13 +69,16 @@ bool validate(const char *dirname) {
snprintf(fullpath, fullpathlen, "%s%s%s", dirname, needsep ? "/" : "", name);
/* The actual test*/
auto [json, error] = simdjson::padded_string::load(fullpath);
simdjson::padded_string json;
auto error = simdjson::padded_string::load(fullpath).get(json);
if (!error) {
simdjson::dom::parser parser;
++how_many;
for (auto result : parser.parse_many(json)) {
error = result.error();
simdjson::dom::document_stream docs;
error = parser.parse_many(json).get(docs);
for (auto doc : docs) {
error = doc.error();
}
}
printf("%s\n", error ? "ok" : "invalid");
tests/pointercheck.cpp
+25 -27
View File
@@ -1,6 +1,7 @@
#include <iostream>
#include "simdjson.h"
#include "test_macros.h"
// we define our own asserts to get around NDEBUG
#ifndef ASSERT
@@ -35,49 +36,46 @@ const padded_string TEST_JSON = R"(
bool json_pointer_success_test(const char *json_pointer, std::string_view expected_value) {
std::cout << "Running successful JSON pointer test '" << json_pointer << "' ..." << std::endl;
dom::parser parser;
auto [value, error] = parser.parse(TEST_JSON).at(json_pointer).get<std::string_view>();
if (error) { std::cerr << "Unexpected Error: " << error << std::endl; return false; }
ASSERT(value == expected_value);
std::string_view value;
ASSERT_SUCCESS( parser.parse(TEST_JSON).at(json_pointer).get(value) );
ASSERT_EQUAL(value, expected_value);
return true;
}
bool json_pointer_success_test(const char *json_pointer) {
std::cout << "Running successful JSON pointer test '" << json_pointer << "' ..." << std::endl;
dom::parser parser;
auto error = parser.parse(TEST_JSON).at(json_pointer).error();
if (error) { std::cerr << "Unexpected Error: " << error << std::endl; return false; }
ASSERT_SUCCESS( parser.parse(TEST_JSON).at(json_pointer).error() );
return true;
}
bool json_pointer_failure_test(const char *json_pointer, error_code expected_failure_test) {
bool json_pointer_failure_test(const char *json_pointer, error_code expected_error) {
std::cout << "Running invalid JSON pointer test '" << json_pointer << "' ..." << std::endl;
dom::parser parser;
auto error = parser.parse(TEST_JSON).at(json_pointer).error();
ASSERT(error == expected_failure_test);
ASSERT_ERROR(parser.parse(TEST_JSON).at(json_pointer).error(), expected_error);
return true;
}
int main() {
if (
json_pointer_success_test("") &&
json_pointer_success_test("~1~001abc") &&
json_pointer_success_test("~1~001abc/1") &&
json_pointer_success_test("~1~001abc/1/\\\" 0") &&
json_pointer_success_test("~1~001abc/1/\\\" 0/0", "value0") &&
json_pointer_success_test("~1~001abc/1/\\\" 0/1", "value1") &&
json_pointer_failure_test("~1~001abc/1/\\\" 0/2", INDEX_OUT_OF_BOUNDS) && // index actually out of bounds
json_pointer_success_test("arr") && // get array
json_pointer_failure_test("arr/0", INDEX_OUT_OF_BOUNDS) && // array index 0 out of bounds on empty array
json_pointer_success_test("~1~001abc") && // get object
json_pointer_success_test("0", "0 ok") && // object index with integer-ish key
json_pointer_success_test("01", "01 ok") && // object index with key that would be an invalid integer
json_pointer_success_test("", "empty ok") && // object index with empty key
json_pointer_failure_test("~01abc", NO_SUCH_FIELD) && // Test that we don't try to compare the literal key
json_pointer_failure_test("~1~001abc/01", INVALID_JSON_POINTER) && // Leading 0 in integer index
json_pointer_failure_test("~1~001abc/", INVALID_JSON_POINTER) && // Empty index to array
json_pointer_failure_test("~1~001abc/-", INDEX_OUT_OF_BOUNDS) && // End index is always out of bounds
true
if (true
&& json_pointer_success_test("")
&& json_pointer_success_test("~1~001abc")
&& json_pointer_success_test("~1~001abc/1")
&& json_pointer_success_test("~1~001abc/1/\\\" 0")
&& json_pointer_success_test("~1~001abc/1/\\\" 0/0", "value0")
&& json_pointer_success_test("~1~001abc/1/\\\" 0/1", "value1")
&& json_pointer_failure_test("~1~001abc/1/\\\" 0/2", INDEX_OUT_OF_BOUNDS) // index actually out of bounds
&& json_pointer_success_test("arr") // get array
&& json_pointer_failure_test("arr/0", INDEX_OUT_OF_BOUNDS) // array index 0 out of bounds on empty array
&& json_pointer_success_test("~1~001abc") // get object
&& json_pointer_success_test("0", "0 ok") // object index with integer-ish key
&& json_pointer_success_test("01", "01 ok") // object index with key that would be an invalid integer
&& json_pointer_success_test("", "empty ok") // object index with empty key
&& json_pointer_failure_test("~01abc", NO_SUCH_FIELD) // Test that we don't try to compare the literal key
&& json_pointer_failure_test("~1~001abc/01", INVALID_JSON_POINTER) // Leading 0 in integer index
&& json_pointer_failure_test("~1~001abc/", INVALID_JSON_POINTER) // Empty index to array
&& json_pointer_failure_test("~1~001abc/-", INDEX_OUT_OF_BOUNDS) // End index is always out of bounds
) {
std::cout << "Success!" << std::endl;
return 0;
tests/readme_examples.cpp
+18 -16
View File
@@ -88,7 +88,7 @@ void basics_dom_4() {
auto abstract_json = R"(
{ "str" : { "123" : {"abc" : 3.14 } } } )"_padded;
dom::parser parser;
double v = parser.parse(abstract_json)["str"]["123"]["abc"].get<double>();
double v = parser.parse(abstract_json)["str"]["123"]["abc"];
cout << "number: " << v << endl;
}
@@ -141,9 +141,10 @@ namespace treewalk_1 {
#ifdef SIMDJSON_CPLUSPLUS17
void basics_cpp17_1() {
dom::parser parser;
padded_string json = R"( { "foo": 1, "bar": 2 } )"_padded;
auto [object, error] = parser.parse(json).get<dom::object>();
dom::parser parser;
dom::object object;
auto error = parser.parse(json).get(object);
if (error) { cerr << error << endl; return; }
for (auto [key, value] : object) {
cout << key << " = " << value << endl;
@@ -153,11 +154,10 @@ void basics_cpp17_1() {
void basics_cpp17_2() {
// C++ 11 version for comparison
dom::parser parser;
padded_string json = R"( { "foo": 1, "bar": 2 } )"_padded;
simdjson::error_code error;
dom::parser parser;
dom::object object;
error = parser.parse(json).get(object);
auto error = parser.parse(json).get(object);
if (!error) { cerr << error << endl; return; }
for (dom::key_value_pair field : object) {
cout << field.key << " = " << field.value << endl;
@@ -216,26 +216,28 @@ SIMDJSON_PUSH_DISABLE_ALL_WARNINGS
// The web_request part of this is aspirational, so we compile as much as we can here
void performance_2() {
dom::parser parser(1000*1000); // Never grow past documents > 1MB
// for (web_request request : listen()) {
auto [doc, error] = parser.parse("1"_padded/*request.body*/);
// // If the document was above our limit, emit 413 = payload too large
/* for (web_request request : listen()) */ {
dom::element doc;
auto error = parser.parse("1"_padded/*request.body*/).get(doc);
// If the document was above our limit, emit 413 = payload too large
if (error == CAPACITY) { /* request.respond(413); continue; */ }
// // ...
// }
// ...
}
}
// The web_request part of this is aspirational, so we compile as much as we can here
void performance_3() {
dom::parser parser(0); // This parser will refuse to automatically grow capacity
simdjson::error_code allocate_error = parser.allocate(1000*1000); // This allocates enough capacity to handle documents <= 1MB
if (allocate_error) { cerr << allocate_error << endl; exit(1); }
auto error = parser.allocate(1000*1000); // This allocates enough capacity to handle documents <= 1MB
if (error) { cerr << error << endl; exit(1); }
// for (web_request request : listen()) {
auto [doc, error] = parser.parse("1"_padded/*request.body*/);
/* for (web_request request : listen()) */ {
dom::element doc;
auto error = parser.parse("1"_padded/*request.body*/).get(doc);
// If the document was above our limit, emit 413 = payload too large
if (error == CAPACITY) { /* request.respond(413); continue; */ }
// ...
// }
}
}
SIMDJSON_POP_DISABLE_WARNINGS
#endif
tests/readme_examples_noexceptions.cpp
+5 -14
View File
@@ -10,7 +10,8 @@ void basics_error_1() {
dom::parser parser;
auto json = "1"_padded;
auto [doc, error] = parser.parse(json); // doc is a dom::element
dom::element doc;
auto error = parser.parse(json).get(doc);
if (error) { cerr << error << endl; exit(1); }
// Use document here now that we've checked for the error
}
@@ -18,14 +19,6 @@ SIMDJSON_POP_DISABLE_WARNINGS
#endif
void basics_error_2() {
dom::parser parser;
auto json = "1"_padded;
dom::element doc;
UNUSED auto error = parser.parse(json).get(doc); // <-- Assigns to doc and error just like "auto [doc, error]"}
}
void basics_error_3() {
auto cars_json = R"( [
{ "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
{ "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
@@ -70,8 +63,7 @@ void basics_error_3() {
}
}
void basics_error_4() {
void basics_error_3() {
auto abstract_json = R"( [
{ "12345" : {"a":12.34, "b":56.78, "c": 9998877} },
{ "12545" : {"a":11.44, "b":12.78, "c": 11111111} }
@@ -102,7 +94,7 @@ void basics_error_4() {
}
}
void basics_error_5() {
void basics_error_4() {
auto abstract_json = R"(
{ "str" : { "123" : {"abc" : 3.14 } } } )"_padded;
dom::parser parser;
@@ -116,7 +108,7 @@ void basics_error_5() {
#ifdef SIMDJSON_CPLUSPLUS17
void basics_error_3_cpp17() {
void basics_error_2_cpp17() {
auto cars_json = R"( [
{ "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
{ "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
@@ -201,6 +193,5 @@ int main() {
basics_error_2();
basics_error_3();
basics_error_4();
basics_error_5();
return EXIT_SUCCESS;
}
tests/singleheadertest.cpp
+4 -3
View File
@@ -6,12 +6,13 @@ using namespace simdjson;
int main() {
const char *filename = SIMDJSON_BENCHMARK_DATA_DIR "/twitter.json";
padded_string p = get_corpus(filename);
dom::parser parser;
auto [doc, error] = parser.parse(p);
if(error) {
dom::element doc;
auto error = parser.load(filename).get(doc);
if (error) {
std::cerr << error << std::endl;
return EXIT_FAILURE;
}
std::cout << doc << std::endl;
return EXIT_SUCCESS;
}
tests/stringparsingcheck.cpp
+2 -1
View File
@@ -337,7 +337,8 @@ bool validate(const char *dirname) {
} else {
strcpy(fullpath + dirlen, name);
}
auto [p, error] = simdjson::padded_string::load(fullpath);
simdjson::padded_string p;
auto error = simdjson::padded_string::load(fullpath).get(p);
if (error) {
std::cerr << "Could not load the file " << fullpath << std::endl;
return EXIT_FAILURE;
tests/test_macros.h
+27 -6
View File
@@ -18,17 +18,38 @@ const char *SMALLDEMO_JSON = SIMDJSON_BENCHMARK_SMALLDATA_DIR "smalldemo.json";
const char *TRUENULL_JSON = SIMDJSON_BENCHMARK_SMALLDATA_DIR "truenull.json";
// For the ASSERT_EQUAL macro
template<typename T>
bool equals_expected(T actual, T expected) {
return actual == expected;
template<typename T, typename S>
really_inline bool equals_expected(T actual, S expected) {
return actual == T(expected);
}
template<>
bool equals_expected<const char *>(const char *actual, const char *expected) {
really_inline bool equals_expected<const char *, const char *>(const char *actual, const char *expected) {
return !strcmp(actual, expected);
}
#define ASSERT_EQUAL(ACTUAL, EXPECTED) if (!equals_expected(ACTUAL, EXPECTED)) { std::cerr << "Expected " << #ACTUAL << " to be " << (EXPECTED) << ", got " << (ACTUAL) << " instead!" << std::endl; return false; }
really_inline simdjson::error_code to_error_code(simdjson::error_code error) {
return error;
}
template<typename T>
really_inline simdjson::error_code to_error_code(const simdjson::simdjson_result<T> &result) {
return result.error();
}
#define TEST_START() { cout << "Running " << __func__ << " ..." << endl; }
#define ASSERT_EQUAL(ACTUAL, EXPECTED) \
do { \
auto _actual = (ACTUAL); \
auto _expected = (EXPECTED); \
if (!equals_expected(_actual, _expected)) { \
std::cerr << "Expected " << (#ACTUAL) << " to be " << _expected << ", got " << _actual << " instead!" << std::endl; \
return false; \
} \
} while(0);
#define ASSERT_ERROR(ACTUAL, EXPECTED) do { auto _actual = to_error_code(ACTUAL); auto _expected = to_error_code(EXPECTED); if (_actual != _expected) { std::cerr << "FAIL: Unexpected error \"" << _actual << "\" (expected \"" << _expected << "\")" << std::endl; return false; } } while (0);
#define ASSERT(RESULT, MESSAGE) if (!(RESULT)) { std::cerr << MESSAGE << std::endl; return false; }
#define RUN_TEST(RESULT) if (!RESULT) { return false; }
#define ASSERT_SUCCESS(ERROR) if (ERROR) { std::cerr << (ERROR) << std::endl; return false; }
#define ASSERT_SUCCESS(ERROR) do { auto _error = to_error_code(ERROR); if (_error) { std::cerr << _error << std::endl; return false; } } while(0);
#define TEST_FAIL(MESSAGE) { std::cerr << "FAIL: " << (MESSAGE) << std::endl; return false; }
#define TEST_SUCCEED() { return true; }
#endif // TEST_MACROS_H
tools/json2json.cpp
+4 -4
View File
@@ -49,10 +49,10 @@ int main(int argc, char *argv[]) {
const char *filename = result["file"].as<std::string>().c_str();
simdjson::dom::parser parser;
auto [doc, error] = parser.load(filename); // do the parsing, return false on error
if (error != simdjson::SUCCESS) {
std::cerr << " Parsing failed. Error is '" << simdjson::error_message(error)
<< "'." << std::endl;
simdjson::dom::element doc;
auto error = parser.load(filename).get(doc); // do the parsing, return false on error
if (error) {
std::cerr << " Parsing failed. Error is '" << error << "'." << std::endl;
return EXIT_FAILURE;
}
if(rawdump) {
tools/jsonpointer.cpp
+5 -4
View File
@@ -20,16 +20,17 @@ int main(int argc, char *argv[]) {
const char *filename = argv[1];
simdjson::dom::parser parser;
auto [doc, error] = parser.load(filename);
simdjson::dom::element doc;
auto error = parser.load(filename).get(doc);
if (error) { std::cerr << "Error parsing " << filename << ": " << error << std::endl; }
std::cout << "[" << std::endl;
for (int idx = 2; idx < argc; idx++) {
const char *json_pointer = argv[idx];
auto [value, pointer_error] = doc[json_pointer];
simdjson::dom::element value;
std::cout << "{\"jsonpath\": \"" << json_pointer << "\"";
if (pointer_error) {
std::cout << ",\"error\":\"" << pointer_error << "\"";
if ((error = doc[json_pointer].get(value))) {
std::cout << ",\"error\":\"" << error << "\"";
} else {
std::cout << ",\"value\":" << value;
}
tools/jsonstats.cpp
+4 -2
View File
@@ -166,7 +166,8 @@ void recurse(simdjson::dom::element element, stat_t &s, size_t depth) {
stat_t simdjson_compute_stats(const simdjson::padded_string &p) {
stat_t s{};
simdjson::dom::parser parser;
auto [doc, error] = parser.parse(p);
simdjson::dom::element doc;
auto error = parser.parse(p).get(doc);
if (error) {
s.valid = false;
std::cerr << error << std::endl;
@@ -217,7 +218,8 @@ int main(int argc, char *argv[]) {
const char *filename = result["file"].as<std::string>().c_str();
auto [p, error] = simdjson::padded_string::load(filename);
simdjson::padded_string p;
auto error = simdjson::padded_string::load(filename).get(p);
if (error) {
std::cerr << "Could not load the file " << filename << std::endl;
return EXIT_FAILURE;
tools/minify.cpp
+2 -1
View File
@@ -56,7 +56,8 @@ int main(int argc, char *argv[]) {
std::string filename = result["file"].as<std::string>();
auto [p, error] = simdjson::padded_string::load(filename);
simdjson::padded_string p;
auto error = simdjson::padded_string::load(filename).get(p);
if (error) {
std::cerr << "Could not load the file " << filename << std::endl;
return EXIT_FAILURE;