admin/simdjson-simdjson
1
0
Fork 0
mirror of https://github.com/simdjson/simdjson synced 2026-06-08 17:27:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #960 from simdjson/jkeiser/idiomatic-get

Browse Source 
Convert simdjson to use .get()
This commit is contained in:
John Keiser
2020-06-23 09:49:41 -07:00
committed by GitHub
parent eef1171944 e369d45b9c
commit c650ea9765
29 changed files with 506 additions and 561 deletions
Download Patch File Download Diff File Expand all files Collapse all files
benchmark/bench_dom_api.cpp
+6 -5
View File
@@ -407,7 +407,7 @@ static void iterator_twitter_default_profile(State& state) {
set<string_view> default_users;
ParsedJson::Iterator iter(pj);
// for (dom::object tweet : doc["statuses"].get<dom::array>()) {
// for (dom::object tweet : doc["statuses"]) {
if (!(iter.move_to_key("statuses") && iter.is_array())) { return; }
if (iter.down()) { // first status
do {
@@ -480,23 +480,24 @@ static void iterator_twitter_image_sizes(State& state) {
set<tuple<uint64_t, uint64_t>> image_sizes;
ParsedJson::Iterator iter(pj);
// for (dom::object tweet : doc["statuses"].get<dom::array>()) {
// for (dom::object tweet : doc["statuses"]) {
if (!(iter.move_to_key("statuses") && iter.is_array())) { return; }
if (iter.down()) { // first status
do {
// auto [media, not_found] = tweet["entities"]["media"];
// dom::object media;
// not_found = tweet["entities"]["media"].get(media);
// if (!not_found) {
if (iter.move_to_key("entities")) {
if (!iter.is_object()) { return; }
if (iter.move_to_key("media")) {
if (!iter.is_array()) { return; }
// for (dom::object image : media.get<dom::array>()) {
// for (dom::object image : media) {
if (iter.down()) { // first media
do {
// for (auto [key, size] : image["sizes"].get<dom::object>()) {
// for (auto [key, size] : dom::object(image["sizes"])) {
if (!(iter.move_to_key("sizes") && iter.is_object())) { return; }
if (iter.down()) { // first size
do {
benchmark/distinctuseridcompetition.cpp
+8 -6
View File
@@ -40,17 +40,18 @@ void print_vec(const std::vector<int64_t> &v) {
// simdjson_recurse below come be implemented like so but it is slow:
/*void simdjson_recurse(std::vector<int64_t> & v, simdjson::dom::element element) {
if (element.is<simdjson::dom::array>()) {
auto [array, array_error] = element.get<simdjson::dom::array>();
error_code error;
if (element.is_array()) {
dom::array array;
error = element.get(array);
for (auto child : array) {
if (child.is<simdjson::dom::array>() || child.is<simdjson::dom::object>()) {
simdjson_recurse(v, child);
}
}
} else if (element.is<simdjson::dom::object>()) {
auto [object, error] = element.get<simdjson::dom::object>();
} else if (element.is_object()) {
int64_t id;
error = object["user"]["id"].get(id);
error = element["user"]["id"].get(id);
if(!error) {
v.push_back(id);
}
@@ -330,7 +331,8 @@ int main(int argc, char *argv[]) {
std::cerr << "warning: ignoring everything after " << argv[optind + 1]
<< std::endl;
}
auto [p, error] = simdjson::padded_string::load(filename);
simdjson::padded_string p;
auto error = simdjson::padded_string::load(filename).get(p);
if (error) {
std::cerr << "Could not load the file " << filename << std::endl;
return EXIT_FAILURE;
benchmark/minifiercompetition.cpp
+2 -1
View File
@@ -75,7 +75,8 @@ int main(int argc, char *argv[]) {
exit(1);
}
const char *filename = argv[optind];
auto [p, error] = simdjson::padded_string::load(filename);
simdjson::padded_string p;
auto error = simdjson::padded_string::load(filename).get(p);
if (error) {
std::cerr << "Could not load the file " << filename << std::endl;
return EXIT_FAILURE;
benchmark/parseandstatcompetition.cpp
+10 -7
View File
@@ -105,7 +105,8 @@ void simdjson_recurse(stat_t &s, simdjson::dom::element element) {
never_inline stat_t simdjson_compute_stats(const simdjson::padded_string &p) {
stat_t s{};
simdjson::dom::parser parser;
auto [doc, error] = parser.parse(p);
simdjson::dom::element doc;
auto error = parser.parse(p).get(doc);
if (error) {
s.valid = false;
return s;
@@ -154,11 +155,11 @@ static void GenStatPlus(Stat &stat, const dom::element &v) {
break;
case dom::element_type::STRING: {
stat.stringCount++;
std::string_view sv = v.get<std::string_view>();
auto sv = std::string_view(v);
stat.stringLength += sv.size();
} break;
case dom::element_type::BOOL:
if (v.get<bool>()) {
if (bool(v)) {
stat.trueCount++;
} else {
stat.falseCount++;
@@ -409,7 +410,8 @@ int main(int argc, char *argv[]) {
std::cerr << "warning: ignoring everything after " << argv[optind + 1]
<< std::endl;
}
auto [p, error] = simdjson::padded_string::load(filename);
simdjson::padded_string p;
auto error = simdjson::padded_string::load(filename).get(p);
if (error) {
std::cerr << "Could not load the file " << filename << std::endl;
return EXIT_FAILURE;
@@ -464,9 +466,10 @@ int main(int argc, char *argv[]) {
printf("API traversal tests\n");
printf("Based on https://github.com/miloyip/nativejson-benchmark\n");
simdjson::dom::parser parser;
auto [doc, err] = parser.parse(p);
if (err) {
std::cerr << err << std::endl;
simdjson::dom::element doc;
auto error = parser.parse(p).get(doc);
if (error) {
std::cerr << error << std::endl;
}
size_t refval = simdjson_compute_stats_refplus(doc).objectCount;
benchmark/parsingcompetition.cpp
+4 -3
View File
@@ -82,9 +82,10 @@ inline void reset_stream(std::stringstream & is) {
bool bench(const char *filename, bool verbose, bool just_data, double repeat_multiplier) {
auto [p, err] = simdjson::padded_string::load(filename);
if (err) {
std::cerr << "Could not load the file " << filename << std::endl;
simdjson::padded_string p;
auto error = simdjson::padded_string::load(filename).get(p);
if (error) {
std::cerr << "Could not load the file " << filename << ": " << error << std::endl;
return false;
}
benchmark/statisticalmodel.cpp
+4 -2
View File
@@ -96,7 +96,8 @@ void simdjson_recurse(stat_t &s, simdjson::dom::element element) {
stat_t simdjson_compute_stats(const simdjson::padded_string &p) {
stat_t answer{};
simdjson::dom::parser parser;
auto [doc, error] = parser.parse(p);
simdjson::dom::element doc;
auto error = parser.parse(p).get(doc);
if (error) {
answer.valid = false;
return answer;
@@ -136,7 +137,8 @@ int main(int argc, char *argv[]) {
std::cerr << "warning: ignoring everything after " << argv[optind + 1]
<< std::endl;
}
auto [p, error] = simdjson::padded_string::load(filename);
simdjson::padded_string p;
auto error = simdjson::padded_string::load(filename).get(p);
if (error) {
std::cerr << "Could not load the file " << filename << std::endl;
return EXIT_FAILURE;
doc/basics.md
+13 -26
View File
@@ -164,7 +164,7 @@ And another one:
auto abstract_json = R"(
{ "str" : { "123" : {"abc" : 3.14 } } } )"_padded;
dom::parser parser;
double v = parser.parse(abstract_json)["str"]["123"]["abc"].get<double>();
double v = parser.parse(abstract_json)["str"]["123"]["abc"];
cout << "number: " << v << endl;
```
@@ -208,14 +208,13 @@ Your input string does not need any padding. Any string will do. The `validate_u
C++17 Support
-------------
While the simdjson library can be used in any project using C++ 11 and above, it has special support
for C++ 17. The APIs for field iteration and error handling in particular are designed to work
nicely with C++17's destructuring syntax. For example:
While the simdjson library can be used in any project using C++ 11 and above, field iteration has special support C++ 17's destructuring syntax. For example:
```c++
dom::parser parser;
padded_string json = R"( { "foo": 1, "bar": 2 } )"_padded;
auto [object, error] = parser.parse(json).get<dom::object>();
dom::parser parser;
dom::object object;
auto error = parser.parse(json).get(object);
if (error) { cerr << error << endl; return; }
for (auto [key, value] : object) {
cout << key << " = " << value << endl;
@@ -226,11 +225,10 @@ For comparison, here is the C++ 11 version of the same code:
```c++
// C++ 11 version for comparison
dom::parser parser;
padded_string json = R"( { "foo": 1, "bar": 2 } )"_padded;
simdjson::error_code error;
dom::parser parser;
dom::object object;
error = parser.parse(json).get(object);
auto error = parser.parse(json).get(object);
if (!error) { cerr << error << endl; return; }
for (dom::key_value_pair field : object) {
cout << field.key << " = " << field.value << endl;
@@ -258,29 +256,18 @@ Error Handling
--------------
All simdjson APIs that can fail return `simdjson_result<T>`, which is a &lt;value, error_code&gt;
pair. The error codes and values can be accessed directly, reading the error like so:
pair. You can retrieve the value with .get(), like so:
```c++
auto [doc, error] = parser.parse(json); // doc is a dom::element
dom::element doc;
auto error = parser.parse(json).get(doc);
if (error) { cerr << error << endl; exit(1); }
// Use document here now that we've checked for the error
```
When you use the code this way, it is your responsibility to check for error before using the
result: if there is an error, the result value will not be valid and using it will caused undefined
behavior.
> Note: because of the way `auto [x, y]` works in C++, you have to define new variables each time you
> use it. If your project treats aliased, this means you can't use the same names in `auto [x, error]`
> without triggering warnings or error (and particularly can't use the word "error" every time). To
> circumvent this, you can use this instead:
>
> ```c++
> dom::element doc;
> auto error = parser.parse(json).get(doc); // <-- Assigns to doc and error just like "auto [doc, error]"
> ```
We can write a "quick start" example where we attempt to parse a file and access some data, without triggering exceptions:
```C++
@@ -288,11 +275,12 @@ We can write a "quick start" example where we attempt to parse a file and access
int main(void) {
simdjson::dom::parser parser;
simdjson::dom::element tweets;
auto error = parser.load("twitter.json").get(tweets);
if (error) { std::cerr << error << std::endl; return EXIT_FAILURE; }
simdjson::dom::element res;
simdjson::dom::element res;
if ((error = tweets["search_metadata"]["count"].get(res))) {
std::cerr << "could not access keys" << std::endl;
return EXIT_FAILURE;
@@ -395,8 +383,7 @@ And another one:
cout << "number: " << v << endl;
```
Notice how we can string several operation (`parser.parse(abstract_json)["str"]["123"]["abc"].get<double>()`) and only check for the error once, a strategy we call *error chaining*.
Notice how we can string several operations (`parser.parse(abstract_json)["str"]["123"]["abc"].get(v)`) and only check for the error once, a strategy we call *error chaining*.
The next two functions will take as input a JSON document containing an array with a single element, either a string or a number. They return true upon success.
doc/performance.md
+6 -4
View File
@@ -68,7 +68,8 @@ without bound:
```c++
dom::parser parser(1000*1000); // Never grow past documents > 1MB
for (web_request request : listen()) {
auto [doc, error] = parser.parse(request.body);
dom::element doc;
auto error = parser.parse(request.body).get(doc);
// If the document was above our limit, emit 413 = payload too large
if (error == CAPACITY) { request.respond(413); continue; }
// ...
@@ -82,11 +83,12 @@ without bound:
```c++
dom::parser parser(0); // This parser will refuse to automatically grow capacity
simdjson::error_code allocate_error = parser.allocate(1000*1000); // This allocates enough capacity to handle documents <= 1MB
if (allocate_error) { cerr << allocate_error << endl; exit(1); }
auto error = parser.allocate(1000*1000); // This allocates enough capacity to handle documents <= 1MB
if (error) { cerr << error << endl; exit(1); }
for (web_request request : listen()) {
auto [doc, error] = parser.parse(request.body);
dom::element doc;
error = parser.parse(request.body).get(doc);
// If the document was above our limit, emit 413 = payload too large
if (error == CAPACITY) { request.respond(413); continue; }
// ...
include/simdjson/dom/array.h
+1 -1
View File
@@ -68,7 +68,7 @@ public:
* Get the value associated with the given JSON pointer.
*
* dom::parser parser;
* array a = parser.parse(R"([ { "foo": { "a": [ 10, 20, 30 ] }} ])");
* array a = parser.parse(R"([ { "foo": { "a": [ 10, 20, 30 ] }} ])"_padded);
* a.at("0/foo/a/1") == 20
* a.at("0")["foo"]["a"].at(1) == 20
*
include/simdjson/dom/element.h
+7 -7
View File
@@ -336,8 +336,8 @@ public:
* The key will be matched against **unescaped** JSON:
*
* dom::parser parser;
* parser.parse(R"({ "a\n": 1 })")["a\n"].get<uint64_t>().value == 1
* parser.parse(R"({ "a\n": 1 })")["a\\n"].get<uint64_t>().error == NO_SUCH_FIELD
* parser.parse(R"({ "a\n": 1 })"_padded)["a\n"].get<uint64_t>().first == 1
* parser.parse(R"({ "a\n": 1 })"_padded)["a\\n"].get<uint64_t>().error() == NO_SUCH_FIELD
*
* @return The value associated with this field, or:
* - NO_SUCH_FIELD if the field does not exist in the object
@@ -351,8 +351,8 @@ public:
* The key will be matched against **unescaped** JSON:
*
* dom::parser parser;
* parser.parse(R"({ "a\n": 1 })")["a\n"].get<uint64_t>().value == 1
* parser.parse(R"({ "a\n": 1 })")["a\\n"].get<uint64_t>().error == NO_SUCH_FIELD
* parser.parse(R"({ "a\n": 1 })"_padded)["a\n"].get<uint64_t>().first == 1
* parser.parse(R"({ "a\n": 1 })"_padded)["a\\n"].get<uint64_t>().error() == NO_SUCH_FIELD
*
* @return The value associated with this field, or:
* - NO_SUCH_FIELD if the field does not exist in the object
@@ -364,7 +364,7 @@ public:
* Get the value associated with the given JSON pointer.
*
* dom::parser parser;
* element doc = parser.parse(R"({ "foo": { "a": [ 10, 20, 30 ] }})");
* element doc = parser.parse(R"({ "foo": { "a": [ 10, 20, 30 ] }})"_padded);
* doc.at("/foo/a/1") == 20
* doc.at("/")["foo"]["a"].at(1) == 20
* doc.at("")["foo"]["a"].at(1) == 20
@@ -391,8 +391,8 @@ public:
* The key will be matched against **unescaped** JSON:
*
* dom::parser parser;
* parser.parse(R"({ "a\n": 1 })")["a\n"].get<uint64_t>().value == 1
* parser.parse(R"({ "a\n": 1 })")["a\\n"].get<uint64_t>().error == NO_SUCH_FIELD
* parser.parse(R"({ "a\n": 1 })"_padded)["a\n"].get<uint64_t>().first == 1
* parser.parse(R"({ "a\n": 1 })"_padded)["a\\n"].get<uint64_t>().error() == NO_SUCH_FIELD
*
* @return The value associated with this field, or:
* - NO_SUCH_FIELD if the field does not exist in the object
include/simdjson/dom/object.h
+7 -7
View File
@@ -101,8 +101,8 @@ public:
* The key will be matched against **unescaped** JSON:
*
* dom::parser parser;
* parser.parse(R"({ "a\n": 1 })")["a\n"].get<uint64_t>().value == 1
* parser.parse(R"({ "a\n": 1 })")["a\\n"].get<uint64_t>().error == NO_SUCH_FIELD
* parser.parse(R"({ "a\n": 1 })"_padded)["a\n"].get<uint64_t>().first == 1
* parser.parse(R"({ "a\n": 1 })"_padded)["a\\n"].get<uint64_t>().error() == NO_SUCH_FIELD
*
* This function has linear-time complexity: the keys are checked one by one.
*
@@ -118,8 +118,8 @@ public:
* The key will be matched against **unescaped** JSON:
*
* dom::parser parser;
* parser.parse(R"({ "a\n": 1 })")["a\n"].get<uint64_t>().value == 1
* parser.parse(R"({ "a\n": 1 })")["a\\n"].get<uint64_t>().error == NO_SUCH_FIELD
* parser.parse(R"({ "a\n": 1 })"_padded)["a\n"].get<uint64_t>().first == 1
* parser.parse(R"({ "a\n": 1 })"_padded)["a\\n"].get<uint64_t>().error() == NO_SUCH_FIELD
*
* This function has linear-time complexity: the keys are checked one by one.
*
@@ -133,7 +133,7 @@ public:
* Get the value associated with the given JSON pointer.
*
* dom::parser parser;
* object obj = parser.parse(R"({ "foo": { "a": [ 10, 20, 30 ] }})");
* object obj = parser.parse(R"({ "foo": { "a": [ 10, 20, 30 ] }})"_padded);
* obj.at("foo/a/1") == 20
* obj.at("foo")["a"].at(1) == 20
*
@@ -151,8 +151,8 @@ public:
* The key will be matched against **unescaped** JSON:
*
* dom::parser parser;
* parser.parse(R"({ "a\n": 1 })")["a\n"].get<uint64_t>().value == 1
* parser.parse(R"({ "a\n": 1 })")["a\\n"].get<uint64_t>().error == NO_SUCH_FIELD
* parser.parse(R"({ "a\n": 1 })"_padded)["a\n"].get<uint64_t>().first == 1
* parser.parse(R"({ "a\n": 1 })"_padded)["a\\n"].get<uint64_t>().error() == NO_SUCH_FIELD
*
* This function has linear-time complexity: the keys are checked one by one.
*
include/simdjson/dom/parser.h
+14 -6
View File
@@ -173,9 +173,13 @@ public:
* the same interface, requiring you to check the error before using the document:
*
* dom::parser parser;
* for (auto [doc, error] : parser.load_many(path)) {
* if (error) { cerr << error << endl; exit(1); }
* cout << std::string(doc["title"]) << endl;
* dom::document_stream docs;
* auto error = parser.load_many(path).get(docs);
* if (error) { cerr << error << endl; exit(1); }
* for (auto doc : docs) {
* std::string_view title;
* if ((error = doc["title"].get(title)) { cerr << error << endl; exit(1); }
* cout << title << endl;
* }
*
* ### Threads
@@ -233,9 +237,13 @@ public:
* the same interface, requiring you to check the error before using the document:
*
* dom::parser parser;
* for (auto [doc, error] : parser.parse_many(buf, len)) {
* if (error) { cerr << error << endl; exit(1); }
* cout << std::string(doc["title"]) << endl;
* dom::document_stream docs;
* auto error = parser.load_many(path).get(docs);
* if (error) { cerr << error << endl; exit(1); }
* for (auto doc : docs) {
* std::string_view title;
* if ((error = doc["title"].get(title)) { cerr << error << endl; exit(1); }
* cout << title << endl;
* }
*
* ### REQUIRED: Buffer Padding
include/simdjson/error.h
+2 -1
View File
@@ -42,7 +42,8 @@ enum error_code {
* Get the error message for the given error code.
*
* dom::parser parser;
* auto [doc, error] = parser.parse("foo");
* dom::element doc;
* auto error = parser.parse("foo").get(doc);
* if (error) { printf("Error: %s\n", error_message(error)); }
*
* @return The error message.
tests/allparserscheckfile.cpp
+9 -8
View File
@@ -63,9 +63,10 @@ int main(int argc, char *argv[]) {
exit(1);
}
const char *filename = argv[optind];
auto [p, loaderr] = simdjson::padded_string::load(filename);
if (loaderr) {
std::cerr << "Could not load the file " << filename << ": " << loaderr << std::endl;
simdjson::padded_string p;
auto error = simdjson::padded_string::load(filename).get(p);
if (error) {
std::cerr << "Could not load the file " << filename << ": " << error << std::endl;
return EXIT_FAILURE;
}
if (verbose) {
@@ -79,7 +80,7 @@ int main(int argc, char *argv[]) {
std::cout << std::endl;
}
simdjson::dom::parser parser;
auto err = parser.parse(p).error();
error = parser.parse(p).error();
rapidjson::Document d;
@@ -95,19 +96,19 @@ int main(int argc, char *argv[]) {
.is_valid();
if (just_favorites) {
printf("our parser : %s \n",
(err == simdjson::error_code::SUCCESS) ? "correct" : "invalid");
(error == simdjson::error_code::SUCCESS) ? "correct" : "invalid");
printf("rapid (check encoding) : %s \n",
rapid_correct_checkencoding ? "correct" : "invalid");
printf("sajson : %s \n",
sajson_correct ? "correct" : "invalid");
if (err == simdjson::DEPTH_ERROR) {
if (error == simdjson::DEPTH_ERROR) {
printf("simdjson encountered a DEPTH_ERROR, it was parametrized to "
"reject documents with depth exceeding %zu.\n",
parser.max_depth());
}
if (((err == simdjson::error_code::SUCCESS) != rapid_correct_checkencoding) ||
if (((error == simdjson::error_code::SUCCESS) != rapid_correct_checkencoding) ||
(rapid_correct_checkencoding != sajson_correct) ||
((err == simdjson::SUCCESS) != sajson_correct)) {
((error == simdjson::SUCCESS) != sajson_correct)) {
printf("WARNING: THEY DISAGREE\n\n");
return EXIT_FAILURE;
}
tests/basictests.cpp
+266 -345
View File
File diff suppressed because it is too large Load Diff
tests/errortests.cpp
+18 -18
View File
@@ -46,15 +46,16 @@ namespace parser_load {
ASSERT_SUCCESS(parser.parse_many(DOC).get(docs));
for (auto doc : docs) {
count++;
auto [val, error] = doc.get<uint64_t>();
uint64_t val;
auto error = doc.get(val);
if (count == 3) {
ASSERT_ERROR(error, TAPE_ERROR);
} else {
if (error) { TEST_FAIL(error); }
if (val != count) { cerr << "FAIL: expected " << count << ", got " << val << endl; return false; }
ASSERT_SUCCESS(error);
ASSERT_EQUAL(val, count);
}
}
if (count != 3) { cerr << "FAIL: expected 2 documents and 1 error, got " << count << " total things" << endl; return false; }
ASSERT_EQUAL(count, 3);
TEST_SUCCEED();
}
@@ -69,7 +70,7 @@ namespace parser_load {
count++;
ASSERT_ERROR(doc.error(), TAPE_ERROR);
}
if (count != 1) { cerr << "FAIL: expected no documents and 1 error, got " << count << " total things" << endl; return false; }
ASSERT_EQUAL(count, 1);
TEST_SUCCEED();
}
@@ -82,36 +83,34 @@ namespace parser_load {
ASSERT_SUCCESS(parser.parse_many(DOC).get(docs));
for (auto doc : docs) {
count++;
auto [val, error] = doc.get<uint64_t>();
uint64_t val;
auto error = doc.get(val);
if (count == 3) {
ASSERT_ERROR(error, TAPE_ERROR);
} else {
if (error) { TEST_FAIL(error); }
if (val != count) { cerr << "FAIL: expected " << count << ", got " << val << endl; return false; }
ASSERT_SUCCESS(error);
ASSERT_EQUAL(val, count);
}
}
if (count != 3) { cerr << "FAIL: expected 2 documents and 1 error, got " << count << " total things" << endl; return false; }
ASSERT_EQUAL(count, 3);
TEST_SUCCEED();
}
bool parser_load_nonexistent() {
TEST_START();
dom::parser parser;
auto error = parser.load(NONEXISTENT_FILE).error();
ASSERT_ERROR(error, IO_ERROR);
ASSERT_ERROR( parser.load(NONEXISTENT_FILE).error(), IO_ERROR );
TEST_SUCCEED();
}
bool parser_load_many_nonexistent() {
TEST_START();
dom::parser parser;
dom::document_stream stream;
ASSERT_ERROR(parser.load_many(NONEXISTENT_FILE).get(stream), IO_ERROR);
ASSERT_ERROR( parser.load_many(NONEXISTENT_FILE).error(), IO_ERROR );
TEST_SUCCEED();
}
bool padded_string_load_nonexistent() {
TEST_START();
auto error = padded_string::load(NONEXISTENT_FILE).error();
ASSERT_ERROR(error, IO_ERROR);
ASSERT_ERROR(padded_string::load(NONEXISTENT_FILE).error(), IO_ERROR);
TEST_SUCCEED();
}
@@ -119,16 +118,17 @@ namespace parser_load {
TEST_START();
dom::parser parser;
UNUSED uint64_t foo;
ASSERT_ERROR( parser.load(NONEXISTENT_FILE)["foo"].get(foo) , IO_ERROR);
ASSERT_ERROR( parser.load(NONEXISTENT_FILE)["foo"].get(foo), IO_ERROR);
TEST_SUCCEED();
}
bool parser_load_many_chain() {
TEST_START();
dom::parser parser;
dom::document_stream stream;
ASSERT_ERROR( parser.load_many(NONEXISTENT_FILE).get(stream) , IO_ERROR );
UNUSED dom::document_stream stream;
ASSERT_ERROR( parser.load_many(NONEXISTENT_FILE).get(stream), IO_ERROR );
TEST_SUCCEED();
}
bool run() {
return true
&& parser_load_capacity()
tests/integer_tests.cpp
+33 -33
View File
@@ -3,6 +3,7 @@
#include <limits>
#include "simdjson.h"
#include "test_macros.h"
// we define our own asserts to get around NDEBUG
#ifndef ASSERT
@@ -29,44 +30,43 @@ template <typename T> static const std::string make_json(T value) {
}
template <typename T>
static void parse_and_validate(const std::string src, T expected) {
static bool parse_and_validate(const std::string src, T expected) {
std::cout << "src: " << src << ", ";
const padded_string pstr{src};
simdjson::dom::parser parser;
bool result;
if constexpr (std::is_same<int64_t, T>::value) {
auto [actual, error] = parser.parse(pstr).get<dom::object>()["key"].get<int64_t>();
if (error) { std::cerr << error << std::endl; abort(); }
result = (expected == actual);
int64_t actual;
ASSERT_SUCCESS( parser.parse(pstr)["key"].get(actual) );
std::cout << std::boolalpha << "test: " << (expected == actual) << std::endl;
ASSERT_EQUAL( expected, actual );
} else {
auto [actual, error] = parser.parse(pstr).get<dom::object>()["key"].get<uint64_t>();
if (error) { std::cerr << error << std::endl; abort(); }
result = (expected == actual);
}
std::cout << std::boolalpha << "test: " << result << std::endl;
if(!result) {
std::cerr << "bug detected" << std::endl;
exit(EXIT_FAILURE);
uint64_t actual;
ASSERT_SUCCESS( parser.parse(pstr)["key"].get(actual) );
std::cout << std::boolalpha << "test: " << (expected == actual) << std::endl;
ASSERT_EQUAL( expected, actual );
}
return true;
}
static bool parse_and_check_signed(const std::string src) {
std::cout << "src: " << src << ", expecting signed" << std::endl;
const padded_string pstr{src};
simdjson::dom::parser parser;
auto [value, error] = parser.parse(pstr).get<dom::object>()["key"];
if (error) { std::cerr << error << std::endl; abort(); }
return value.is<int64_t>();
simdjson::dom::element value;
ASSERT_SUCCESS( parser.parse(pstr).get<dom::object>()["key"].get(value) );
ASSERT_EQUAL( value.is<int64_t>(), true );
return true;
}
static bool parse_and_check_unsigned(const std::string src) {
std::cout << "src: " << src << ", expecting signed" << std::endl;
const padded_string pstr{src};
simdjson::dom::parser parser;
auto [value, error] = parser.parse(pstr).get<dom::object>()["key"];
if (error) { std::cerr << error << std::endl; abort(); }
return value.is<uint64_t>();
simdjson::dom::element value;
ASSERT_SUCCESS( parser.parse(pstr).get<dom::object>()["key"].get(value) );
ASSERT_EQUAL( value.is<uint64_t>(), true );
return true;
}
int main() {
@@ -75,21 +75,21 @@ int main() {
constexpr auto int64_min = numeric_limits<int64_t>::lowest();
constexpr auto uint64_max = numeric_limits<uint64_t>::max();
constexpr auto uint64_min = numeric_limits<uint64_t>::lowest();
parse_and_validate(make_json(int64_max), int64_max);
parse_and_validate(make_json(int64_min), int64_min);
parse_and_validate(make_json(uint64_max), uint64_max);
parse_and_validate(make_json(uint64_min), uint64_min);
constexpr auto int64_max_plus1 = static_cast<uint64_t>(int64_max) + 1;
parse_and_validate(make_json(int64_max_plus1), int64_max_plus1);
if(!parse_and_check_signed(make_json(int64_max))) {
std::cerr << "bug: large signed integers should be represented as signed integers" << std::endl;
return EXIT_FAILURE;
if (true
&& parse_and_validate(make_json(int64_max), int64_max)
&& parse_and_validate(make_json(uint64_max), uint64_max)
&& parse_and_validate(make_json(uint64_min), uint64_min)
&& parse_and_validate(make_json(int64_min), int64_min)
&& parse_and_validate(make_json(uint64_max), uint64_max)
&& parse_and_validate(make_json(uint64_min), uint64_min)
&& parse_and_validate(make_json(int64_max_plus1), int64_max_plus1)
&& parse_and_check_signed(make_json(int64_max))
&& parse_and_check_unsigned(make_json(uint64_max))
) {
std::cout << "All ok." << std::endl;
return EXIT_SUCCESS;
}
if(!parse_and_check_unsigned(make_json(uint64_max))) {
std::cerr << "bug: a large unsigned integers is not represented as an unsigned integer" << std::endl;
return EXIT_FAILURE;
}
std::cout << "All ok." << std::endl;
return EXIT_SUCCESS;
return EXIT_FAILURE;
}
tests/jsoncheck.cpp
+2 -1
View File
@@ -60,7 +60,8 @@ bool validate(const char *dirname) {
char *fullpath = static_cast<char *>(malloc(fullpathlen));
snprintf(fullpath, fullpathlen, "%s%s%s", dirname, needsep ? "/" : "", name);
auto [p, error] = simdjson::padded_string::load(fullpath);
simdjson::padded_string p;
auto error = simdjson::padded_string::load(fullpath).get(p);
if (error) {
std::cerr << "Could not load the file " << fullpath << std::endl;
return EXIT_FAILURE;
tests/numberparsingcheck.cpp
+2 -1
View File
@@ -172,7 +172,8 @@ bool validate(const char *dirname) {
} else {
strcpy(fullpath + dirlen, name);
}
auto [p, error] = simdjson::padded_string::load(fullpath);
simdjson::padded_string p;
auto error = simdjson::padded_string::load(fullpath).get(p);
if (error) {
std::cerr << "Could not load the file " << fullpath << std::endl;
return EXIT_FAILURE;
tests/pointercheck.cpp
+25 -27
View File
@@ -1,6 +1,7 @@
#include <iostream>
#include "simdjson.h"
#include "test_macros.h"
// we define our own asserts to get around NDEBUG
#ifndef ASSERT
@@ -35,49 +36,46 @@ const padded_string TEST_JSON = R"(
bool json_pointer_success_test(const char *json_pointer, std::string_view expected_value) {
std::cout << "Running successful JSON pointer test '" << json_pointer << "' ..." << std::endl;
dom::parser parser;
auto [value, error] = parser.parse(TEST_JSON).at(json_pointer).get<std::string_view>();
if (error) { std::cerr << "Unexpected Error: " << error << std::endl; return false; }
ASSERT(value == expected_value);
std::string_view value;
ASSERT_SUCCESS( parser.parse(TEST_JSON).at(json_pointer).get(value) );
ASSERT_EQUAL(value, expected_value);
return true;
}
bool json_pointer_success_test(const char *json_pointer) {
std::cout << "Running successful JSON pointer test '" << json_pointer << "' ..." << std::endl;
dom::parser parser;
auto error = parser.parse(TEST_JSON).at(json_pointer).error();
if (error) { std::cerr << "Unexpected Error: " << error << std::endl; return false; }
ASSERT_SUCCESS( parser.parse(TEST_JSON).at(json_pointer).error() );
return true;
}
bool json_pointer_failure_test(const char *json_pointer, error_code expected_failure_test) {
bool json_pointer_failure_test(const char *json_pointer, error_code expected_error) {
std::cout << "Running invalid JSON pointer test '" << json_pointer << "' ..." << std::endl;
dom::parser parser;
auto error = parser.parse(TEST_JSON).at(json_pointer).error();
ASSERT(error == expected_failure_test);
ASSERT_ERROR(parser.parse(TEST_JSON).at(json_pointer).error(), expected_error);
return true;
}
int main() {
if (
json_pointer_success_test("") &&
json_pointer_success_test("~1~001abc") &&
json_pointer_success_test("~1~001abc/1") &&
json_pointer_success_test("~1~001abc/1/\\\" 0") &&
json_pointer_success_test("~1~001abc/1/\\\" 0/0", "value0") &&
json_pointer_success_test("~1~001abc/1/\\\" 0/1", "value1") &&
json_pointer_failure_test("~1~001abc/1/\\\" 0/2", INDEX_OUT_OF_BOUNDS) && // index actually out of bounds
json_pointer_success_test("arr") && // get array
json_pointer_failure_test("arr/0", INDEX_OUT_OF_BOUNDS) && // array index 0 out of bounds on empty array
json_pointer_success_test("~1~001abc") && // get object
json_pointer_success_test("0", "0 ok") && // object index with integer-ish key
json_pointer_success_test("01", "01 ok") && // object index with key that would be an invalid integer
json_pointer_success_test("", "empty ok") && // object index with empty key
json_pointer_failure_test("~01abc", NO_SUCH_FIELD) && // Test that we don't try to compare the literal key
json_pointer_failure_test("~1~001abc/01", INVALID_JSON_POINTER) && // Leading 0 in integer index
json_pointer_failure_test("~1~001abc/", INVALID_JSON_POINTER) && // Empty index to array
json_pointer_failure_test("~1~001abc/-", INDEX_OUT_OF_BOUNDS) && // End index is always out of bounds
true
if (true
&& json_pointer_success_test("")
&& json_pointer_success_test("~1~001abc")
&& json_pointer_success_test("~1~001abc/1")
&& json_pointer_success_test("~1~001abc/1/\\\" 0")
&& json_pointer_success_test("~1~001abc/1/\\\" 0/0", "value0")
&& json_pointer_success_test("~1~001abc/1/\\\" 0/1", "value1")
&& json_pointer_failure_test("~1~001abc/1/\\\" 0/2", INDEX_OUT_OF_BOUNDS) // index actually out of bounds
&& json_pointer_success_test("arr") // get array
&& json_pointer_failure_test("arr/0", INDEX_OUT_OF_BOUNDS) // array index 0 out of bounds on empty array
&& json_pointer_success_test("~1~001abc") // get object
&& json_pointer_success_test("0", "0 ok") // object index with integer-ish key
&& json_pointer_success_test("01", "01 ok") // object index with key that would be an invalid integer
&& json_pointer_success_test("", "empty ok") // object index with empty key
&& json_pointer_failure_test("~01abc", NO_SUCH_FIELD) // Test that we don't try to compare the literal key
&& json_pointer_failure_test("~1~001abc/01", INVALID_JSON_POINTER) // Leading 0 in integer index
&& json_pointer_failure_test("~1~001abc/", INVALID_JSON_POINTER) // Empty index to array
&& json_pointer_failure_test("~1~001abc/-", INDEX_OUT_OF_BOUNDS) // End index is always out of bounds
) {
std::cout << "Success!" << std::endl;
return 0;
tests/readme_examples.cpp
+18 -16
View File
@@ -88,7 +88,7 @@ void basics_dom_4() {
auto abstract_json = R"(
{ "str" : { "123" : {"abc" : 3.14 } } } )"_padded;
dom::parser parser;
double v = parser.parse(abstract_json)["str"]["123"]["abc"].get<double>();
double v = parser.parse(abstract_json)["str"]["123"]["abc"];
cout << "number: " << v << endl;
}
@@ -141,9 +141,10 @@ namespace treewalk_1 {
#ifdef SIMDJSON_CPLUSPLUS17
void basics_cpp17_1() {
dom::parser parser;
padded_string json = R"( { "foo": 1, "bar": 2 } )"_padded;
auto [object, error] = parser.parse(json).get<dom::object>();
dom::parser parser;
dom::object object;
auto error = parser.parse(json).get(object);
if (error) { cerr << error << endl; return; }
for (auto [key, value] : object) {
cout << key << " = " << value << endl;
@@ -153,11 +154,10 @@ void basics_cpp17_1() {
void basics_cpp17_2() {
// C++ 11 version for comparison
dom::parser parser;
padded_string json = R"( { "foo": 1, "bar": 2 } )"_padded;
simdjson::error_code error;
dom::parser parser;
dom::object object;
error = parser.parse(json).get(object);
auto error = parser.parse(json).get(object);
if (!error) { cerr << error << endl; return; }
for (dom::key_value_pair field : object) {
cout << field.key << " = " << field.value << endl;
@@ -216,26 +216,28 @@ SIMDJSON_PUSH_DISABLE_ALL_WARNINGS
// The web_request part of this is aspirational, so we compile as much as we can here
void performance_2() {
dom::parser parser(1000*1000); // Never grow past documents > 1MB
// for (web_request request : listen()) {
auto [doc, error] = parser.parse("1"_padded/*request.body*/);
// // If the document was above our limit, emit 413 = payload too large
/* for (web_request request : listen()) */ {
dom::element doc;
auto error = parser.parse("1"_padded/*request.body*/).get(doc);
// If the document was above our limit, emit 413 = payload too large
if (error == CAPACITY) { /* request.respond(413); continue; */ }
// // ...
// }
// ...
}
}
// The web_request part of this is aspirational, so we compile as much as we can here
void performance_3() {
dom::parser parser(0); // This parser will refuse to automatically grow capacity
simdjson::error_code allocate_error = parser.allocate(1000*1000); // This allocates enough capacity to handle documents <= 1MB
if (allocate_error) { cerr << allocate_error << endl; exit(1); }
auto error = parser.allocate(1000*1000); // This allocates enough capacity to handle documents <= 1MB
if (error) { cerr << error << endl; exit(1); }
// for (web_request request : listen()) {
auto [doc, error] = parser.parse("1"_padded/*request.body*/);
/* for (web_request request : listen()) */ {
dom::element doc;
auto error = parser.parse("1"_padded/*request.body*/).get(doc);
// If the document was above our limit, emit 413 = payload too large
if (error == CAPACITY) { /* request.respond(413); continue; */ }
// ...
// }
}
}
SIMDJSON_POP_DISABLE_WARNINGS
#endif
tests/readme_examples_noexceptions.cpp
+5 -14
View File
@@ -10,7 +10,8 @@ void basics_error_1() {
dom::parser parser;
auto json = "1"_padded;
auto [doc, error] = parser.parse(json); // doc is a dom::element
dom::element doc;
auto error = parser.parse(json).get(doc);
if (error) { cerr << error << endl; exit(1); }
// Use document here now that we've checked for the error
}
@@ -18,14 +19,6 @@ SIMDJSON_POP_DISABLE_WARNINGS
#endif
void basics_error_2() {
dom::parser parser;
auto json = "1"_padded;
dom::element doc;
UNUSED auto error = parser.parse(json).get(doc); // <-- Assigns to doc and error just like "auto [doc, error]"}
}
void basics_error_3() {
auto cars_json = R"( [
{ "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
{ "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
@@ -70,8 +63,7 @@ void basics_error_3() {
}
}
void basics_error_4() {
void basics_error_3() {
auto abstract_json = R"( [
{ "12345" : {"a":12.34, "b":56.78, "c": 9998877} },
{ "12545" : {"a":11.44, "b":12.78, "c": 11111111} }
@@ -102,7 +94,7 @@ void basics_error_4() {
}
}
void basics_error_5() {
void basics_error_4() {
auto abstract_json = R"(
{ "str" : { "123" : {"abc" : 3.14 } } } )"_padded;
dom::parser parser;
@@ -116,7 +108,7 @@ void basics_error_5() {
#ifdef SIMDJSON_CPLUSPLUS17
void basics_error_3_cpp17() {
void basics_error_2_cpp17() {
auto cars_json = R"( [
{ "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
{ "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
@@ -201,6 +193,5 @@ int main() {
basics_error_2();
basics_error_3();
basics_error_4();
basics_error_5();
return EXIT_SUCCESS;
}
tests/singleheadertest.cpp
+4 -3
View File
@@ -6,12 +6,13 @@ using namespace simdjson;
int main() {
const char *filename = SIMDJSON_BENCHMARK_DATA_DIR "/twitter.json";
padded_string p = get_corpus(filename);
dom::parser parser;
auto [doc, error] = parser.parse(p);
if(error) {
dom::element doc;
auto error = parser.load(filename).get(doc);
if (error) {
std::cerr << error << std::endl;
return EXIT_FAILURE;
}
std::cout << doc << std::endl;
return EXIT_SUCCESS;
}
tests/stringparsingcheck.cpp
+2 -1
View File
@@ -337,7 +337,8 @@ bool validate(const char *dirname) {
} else {
strcpy(fullpath + dirlen, name);
}
auto [p, error] = simdjson::padded_string::load(fullpath);
simdjson::padded_string p;
auto error = simdjson::padded_string::load(fullpath).get(p);
if (error) {
std::cerr << "Could not load the file " << fullpath << std::endl;
return EXIT_FAILURE;
tests/test_macros.h
+23 -7
View File
@@ -18,21 +18,37 @@ const char *SMALLDEMO_JSON = SIMDJSON_BENCHMARK_SMALLDATA_DIR "smalldemo.json";
const char *TRUENULL_JSON = SIMDJSON_BENCHMARK_SMALLDATA_DIR "truenull.json";
// For the ASSERT_EQUAL macro
template<typename T>
bool equals_expected(T actual, T expected) {
return actual == expected;
template<typename T, typename S>
really_inline bool equals_expected(T actual, S expected) {
return actual == T(expected);
}
template<>
bool equals_expected<const char *>(const char *actual, const char *expected) {
really_inline bool equals_expected<const char *, const char *>(const char *actual, const char *expected) {
return !strcmp(actual, expected);
}
really_inline simdjson::error_code to_error_code(simdjson::error_code error) {
return error;
}
template<typename T>
really_inline simdjson::error_code to_error_code(const simdjson::simdjson_result<T> &result) {
return result.error();
}
#define TEST_START() { cout << "Running " << __func__ << " ..." << endl; }
#define ASSERT_EQUAL(ACTUAL, EXPECTED) do { auto _actual = (ACTUAL); auto _expected = (EXPECTED); if (!equals_expected(_actual, _expected)) { std::cerr << "Expected " << #ACTUAL << " to be " << _expected << ", got " << _actual << " instead!" << std::endl; return false; } } while(0);
#define ASSERT_ERROR(ACTUAL, EXPECTED) do { auto _actual = (ACTUAL); auto _expected = (EXPECTED); if (_actual != _expected) { std::cerr << "FAIL: Unexpected error \"" << _actual << "\" (expected \"" << _expected << "\")" << std::endl; return false; } } while (0);
#define ASSERT_EQUAL(ACTUAL, EXPECTED) \
do { \
auto _actual = (ACTUAL); \
auto _expected = (EXPECTED); \
if (!equals_expected(_actual, _expected)) { \
std::cerr << "Expected " << (#ACTUAL) << " to be " << _expected << ", got " << _actual << " instead!" << std::endl; \
return false; \
} \
} while(0);
#define ASSERT_ERROR(ACTUAL, EXPECTED) do { auto _actual = to_error_code(ACTUAL); auto _expected = to_error_code(EXPECTED); if (_actual != _expected) { std::cerr << "FAIL: Unexpected error \"" << _actual << "\" (expected \"" << _expected << "\")" << std::endl; return false; } } while (0);
#define ASSERT(RESULT, MESSAGE) if (!(RESULT)) { std::cerr << MESSAGE << std::endl; return false; }
#define RUN_TEST(RESULT) if (!RESULT) { return false; }
#define ASSERT_SUCCESS(ERROR) do { auto _error = (ERROR); if (_error) { std::cerr << _error << std::endl; return false; } } while(0);
#define ASSERT_SUCCESS(ERROR) do { auto _error = to_error_code(ERROR); if (_error) { std::cerr << _error << std::endl; return false; } } while(0);
#define TEST_FAIL(MESSAGE) { std::cerr << "FAIL: " << (MESSAGE) << std::endl; return false; }
#define TEST_SUCCEED() { return true; }
tools/json2json.cpp
+4 -4
View File
@@ -49,10 +49,10 @@ int main(int argc, char *argv[]) {
const char *filename = result["file"].as<std::string>().c_str();
simdjson::dom::parser parser;
auto [doc, error] = parser.load(filename); // do the parsing, return false on error
if (error != simdjson::SUCCESS) {
std::cerr << " Parsing failed. Error is '" << simdjson::error_message(error)
<< "'." << std::endl;
simdjson::dom::element doc;
auto error = parser.load(filename).get(doc); // do the parsing, return false on error
if (error) {
std::cerr << " Parsing failed. Error is '" << error << "'." << std::endl;
return EXIT_FAILURE;
}
if(rawdump) {
tools/jsonpointer.cpp
+5 -4
View File
@@ -20,16 +20,17 @@ int main(int argc, char *argv[]) {
const char *filename = argv[1];
simdjson::dom::parser parser;
auto [doc, error] = parser.load(filename);
simdjson::dom::element doc;
auto error = parser.load(filename).get(doc);
if (error) { std::cerr << "Error parsing " << filename << ": " << error << std::endl; }
std::cout << "[" << std::endl;
for (int idx = 2; idx < argc; idx++) {
const char *json_pointer = argv[idx];
auto [value, pointer_error] = doc[json_pointer];
simdjson::dom::element value;
std::cout << "{\"jsonpath\": \"" << json_pointer << "\"";
if (pointer_error) {
std::cout << ",\"error\":\"" << pointer_error << "\"";
if ((error = doc[json_pointer].get(value))) {
std::cout << ",\"error\":\"" << error << "\"";
} else {
std::cout << ",\"value\":" << value;
}
tools/jsonstats.cpp
+4 -2
View File
@@ -166,7 +166,8 @@ void recurse(simdjson::dom::element element, stat_t &s, size_t depth) {
stat_t simdjson_compute_stats(const simdjson::padded_string &p) {
stat_t s{};
simdjson::dom::parser parser;
auto [doc, error] = parser.parse(p);
simdjson::dom::element doc;
auto error = parser.parse(p).get(doc);
if (error) {
s.valid = false;
std::cerr << error << std::endl;
@@ -217,7 +218,8 @@ int main(int argc, char *argv[]) {
const char *filename = result["file"].as<std::string>().c_str();
auto [p, error] = simdjson::padded_string::load(filename);
simdjson::padded_string p;
auto error = simdjson::padded_string::load(filename).get(p);
if (error) {
std::cerr << "Could not load the file " << filename << std::endl;
return EXIT_FAILURE;
tools/minify.cpp
+2 -1
View File
@@ -56,7 +56,8 @@ int main(int argc, char *argv[]) {
std::string filename = result["file"].as<std::string>();
auto [p, error] = simdjson::padded_string::load(filename);
simdjson::padded_string p;
auto error = simdjson::padded_string::load(filename).get(p);
if (error) {
std::cerr << "Could not load the file " << filename << std::endl;
return EXIT_FAILURE;