diff --git a/benchmark/bench_dom_api.cpp b/benchmark/bench_dom_api.cpp index 1440942c7..ed6467bd9 100644 --- a/benchmark/bench_dom_api.cpp +++ b/benchmark/bench_dom_api.cpp @@ -407,7 +407,7 @@ static void iterator_twitter_default_profile(State& state) { set default_users; ParsedJson::Iterator iter(pj); - // for (dom::object tweet : doc["statuses"].get()) { + // for (dom::object tweet : doc["statuses"]) { if (!(iter.move_to_key("statuses") && iter.is_array())) { return; } if (iter.down()) { // first status do { @@ -480,23 +480,24 @@ static void iterator_twitter_image_sizes(State& state) { set> image_sizes; ParsedJson::Iterator iter(pj); - // for (dom::object tweet : doc["statuses"].get()) { + // for (dom::object tweet : doc["statuses"]) { if (!(iter.move_to_key("statuses") && iter.is_array())) { return; } if (iter.down()) { // first status do { - // auto [media, not_found] = tweet["entities"]["media"]; + // dom::object media; + // not_found = tweet["entities"]["media"].get(media); // if (!not_found) { if (iter.move_to_key("entities")) { if (!iter.is_object()) { return; } if (iter.move_to_key("media")) { if (!iter.is_array()) { return; } - // for (dom::object image : media.get()) { + // for (dom::object image : media) { if (iter.down()) { // first media do { - // for (auto [key, size] : image["sizes"].get()) { + // for (auto [key, size] : dom::object(image["sizes"])) { if (!(iter.move_to_key("sizes") && iter.is_object())) { return; } if (iter.down()) { // first size do { diff --git a/benchmark/distinctuseridcompetition.cpp b/benchmark/distinctuseridcompetition.cpp index 100c48f66..f2c4a6434 100644 --- a/benchmark/distinctuseridcompetition.cpp +++ b/benchmark/distinctuseridcompetition.cpp @@ -40,17 +40,18 @@ void print_vec(const std::vector &v) { // simdjson_recurse below come be implemented like so but it is slow: /*void simdjson_recurse(std::vector & v, simdjson::dom::element element) { - if (element.is()) { - auto [array, array_error] = element.get(); + error_code error; + if (element.is_array()) { + dom::array array; + error = element.get(array); for (auto child : array) { if (child.is() || child.is()) { simdjson_recurse(v, child); } } - } else if (element.is()) { - auto [object, error] = element.get(); + } else if (element.is_object()) { int64_t id; - error = object["user"]["id"].get(id); + error = element["user"]["id"].get(id); if(!error) { v.push_back(id); } @@ -330,7 +331,8 @@ int main(int argc, char *argv[]) { std::cerr << "warning: ignoring everything after " << argv[optind + 1] << std::endl; } - auto [p, error] = simdjson::padded_string::load(filename); + simdjson::padded_string p; + auto error = simdjson::padded_string::load(filename).get(p); if (error) { std::cerr << "Could not load the file " << filename << std::endl; return EXIT_FAILURE; diff --git a/benchmark/minifiercompetition.cpp b/benchmark/minifiercompetition.cpp index 2a8c1611a..08c80f7aa 100644 --- a/benchmark/minifiercompetition.cpp +++ b/benchmark/minifiercompetition.cpp @@ -75,7 +75,8 @@ int main(int argc, char *argv[]) { exit(1); } const char *filename = argv[optind]; - auto [p, error] = simdjson::padded_string::load(filename); + simdjson::padded_string p; + auto error = simdjson::padded_string::load(filename).get(p); if (error) { std::cerr << "Could not load the file " << filename << std::endl; return EXIT_FAILURE; diff --git a/benchmark/parse_stream.cpp b/benchmark/parse_stream.cpp index 5735d4176..d893fd1d4 100644 --- a/benchmark/parse_stream.cpp +++ b/benchmark/parse_stream.cpp @@ -87,10 +87,15 @@ int main(int argc, char *argv[]) { auto start = std::chrono::steady_clock::now(); count = 0; - for (auto result : parser.parse_many(p, i)) { + simdjson::dom::document_stream docs; + if ((error = parser.parse_many(p, i).get(docs))) { + std::wcerr << "Parsing failed with: " << error << std::endl; + exit(1); + } + for (auto result : docs) { error = result.error(); - if (error != simdjson::SUCCESS) { - std::wcerr << "Parsing failed with: " << error_message(error) << std::endl; + if (error) { + std::wcerr << "Parsing failed with: " << error << std::endl; exit(1); } count++; @@ -134,10 +139,15 @@ int main(int argc, char *argv[]) { auto start = std::chrono::steady_clock::now(); // This includes allocation of the parser - for (auto result : parser.parse_many(p, optimal_batch_size)) { + simdjson::dom::document_stream docs; + if ((error = parser.parse_many(p, optimal_batch_size).get(docs))) { + std::wcerr << "Parsing failed with: " << error << std::endl; + exit(1); + } + for (auto result : docs) { error = result.error(); - if (error != simdjson::SUCCESS) { - std::wcerr << "Parsing failed with: " << error_message(error) << std::endl; + if (error) { + std::wcerr << "Parsing failed with: " << error << std::endl; exit(1); } } diff --git a/benchmark/parseandstatcompetition.cpp b/benchmark/parseandstatcompetition.cpp index eb017ebd9..72972ca92 100644 --- a/benchmark/parseandstatcompetition.cpp +++ b/benchmark/parseandstatcompetition.cpp @@ -105,7 +105,8 @@ void simdjson_recurse(stat_t &s, simdjson::dom::element element) { never_inline stat_t simdjson_compute_stats(const simdjson::padded_string &p) { stat_t s{}; simdjson::dom::parser parser; - auto [doc, error] = parser.parse(p); + simdjson::dom::element doc; + auto error = parser.parse(p).get(doc); if (error) { s.valid = false; return s; @@ -154,11 +155,11 @@ static void GenStatPlus(Stat &stat, const dom::element &v) { break; case dom::element_type::STRING: { stat.stringCount++; - std::string_view sv = v.get(); + auto sv = std::string_view(v); stat.stringLength += sv.size(); } break; case dom::element_type::BOOL: - if (v.get()) { + if (bool(v)) { stat.trueCount++; } else { stat.falseCount++; @@ -409,7 +410,8 @@ int main(int argc, char *argv[]) { std::cerr << "warning: ignoring everything after " << argv[optind + 1] << std::endl; } - auto [p, error] = simdjson::padded_string::load(filename); + simdjson::padded_string p; + auto error = simdjson::padded_string::load(filename).get(p); if (error) { std::cerr << "Could not load the file " << filename << std::endl; return EXIT_FAILURE; @@ -464,9 +466,10 @@ int main(int argc, char *argv[]) { printf("API traversal tests\n"); printf("Based on https://github.com/miloyip/nativejson-benchmark\n"); simdjson::dom::parser parser; - auto [doc, err] = parser.parse(p); - if (err) { - std::cerr << err << std::endl; + simdjson::dom::element doc; + auto error = parser.parse(p).get(doc); + if (error) { + std::cerr << error << std::endl; } size_t refval = simdjson_compute_stats_refplus(doc).objectCount; diff --git a/benchmark/parsingcompetition.cpp b/benchmark/parsingcompetition.cpp index 80a2a76d9..509e44087 100644 --- a/benchmark/parsingcompetition.cpp +++ b/benchmark/parsingcompetition.cpp @@ -82,9 +82,10 @@ inline void reset_stream(std::stringstream & is) { bool bench(const char *filename, bool verbose, bool just_data, double repeat_multiplier) { - auto [p, err] = simdjson::padded_string::load(filename); - if (err) { - std::cerr << "Could not load the file " << filename << std::endl; + simdjson::padded_string p; + auto error = simdjson::padded_string::load(filename).get(p); + if (error) { + std::cerr << "Could not load the file " << filename << ": " << error << std::endl; return false; } diff --git a/benchmark/statisticalmodel.cpp b/benchmark/statisticalmodel.cpp index 3838f5ef9..89efe82d6 100644 --- a/benchmark/statisticalmodel.cpp +++ b/benchmark/statisticalmodel.cpp @@ -96,7 +96,8 @@ void simdjson_recurse(stat_t &s, simdjson::dom::element element) { stat_t simdjson_compute_stats(const simdjson::padded_string &p) { stat_t answer{}; simdjson::dom::parser parser; - auto [doc, error] = parser.parse(p); + simdjson::dom::element doc; + auto error = parser.parse(p).get(doc); if (error) { answer.valid = false; return answer; @@ -136,7 +137,8 @@ int main(int argc, char *argv[]) { std::cerr << "warning: ignoring everything after " << argv[optind + 1] << std::endl; } - auto [p, error] = simdjson::padded_string::load(filename); + simdjson::padded_string p; + auto error = simdjson::padded_string::load(filename).get(p); if (error) { std::cerr << "Could not load the file " << filename << std::endl; return EXIT_FAILURE; diff --git a/doc/basics.md b/doc/basics.md index 3ce7e9be4..f3e91e986 100644 --- a/doc/basics.md +++ b/doc/basics.md @@ -187,21 +187,20 @@ And another one: auto abstract_json = R"( { "str" : { "123" : {"abc" : 3.14 } } } )"_padded; dom::parser parser; - double v = parser.parse(abstract_json)["str"]["123"]["abc"].get(); + double v = parser.parse(abstract_json)["str"]["123"]["abc"]; cout << "number: " << v << endl; ``` C++17 Support ------------- -While the simdjson library can be used in any project using C++ 11 and above, it has special support -for C++ 17. The APIs for field iteration and error handling in particular are designed to work -nicely with C++17's destructuring syntax. For example: +While the simdjson library can be used in any project using C++ 11 and above, field iteration has special support C++ 17's destructuring syntax. For example: ```c++ -dom::parser parser; padded_string json = R"( { "foo": 1, "bar": 2 } )"_padded; -auto [object, error] = parser.parse(json).get(); +dom::parser parser; +dom::object object; +auto error = parser.parse(json).get(object); if (error) { cerr << error << endl; return; } for (auto [key, value] : object) { cout << key << " = " << value << endl; @@ -212,11 +211,10 @@ For comparison, here is the C++ 11 version of the same code: ```c++ // C++ 11 version for comparison -dom::parser parser; padded_string json = R"( { "foo": 1, "bar": 2 } )"_padded; -simdjson::error_code error; +dom::parser parser; dom::object object; -error = parser.parse(json).get(object); +auto error = parser.parse(json).get(object); if (!error) { cerr << error << endl; return; } for (dom::key_value_pair field : object) { cout << field.key << " = " << field.value << endl; @@ -259,7 +257,6 @@ The UTF-8 validation function merely checks that the input is valid UTF-8: it wo Your input string does not need any padding. Any string will do. The `validate_utf8` function does not do any memory allocation on the heap, and it does not throw exceptions. - JSON Pointer ------------ @@ -281,29 +278,18 @@ Error Handling -------------- All simdjson APIs that can fail return `simdjson_result`, which is a <value, error_code> -pair. The error codes and values can be accessed directly, reading the error like so: +pair. You can retrieve the value with .get(), like so: ```c++ -auto [doc, error] = parser.parse(json); // doc is a dom::element +dom::element doc; +auto error = parser.parse(json).get(doc); if (error) { cerr << error << endl; exit(1); } -// Use document here now that we've checked for the error ``` When you use the code this way, it is your responsibility to check for error before using the result: if there is an error, the result value will not be valid and using it will caused undefined behavior. -> Note: because of the way `auto [x, y]` works in C++, you have to define new variables each time you -> use it. If your project treats aliased, this means you can't use the same names in `auto [x, error]` -> without triggering warnings or error (and particularly can't use the word "error" every time). To -> circumvent this, you can use this instead: -> -> ```c++ -> dom::element doc; -> auto error = parser.parse(json).get(doc); // <-- Assigns to doc and error just like "auto [doc, error]" -> ``` - - We can write a "quick start" example where we attempt to parse a file and access some data, without triggering exceptions: ```C++ @@ -311,11 +297,12 @@ We can write a "quick start" example where we attempt to parse a file and access int main(void) { simdjson::dom::parser parser; + simdjson::dom::element tweets; auto error = parser.load("twitter.json").get(tweets); if (error) { std::cerr << error << std::endl; return EXIT_FAILURE; } - simdjson::dom::element res; + simdjson::dom::element res; if ((error = tweets["search_metadata"]["count"].get(res))) { std::cerr << "could not access keys" << std::endl; return EXIT_FAILURE; @@ -418,8 +405,7 @@ And another one: cout << "number: " << v << endl; ``` -Notice how we can string several operation (`parser.parse(abstract_json)["str"]["123"]["abc"].get()`) and only check for the error once, a strategy we call *error chaining*. - +Notice how we can string several operations (`parser.parse(abstract_json)["str"]["123"]["abc"].get(v)`) and only check for the error once, a strategy we call *error chaining*. The next two functions will take as input a JSON document containing an array with a single element, either a string or a number. They return true upon success. @@ -529,7 +515,8 @@ Here is a simple example, given "x.json" with this content: ```c++ dom::parser parser; -for (dom::element doc : parser.load_many(filename)) { +dom::document_stream docs = parser.load_many(filename); +for (dom::element doc : docs) { cout << doc["foo"] << endl; } // Prints 1 2 3 diff --git a/doc/performance.md b/doc/performance.md index bf022a4cd..1b65e995b 100644 --- a/doc/performance.md +++ b/doc/performance.md @@ -68,7 +68,8 @@ without bound: ```c++ dom::parser parser(1000*1000); // Never grow past documents > 1MB for (web_request request : listen()) { - auto [doc, error] = parser.parse(request.body); + dom::element doc; + auto error = parser.parse(request.body).get(doc); // If the document was above our limit, emit 413 = payload too large if (error == CAPACITY) { request.respond(413); continue; } // ... @@ -82,11 +83,12 @@ without bound: ```c++ dom::parser parser(0); // This parser will refuse to automatically grow capacity - simdjson::error_code allocate_error = parser.allocate(1000*1000); // This allocates enough capacity to handle documents <= 1MB - if (allocate_error) { cerr << allocate_error << endl; exit(1); } + auto error = parser.allocate(1000*1000); // This allocates enough capacity to handle documents <= 1MB + if (error) { cerr << error << endl; exit(1); } for (web_request request : listen()) { - auto [doc, error] = parser.parse(request.body); + dom::element doc; + error = parser.parse(request.body).get(doc); // If the document was above our limit, emit 413 = payload too large if (error == CAPACITY) { request.respond(413); continue; } // ... diff --git a/include/simdjson/dom/array.h b/include/simdjson/dom/array.h index 55d3ec8a8..a54383b41 100644 --- a/include/simdjson/dom/array.h +++ b/include/simdjson/dom/array.h @@ -68,7 +68,7 @@ public: * Get the value associated with the given JSON pointer. * * dom::parser parser; - * array a = parser.parse(R"([ { "foo": { "a": [ 10, 20, 30 ] }} ])"); + * array a = parser.parse(R"([ { "foo": { "a": [ 10, 20, 30 ] }} ])"_padded); * a.at("0/foo/a/1") == 20 * a.at("0")["foo"]["a"].at(1) == 20 * diff --git a/include/simdjson/dom/document_stream.h b/include/simdjson/dom/document_stream.h index 947ebf8ce..eb4f53c54 100644 --- a/include/simdjson/dom/document_stream.h +++ b/include/simdjson/dom/document_stream.h @@ -72,8 +72,20 @@ private: */ class document_stream { public: + /** + * Construct an uninitialized document_stream. + * + * ```c++ + * document_stream docs; + * error = parser.parse_many(json).get(docs); + * ``` + */ + really_inline document_stream() noexcept; /** Move one document_stream to another. */ - really_inline document_stream(document_stream && other) noexcept = default; + really_inline document_stream(document_stream &&other) noexcept = default; + /** Move one document_stream to another. */ + really_inline document_stream &operator=(document_stream &&other) noexcept = default; + really_inline ~document_stream() noexcept; /** @@ -99,9 +111,8 @@ public: * * Gives the current index in the input document in bytes. * - * auto stream = parser.parse_many(json,window); - * auto i = stream.begin(); - * for(; i != stream.end(); ++i) { + * document_stream stream = parser.parse_many(json,window); + * for(auto i = stream.begin(); i != stream.end(); ++i) { * auto doc = *i; * size_t index = i.current_index(); * } @@ -132,8 +143,7 @@ public: private: document_stream &operator=(const document_stream &) = delete; // Disallow copying - - document_stream(document_stream &other) = delete; // Disallow copying + document_stream(const document_stream &other) = delete; // Disallow copying /** * Construct a document_stream. Does not allocate or parse anything until the iterator is @@ -141,18 +151,9 @@ private: */ really_inline document_stream( dom::parser &parser, - size_t batch_size, const uint8_t *buf, - size_t len - ) noexcept; - - /** - * Construct a document_stream with an initial error. - */ - really_inline document_stream( - dom::parser &parser, - size_t batch_size, - error_code error + size_t len, + size_t batch_size ) noexcept; /** @@ -199,13 +200,14 @@ private: /** Pass the next batch through stage 1 with the given parser. */ inline error_code run_stage1(dom::parser &p, size_t batch_start) noexcept; - dom::parser &parser; + dom::parser *parser; const uint8_t *buf; - const size_t len; - const size_t batch_size; - size_t batch_start{0}; + size_t len; + size_t batch_size; /** The error (or lack thereof) from the current document. */ error_code error; + size_t batch_start{0}; + size_t doc_index{}; #ifdef SIMDJSON_THREADS_ENABLED inline void load_from_stage1_thread() noexcept; @@ -229,12 +231,31 @@ private: #endif // SIMDJSON_THREADS_ENABLED friend class dom::parser; - - size_t doc_index{}; + friend struct simdjson_result; + friend struct internal::simdjson_result_base; }; // class document_stream } // namespace dom + +template<> +struct simdjson_result : public internal::simdjson_result_base { +public: + really_inline simdjson_result() noexcept; ///< @private + really_inline simdjson_result(error_code error) noexcept; ///< @private + really_inline simdjson_result(dom::document_stream &&value) noexcept; ///< @private + +#if SIMDJSON_EXCEPTIONS + really_inline dom::document_stream::iterator begin() noexcept(false); + really_inline dom::document_stream::iterator end() noexcept(false); +#else // SIMDJSON_EXCEPTIONS + [[deprecated("parse_many() and load_many() may return errors. Use document_stream stream; error = parser.parse_many().get(doc); instead.")]] + really_inline dom::document_stream::iterator begin() noexcept; + [[deprecated("parse_many() and load_many() may return errors. Use document_stream stream; error = parser.parse_many().get(doc); instead.")]] + really_inline dom::document_stream::iterator end() noexcept; +#endif // SIMDJSON_EXCEPTIONS +}; // struct simdjson_result + } // namespace simdjson #endif // SIMDJSON_DOCUMENT_STREAM_H diff --git a/include/simdjson/dom/element.h b/include/simdjson/dom/element.h index ea84d4f00..dbdc08204 100644 --- a/include/simdjson/dom/element.h +++ b/include/simdjson/dom/element.h @@ -243,25 +243,6 @@ public: template inline void tie(T &value, error_code &error) && noexcept; - /** - * Get the value as the provided type (T). - * - * Supported types: - * - Boolean: bool - * - Number: double, uint64_t, int64_t - * - String: std::string_view, const char * - * - Array: dom::array - * - Object: dom::object - * - * @tparam T bool, double, uint64_t, int64_t, std::string_view, const char *, dom::array, dom::object - * - * @param value The variable to set to the given type. value is undefined if there is an error. - * - * @returns true if the value was able to be set, false if there was an error. - */ - template - WARN_UNUSED inline bool tie(T &value) && noexcept; - #if SIMDJSON_EXCEPTIONS /** * Read this element as a boolean. @@ -355,8 +336,8 @@ public: * The key will be matched against **unescaped** JSON: * * dom::parser parser; - * parser.parse(R"({ "a\n": 1 })")["a\n"].get().value == 1 - * parser.parse(R"({ "a\n": 1 })")["a\\n"].get().error == NO_SUCH_FIELD + * parser.parse(R"({ "a\n": 1 })"_padded)["a\n"].get().first == 1 + * parser.parse(R"({ "a\n": 1 })"_padded)["a\\n"].get().error() == NO_SUCH_FIELD * * @return The value associated with this field, or: * - NO_SUCH_FIELD if the field does not exist in the object @@ -370,8 +351,8 @@ public: * The key will be matched against **unescaped** JSON: * * dom::parser parser; - * parser.parse(R"({ "a\n": 1 })")["a\n"].get().value == 1 - * parser.parse(R"({ "a\n": 1 })")["a\\n"].get().error == NO_SUCH_FIELD + * parser.parse(R"({ "a\n": 1 })"_padded)["a\n"].get().first == 1 + * parser.parse(R"({ "a\n": 1 })"_padded)["a\\n"].get().error() == NO_SUCH_FIELD * * @return The value associated with this field, or: * - NO_SUCH_FIELD if the field does not exist in the object @@ -383,7 +364,7 @@ public: * Get the value associated with the given JSON pointer. * * dom::parser parser; - * element doc = parser.parse(R"({ "foo": { "a": [ 10, 20, 30 ] }})"); + * element doc = parser.parse(R"({ "foo": { "a": [ 10, 20, 30 ] }})"_padded); * doc.at("/foo/a/1") == 20 * doc.at("/")["foo"]["a"].at(1) == 20 * doc.at("")["foo"]["a"].at(1) == 20 @@ -410,8 +391,8 @@ public: * The key will be matched against **unescaped** JSON: * * dom::parser parser; - * parser.parse(R"({ "a\n": 1 })")["a\n"].get().value == 1 - * parser.parse(R"({ "a\n": 1 })")["a\\n"].get().error == NO_SUCH_FIELD + * parser.parse(R"({ "a\n": 1 })"_padded)["a\n"].get().first == 1 + * parser.parse(R"({ "a\n": 1 })"_padded)["a\\n"].get().error() == NO_SUCH_FIELD * * @return The value associated with this field, or: * - NO_SUCH_FIELD if the field does not exist in the object @@ -474,7 +455,7 @@ public: really_inline simdjson_result type() const noexcept; template - really_inline simdjson_result is() const noexcept; + really_inline bool is() const noexcept; template really_inline simdjson_result get() const noexcept; template @@ -489,14 +470,14 @@ public: really_inline simdjson_result get_double() const noexcept; really_inline simdjson_result get_bool() const noexcept; - really_inline simdjson_result is_array() const noexcept; - really_inline simdjson_result is_object() const noexcept; - really_inline simdjson_result is_string() const noexcept; - really_inline simdjson_result is_int64_t() const noexcept; - really_inline simdjson_result is_uint64_t() const noexcept; - really_inline simdjson_result is_double() const noexcept; - really_inline simdjson_result is_bool() const noexcept; - really_inline simdjson_result is_null() const noexcept; + really_inline bool is_array() const noexcept; + really_inline bool is_object() const noexcept; + really_inline bool is_string() const noexcept; + really_inline bool is_int64_t() const noexcept; + really_inline bool is_uint64_t() const noexcept; + really_inline bool is_double() const noexcept; + really_inline bool is_bool() const noexcept; + really_inline bool is_null() const noexcept; really_inline simdjson_result operator[](const std::string_view &key) const noexcept; really_inline simdjson_result operator[](const char *key) const noexcept; diff --git a/include/simdjson/dom/object.h b/include/simdjson/dom/object.h index 9316914d9..3be058eb4 100644 --- a/include/simdjson/dom/object.h +++ b/include/simdjson/dom/object.h @@ -101,8 +101,8 @@ public: * The key will be matched against **unescaped** JSON: * * dom::parser parser; - * parser.parse(R"({ "a\n": 1 })")["a\n"].get().value == 1 - * parser.parse(R"({ "a\n": 1 })")["a\\n"].get().error == NO_SUCH_FIELD + * parser.parse(R"({ "a\n": 1 })"_padded)["a\n"].get().first == 1 + * parser.parse(R"({ "a\n": 1 })"_padded)["a\\n"].get().error() == NO_SUCH_FIELD * * This function has linear-time complexity: the keys are checked one by one. * @@ -118,8 +118,8 @@ public: * The key will be matched against **unescaped** JSON: * * dom::parser parser; - * parser.parse(R"({ "a\n": 1 })")["a\n"].get().value == 1 - * parser.parse(R"({ "a\n": 1 })")["a\\n"].get().error == NO_SUCH_FIELD + * parser.parse(R"({ "a\n": 1 })"_padded)["a\n"].get().first == 1 + * parser.parse(R"({ "a\n": 1 })"_padded)["a\\n"].get().error() == NO_SUCH_FIELD * * This function has linear-time complexity: the keys are checked one by one. * @@ -133,7 +133,7 @@ public: * Get the value associated with the given JSON pointer. * * dom::parser parser; - * object obj = parser.parse(R"({ "foo": { "a": [ 10, 20, 30 ] }})"); + * object obj = parser.parse(R"({ "foo": { "a": [ 10, 20, 30 ] }})"_padded); * obj.at("foo/a/1") == 20 * obj.at("foo")["a"].at(1) == 20 * @@ -151,8 +151,8 @@ public: * The key will be matched against **unescaped** JSON: * * dom::parser parser; - * parser.parse(R"({ "a\n": 1 })")["a\n"].get().value == 1 - * parser.parse(R"({ "a\n": 1 })")["a\\n"].get().error == NO_SUCH_FIELD + * parser.parse(R"({ "a\n": 1 })"_padded)["a\n"].get().first == 1 + * parser.parse(R"({ "a\n": 1 })"_padded)["a\\n"].get().error() == NO_SUCH_FIELD * * This function has linear-time complexity: the keys are checked one by one. * diff --git a/include/simdjson/dom/parser.h b/include/simdjson/dom/parser.h index 49ca234da..25eccf430 100644 --- a/include/simdjson/dom/parser.h +++ b/include/simdjson/dom/parser.h @@ -173,9 +173,13 @@ public: * the same interface, requiring you to check the error before using the document: * * dom::parser parser; - * for (auto [doc, error] : parser.load_many(path)) { - * if (error) { cerr << error << endl; exit(1); } - * cout << std::string(doc["title"]) << endl; + * dom::document_stream docs; + * auto error = parser.load_many(path).get(docs); + * if (error) { cerr << error << endl; exit(1); } + * for (auto doc : docs) { + * std::string_view title; + * if ((error = doc["title"].get(title)) { cerr << error << endl; exit(1); } + * cout << title << endl; * } * * ### Threads @@ -193,20 +197,19 @@ public: * spot is cache-related: small enough to fit in cache, yet big enough to * parse as many documents as possible in one tight loop. * Defaults to 10MB, which has been a reasonable sweet spot in our tests. - * @return The stream. If there is an error, it will be returned during iteration. An empty input - * will yield 0 documents rather than an EMPTY error. Errors: + * @return The stream, or an error. An empty input will yield 0 documents rather than an EMPTY error. Errors: * - IO_ERROR if there was an error opening or reading the file. * - MEMALLOC if the parser does not have enough capacity and memory allocation fails. * - CAPACITY if the parser does not have enough capacity and batch_size > max_capacity. * - other json errors if parsing fails. */ - inline document_stream load_many(const std::string &path, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept; + inline simdjson_result load_many(const std::string &path, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept; /** * Parse a buffer containing many JSON documents. * * dom::parser parser; - * for (const element doc : parser.parse_many(buf, len)) { + * for (element doc : parser.parse_many(buf, len)) { * cout << std::string(doc["title"]) << endl; * } * @@ -234,9 +237,13 @@ public: * the same interface, requiring you to check the error before using the document: * * dom::parser parser; - * for (auto [doc, error] : parser.parse_many(buf, len)) { - * if (error) { cerr << error << endl; exit(1); } - * cout << std::string(doc["title"]) << endl; + * dom::document_stream docs; + * auto error = parser.load_many(path).get(docs); + * if (error) { cerr << error << endl; exit(1); } + * for (auto doc : docs) { + * std::string_view title; + * if ((error = doc["title"].get(title)) { cerr << error << endl; exit(1); } + * cout << title << endl; * } * * ### REQUIRED: Buffer Padding @@ -260,22 +267,21 @@ public: * spot is cache-related: small enough to fit in cache, yet big enough to * parse as many documents as possible in one tight loop. * Defaults to 10MB, which has been a reasonable sweet spot in our tests. - * @return The stream. If there is an error, it will be returned during iteration. An empty input - * will yield 0 documents rather than an EMPTY error. Errors: + * @return The stream, or an error. An empty input will yield 0 documents rather than an EMPTY error. Errors: * - MEMALLOC if the parser does not have enough capacity and memory allocation fails * - CAPACITY if the parser does not have enough capacity and batch_size > max_capacity. * - other json errors if parsing fails. */ - inline document_stream parse_many(const uint8_t *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept; + inline simdjson_result parse_many(const uint8_t *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept; /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ - inline document_stream parse_many(const char *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept; + inline simdjson_result parse_many(const char *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept; /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ - inline document_stream parse_many(const std::string &s, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept; + inline simdjson_result parse_many(const std::string &s, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept; /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ - inline document_stream parse_many(const padded_string &s, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept; + inline simdjson_result parse_many(const padded_string &s, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept; /** @private We do not want to allow implicit conversion from C string to std::string. */ - really_inline simdjson_result parse_many(const char *buf, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept = delete; + simdjson_result parse_many(const char *buf, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept = delete; /** * Ensure this parser has enough memory to process JSON documents up to `capacity` bytes in length diff --git a/include/simdjson/error.h b/include/simdjson/error.h index f272abd83..308bc1f0b 100644 --- a/include/simdjson/error.h +++ b/include/simdjson/error.h @@ -42,7 +42,8 @@ enum error_code { * Get the error message for the given error code. * * dom::parser parser; - * auto [doc, error] = parser.parse("foo"); + * dom::element doc; + * auto error = parser.parse("foo").get(doc); * if (error) { printf("Error: %s\n", error_message(error)); } * * @return The error message. diff --git a/include/simdjson/inline/document_stream.h b/include/simdjson/inline/document_stream.h index 3e72305b8..ccd4d9e31 100644 --- a/include/simdjson/inline/document_stream.h +++ b/include/simdjson/inline/document_stream.h @@ -66,11 +66,11 @@ inline void stage1_worker::run(document_stream * ds, dom::parser * stage1, size_ really_inline document_stream::document_stream( dom::parser &_parser, - size_t _batch_size, const uint8_t *_buf, - size_t _len + size_t _len, + size_t _batch_size ) noexcept - : parser{_parser}, + : parser{&_parser}, buf{_buf}, len{_len}, batch_size{_batch_size}, @@ -83,21 +83,15 @@ really_inline document_stream::document_stream( #endif } -really_inline document_stream::document_stream( - dom::parser &_parser, - size_t _batch_size, - error_code _error -) noexcept - : parser{_parser}, +really_inline document_stream::document_stream() noexcept + : parser{nullptr}, buf{nullptr}, len{0}, - batch_size{_batch_size}, - error{_error} -{ - assert(_error); + batch_size{0}, + error{UNINITIALIZED} { } -inline document_stream::~document_stream() noexcept { +really_inline document_stream::~document_stream() noexcept { } really_inline document_stream::iterator document_stream::begin() noexcept { @@ -117,7 +111,7 @@ really_inline document_stream::iterator::iterator(document_stream& _stream, bool really_inline simdjson_result document_stream::iterator::operator*() noexcept { // Once we have yielded any errors, we're finished. if (stream.error) { finished = true; return stream.error; } - return stream.parser.doc.root(); + return stream.parser->doc.root(); } really_inline document_stream::iterator& document_stream::iterator::operator++() noexcept { @@ -134,12 +128,12 @@ really_inline bool document_stream::iterator::operator!=(const document_stream:: inline void document_stream::start() noexcept { if (error) { return; } - error = parser.ensure_capacity(batch_size); + error = parser->ensure_capacity(batch_size); if (error) { return; } // Always run the first stage 1 parse immediately batch_start = 0; - error = run_stage1(parser, batch_start); + error = run_stage1(*parser, batch_start); if (error) { return; } #ifdef SIMDJSON_THREADS_ENABLED @@ -163,8 +157,8 @@ inline void document_stream::next() noexcept { if (error) { return; } // Load the next document from the batch - doc_index = batch_start + parser.implementation->structural_indexes[parser.implementation->next_structural_index]; - error = parser.implementation->stage2_next(parser.doc); + doc_index = batch_start + parser->implementation->structural_indexes[parser->implementation->next_structural_index]; + error = parser->implementation->stage2_next(parser->doc); // If that was the last document in the batch, load another batch (if available) while (error == EMPTY) { batch_start = next_batch_start(); @@ -173,17 +167,17 @@ inline void document_stream::next() noexcept { #ifdef SIMDJSON_THREADS_ENABLED load_from_stage1_thread(); #else - error = run_stage1(parser, batch_start); + error = run_stage1(*parser, batch_start); #endif if (error) { continue; } // If the error was EMPTY, we may want to load another batch. // Run stage 2 on the first document in the batch - doc_index = batch_start + parser.implementation->structural_indexes[parser.implementation->next_structural_index]; - error = parser.implementation->stage2_next(parser.doc); + doc_index = batch_start + parser->implementation->structural_indexes[parser->implementation->next_structural_index]; + error = parser->implementation->stage2_next(parser->doc); } } inline size_t document_stream::next_batch_start() const noexcept { - return batch_start + parser.implementation->structural_indexes[parser.implementation->n_structural_indexes]; + return batch_start + parser->implementation->structural_indexes[parser->implementation->n_structural_indexes]; } inline error_code document_stream::run_stage1(dom::parser &p, size_t _batch_start) noexcept { @@ -202,7 +196,7 @@ inline void document_stream::load_from_stage1_thread() noexcept { worker->finish(); // Swap to the parser that was loaded up in the thread. Make sure the parser has // enough memory to swap to, as well. - std::swap(parser, stage1_thread_parser); + std::swap(*parser, stage1_thread_parser); error = stage1_thread_error; if (error) { return; } @@ -226,5 +220,36 @@ inline void document_stream::start_stage1_thread() noexcept { #endif // SIMDJSON_THREADS_ENABLED } // namespace dom + +really_inline simdjson_result::simdjson_result() noexcept + : simdjson_result_base() { +} +really_inline simdjson_result::simdjson_result(error_code error) noexcept + : simdjson_result_base(error) { +} +really_inline simdjson_result::simdjson_result(dom::document_stream &&value) noexcept + : simdjson_result_base(std::forward(value)) { +} + +#if SIMDJSON_EXCEPTIONS +really_inline dom::document_stream::iterator simdjson_result::begin() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first.begin(); +} +really_inline dom::document_stream::iterator simdjson_result::end() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first.end(); +} +#else // SIMDJSON_EXCEPTIONS +really_inline dom::document_stream::iterator simdjson_result::begin() noexcept { + first.error = error(); + return first.begin(); +} +really_inline dom::document_stream::iterator simdjson_result::end() noexcept { + first.error = error(); + return first.end(); +} +#endif // SIMDJSON_EXCEPTIONS + } // namespace simdjson #endif // SIMDJSON_INLINE_DOCUMENT_STREAM_H diff --git a/include/simdjson/inline/element.h b/include/simdjson/inline/element.h index 690f40ff6..fc983eedb 100644 --- a/include/simdjson/inline/element.h +++ b/include/simdjson/inline/element.h @@ -24,9 +24,8 @@ inline simdjson_result simdjson_result::type() } template -really_inline simdjson_result simdjson_result::is() const noexcept { - if (error()) { return error(); } - return first.is(); +really_inline bool simdjson_result::is() const noexcept { + return !error() && first.is(); } template really_inline simdjson_result simdjson_result::get() const noexcept { @@ -72,38 +71,30 @@ really_inline simdjson_result simdjson_result::get_bool() co return first.get_bool(); } -really_inline simdjson_result simdjson_result::is_array() const noexcept { - if (error()) { return error(); } - return first.is_array(); +really_inline bool simdjson_result::is_array() const noexcept { + return !error() && first.is_array(); } -really_inline simdjson_result simdjson_result::is_object() const noexcept { - if (error()) { return error(); } - return first.is_object(); +really_inline bool simdjson_result::is_object() const noexcept { + return !error() && first.is_object(); } -really_inline simdjson_result simdjson_result::is_string() const noexcept { - if (error()) { return error(); } - return first.is_string(); +really_inline bool simdjson_result::is_string() const noexcept { + return !error() && first.is_string(); } -really_inline simdjson_result simdjson_result::is_int64_t() const noexcept { - if (error()) { return error(); } - return first.is_int64_t(); +really_inline bool simdjson_result::is_int64_t() const noexcept { + return !error() && first.is_int64_t(); } -really_inline simdjson_result simdjson_result::is_uint64_t() const noexcept { - if (error()) { return error(); } - return first.is_uint64_t(); +really_inline bool simdjson_result::is_uint64_t() const noexcept { + return !error() && first.is_uint64_t(); } -really_inline simdjson_result simdjson_result::is_double() const noexcept { - if (error()) { return error(); } - return first.is_double(); +really_inline bool simdjson_result::is_double() const noexcept { + return !error() && first.is_double(); } -really_inline simdjson_result simdjson_result::is_bool() const noexcept { - if (error()) { return error(); } - return first.is_bool(); +really_inline bool simdjson_result::is_bool() const noexcept { + return !error() && first.is_bool(); } -really_inline simdjson_result simdjson_result::is_null() const noexcept { - if (error()) { return error(); } - return first.is_null(); +really_inline bool simdjson_result::is_null() const noexcept { + return !error() && first.is_null(); } really_inline simdjson_result simdjson_result::operator[](const std::string_view &key) const noexcept { diff --git a/include/simdjson/inline/parser.h b/include/simdjson/inline/parser.h index 16bd47c62..fdfe49e6b 100644 --- a/include/simdjson/inline/parser.h +++ b/include/simdjson/inline/parser.h @@ -80,17 +80,14 @@ inline simdjson_result parser::load(const std::string &path) & noexcept size_t len; auto _error = read_file(path).get(len); if (_error) { return _error; } - return parse(loaded_bytes.get(), len, false); } -inline document_stream parser::load_many(const std::string &path, size_t batch_size) noexcept { +inline simdjson_result parser::load_many(const std::string &path, size_t batch_size) noexcept { size_t len; auto _error = read_file(path).get(len); - if (_error) { - return document_stream(*this, batch_size, _error); - } - return document_stream(*this, batch_size, (const uint8_t*)loaded_bytes.get(), len); + if (_error) { return _error; } + return document_stream(*this, (const uint8_t*)loaded_bytes.get(), len, batch_size); } inline simdjson_result parser::parse(const uint8_t *buf, size_t len, bool realloc_if_needed) & noexcept { @@ -123,16 +120,16 @@ really_inline simdjson_result parser::parse(const padded_string &s) & n return parse(s.data(), s.length(), false); } -inline document_stream parser::parse_many(const uint8_t *buf, size_t len, size_t batch_size) noexcept { - return document_stream(*this, batch_size, buf, len); +inline simdjson_result parser::parse_many(const uint8_t *buf, size_t len, size_t batch_size) noexcept { + return document_stream(*this, buf, len, batch_size); } -inline document_stream parser::parse_many(const char *buf, size_t len, size_t batch_size) noexcept { +inline simdjson_result parser::parse_many(const char *buf, size_t len, size_t batch_size) noexcept { return parse_many((const uint8_t *)buf, len, batch_size); } -inline document_stream parser::parse_many(const std::string &s, size_t batch_size) noexcept { +inline simdjson_result parser::parse_many(const std::string &s, size_t batch_size) noexcept { return parse_many(s.data(), s.length(), batch_size); } -inline document_stream parser::parse_many(const padded_string &s, size_t batch_size) noexcept { +inline simdjson_result parser::parse_many(const padded_string &s, size_t batch_size) noexcept { return parse_many(s.data(), s.length(), batch_size); } diff --git a/singleheader/amalgamate.sh b/singleheader/amalgamate.sh index d6213ba0e..901cd1467 100755 --- a/singleheader/amalgamate.sh +++ b/singleheader/amalgamate.sh @@ -135,9 +135,8 @@ int main(int argc, char *argv[]) { } const char * filename = argv[1]; simdjson::dom::parser parser; - simdjson::error_code error; UNUSED simdjson::dom::element elem; - parser.load(filename).tie(elem, error); // do the parsing + auto error = parser.load(filename).get(elem); // do the parsing if (error) { std::cout << "parse failed" << std::endl; std::cout << "error code: " << error << std::endl; @@ -152,8 +151,12 @@ int main(int argc, char *argv[]) { // parse_many const char * filename2 = argv[2]; - for (auto result : parser.load_many(filename2)) { - error = result.error(); + simdjson::dom::document_stream stream; + error = parser.load_many(filename2).get(stream); + if (!error) { + for (auto result : stream) { + error = result.error(); + } } if (error) { std::cout << "parse_many failed" << std::endl; diff --git a/singleheader/amalgamate_demo.cpp b/singleheader/amalgamate_demo.cpp index a6ab5cf4d..3dfbddcf5 100644 --- a/singleheader/amalgamate_demo.cpp +++ b/singleheader/amalgamate_demo.cpp @@ -1,4 +1,4 @@ -/* auto-generated on Fri 12 Jun 2020 13:09:36 EDT. Do not edit! */ +/* auto-generated on Sun Jun 21 11:49:12 PDT 2020. Do not edit! */ #include #include "simdjson.h" @@ -9,9 +9,8 @@ int main(int argc, char *argv[]) { } const char * filename = argv[1]; simdjson::dom::parser parser; - simdjson::error_code error; UNUSED simdjson::dom::element elem; - parser.load(filename).tie(elem, error); // do the parsing + auto error = parser.load(filename).get(elem); // do the parsing if (error) { std::cout << "parse failed" << std::endl; std::cout << "error code: " << error << std::endl; @@ -26,8 +25,12 @@ int main(int argc, char *argv[]) { // parse_many const char * filename2 = argv[2]; - for (auto result : parser.load_many(filename2)) { - error = result.error(); + simdjson::dom::document_stream stream; + error = parser.load_many(filename2).get(stream); + if (!error) { + for (auto result : stream) { + error = result.error(); + } } if (error) { std::cout << "parse_many failed" << std::endl; diff --git a/singleheader/simdjson.cpp b/singleheader/simdjson.cpp index d99dc8b67..d78ad5df6 100644 --- a/singleheader/simdjson.cpp +++ b/singleheader/simdjson.cpp @@ -1,4 +1,4 @@ -/* auto-generated on Fri 12 Jun 2020 13:09:36 EDT. Do not edit! */ +/* auto-generated on Sun Jun 21 11:49:12 PDT 2020. Do not edit! */ /* begin file src/simdjson.cpp */ #include "simdjson.h" @@ -586,6 +586,11 @@ const implementation *detect_best_supported_implementation_on_first_use::set_bes SIMDJSON_DLLIMPORTEXPORT const internal::available_implementation_list available_implementations{}; SIMDJSON_DLLIMPORTEXPORT internal::atomic_ptr active_implementation{&internal::detect_best_supported_implementation_on_first_use_singleton}; +WARN_UNUSED error_code minify(const char *buf, size_t len, char *dst, size_t &dst_len) noexcept { + return active_implementation->minify((const uint8_t *)buf, len, (uint8_t *)dst, dst_len); +} + + } // namespace simdjson /* end file src/fallback/implementation.h */ @@ -2794,6 +2799,12 @@ really_inline simd8 must_be_continuation(simd8 prev1, simd8 must_be_2_3_continuation(simd8 prev2, simd8 prev3) { + simd8 is_third_byte = prev2 >= uint8_t(0b11100000u); + simd8 is_fourth_byte = prev3 >= uint8_t(0b11110000u); + return is_third_byte ^ is_fourth_byte; +} + /* begin file src/generic/stage1/buf_block_reader.h */ // Walks through a buffer in block-sized increments, loading the last part with spaces template @@ -2921,7 +2932,9 @@ public: really_inline error_code finish(bool streaming); private: + // Intended to be defined by the implementation really_inline uint64_t find_escaped(uint64_t escape); + really_inline uint64_t find_escaped_branchless(uint64_t escape); // Whether the last iteration was still inside a string (all 1's = true, all 0's = false). uint64_t prev_in_string = 0ULL; @@ -2956,7 +2969,7 @@ private: // desired | x | x x x x x x x x | // text | \\\ | \\\"\\\" \\\" \\"\\" | // -really_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { +really_inline uint64_t json_string_scanner::find_escaped_branchless(uint64_t backslash) { // If there was overflow, pretend the first character isn't a backslash backslash &= ~prev_escaped; uint64_t follows_escape = backslash << 1 | prev_escaped; @@ -2985,13 +2998,23 @@ really_inline json_string_block json_string_scanner::next(const simd::simd8x64(in_string) >> 63); + // Use ^ to turn the beginning quote off, and the end quote on. return { backslash, @@ -3117,6 +3140,15 @@ really_inline error_code json_scanner::finish(bool streaming) { } // namespace stage1 /* end file src/generic/stage1/json_scanner.h */ +namespace stage1 { +really_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { + // On ARM, we don't short-circuit this if there are no backslashes, because the branch gives us no + // benefit and therefore makes things worse. + // if (!backslash) { uint64_t escaped = prev_escaped; prev_escaped = 0; return escaped; } + return find_escaped_branchless(backslash); +} +} + /* begin file src/generic/stage1/json_minifier.h */ // This file contains the common code every implementation uses in stage1 // It is intended to be included multiple times and compiled multiple times @@ -3288,7 +3320,7 @@ really_inline static size_t trim_partial_utf8(const uint8_t *buf, size_t len) { return len; } /* end file src/generic/stage1/find_next_document_index.h */ -/* begin file src/generic/stage1/utf8_lookup2_algorithm.h */ +/* begin file src/generic/stage1/utf8_lookup3_algorithm.h */ // // Detect Unicode errors. // @@ -3380,67 +3412,79 @@ namespace utf8_validation { static const int TOO_LARGE = 0x10; // 11110100 (1001|101_)____ static const int TOO_LARGE_2 = 0x20; // 1111(1___|011_|0101) 10______ + // New with lookup3. We want to catch the case where an non-continuation + // follows a leading byte + static const int TOO_SHORT_2_3_4 = 0x40; // (110_|1110|1111) ____ (0___|110_|1111) ____ + // We also want to catch a continuation that is preceded by an ASCII byte + static const int LONELY_CONTINUATION = 0x80; // 0___ ____ 01__ ____ + // After processing the rest of byte 1 (the low bits), we're still not done--we have to check // byte 2 to be sure which things are errors and which aren't. // Since high_bits is byte 5, byte 2 is high_bits.prev<3> static const int CARRY = OVERLONG_2 | TOO_LARGE_2; const simd8 byte_2_high = input.shr<4>().lookup_16( // ASCII: ________ [0___]____ - CARRY, CARRY, CARRY, CARRY, + CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4, + CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4, // ASCII: ________ [0___]____ - CARRY, CARRY, CARRY, CARRY, + CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4, + CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4, // Continuations: ________ [10__]____ - CARRY | OVERLONG_3 | OVERLONG_4, // ________ [1000]____ - CARRY | OVERLONG_3 | TOO_LARGE, // ________ [1001]____ - CARRY | TOO_LARGE | SURROGATE, // ________ [1010]____ - CARRY | TOO_LARGE | SURROGATE, // ________ [1011]____ + CARRY | OVERLONG_3 | OVERLONG_4 | LONELY_CONTINUATION, // ________ [1000]____ + CARRY | OVERLONG_3 | TOO_LARGE | LONELY_CONTINUATION, // ________ [1001]____ + CARRY | TOO_LARGE | SURROGATE | LONELY_CONTINUATION, // ________ [1010]____ + CARRY | TOO_LARGE | SURROGATE | LONELY_CONTINUATION, // ________ [1011]____ // Multibyte Leads: ________ [11__]____ - CARRY, CARRY, CARRY, CARRY + CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4, // 110_ + CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4 ); - const simd8 byte_1_high = prev1.shr<4>().lookup_16( // [0___]____ (ASCII) - 0, 0, 0, 0, - 0, 0, 0, 0, + LONELY_CONTINUATION, LONELY_CONTINUATION, LONELY_CONTINUATION, LONELY_CONTINUATION, + LONELY_CONTINUATION, LONELY_CONTINUATION, LONELY_CONTINUATION, LONELY_CONTINUATION, // [10__]____ (continuation) 0, 0, 0, 0, // [11__]____ (2+-byte leads) - OVERLONG_2, 0, // [110_]____ (2-byte lead) - OVERLONG_3 | SURROGATE, // [1110]____ (3-byte lead) - OVERLONG_4 | TOO_LARGE | TOO_LARGE_2 // [1111]____ (4+-byte lead) + OVERLONG_2 | TOO_SHORT_2_3_4, TOO_SHORT_2_3_4, // [110_]____ (2-byte lead) + OVERLONG_3 | SURROGATE | TOO_SHORT_2_3_4, // [1110]____ (3-byte lead) + OVERLONG_4 | TOO_LARGE | TOO_LARGE_2 | TOO_SHORT_2_3_4 // [1111]____ (4+-byte lead) ); - const simd8 byte_1_low = (prev1 & 0x0F).lookup_16( // ____[00__] ________ - OVERLONG_2 | OVERLONG_3 | OVERLONG_4, // ____[0000] ________ - OVERLONG_2, // ____[0001] ________ - 0, 0, + OVERLONG_2 | OVERLONG_3 | OVERLONG_4 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[0000] ________ + OVERLONG_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[0001] ________ + TOO_SHORT_2_3_4 | LONELY_CONTINUATION, + TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[01__] ________ - TOO_LARGE, // ____[0100] ________ - TOO_LARGE_2, - TOO_LARGE_2, - TOO_LARGE_2, + TOO_LARGE | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[0100] ________ + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[10__] ________ - TOO_LARGE_2, TOO_LARGE_2, TOO_LARGE_2, TOO_LARGE_2, + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[11__] ________ - TOO_LARGE_2, - TOO_LARGE_2 | SURROGATE, // ____[1101] ________ - TOO_LARGE_2, TOO_LARGE_2 + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, + TOO_LARGE_2 | SURROGATE | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[1101] ________ + TOO_LARGE_2 | TOO_SHORT_2_3_4| LONELY_CONTINUATION, + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION ); - return byte_1_high & byte_1_low & byte_2_high; } - really_inline simd8 check_multibyte_lengths(simd8 input, simd8 prev_input, simd8 prev1) { + really_inline simd8 check_multibyte_lengths(simd8 input, simd8 prev_input, + simd8 prev1) { simd8 prev2 = input.prev<2>(prev_input); simd8 prev3 = input.prev<3>(prev_input); - - // Cont is 10000000-101111111 (-65...-128) - simd8 is_continuation = simd8(input) < int8_t(-64); - // must_be_continuation is architecture-specific because Intel doesn't have unsigned comparisons - return simd8(must_be_continuation(prev1, prev2, prev3) ^ is_continuation); + // is_2_3_continuation uses one more instruction than lookup2 + simd8 is_2_3_continuation = (simd8(input).max(simd8(prev1))) < int8_t(-64); + // must_be_2_3_continuation has two fewer instructions than lookup 2 + return simd8(must_be_2_3_continuation(prev2, prev3) ^ is_2_3_continuation); } + // // Return nonzero if there are incomplete multibyte characters at the end of the block: // e.g. if there is a 4-byte character, but it's 3 bytes from the end. @@ -3507,7 +3551,7 @@ namespace utf8_validation { } using utf8_validation::utf8_checker; -/* end file src/generic/stage1/utf8_lookup2_algorithm.h */ +/* end file src/generic/stage1/utf8_lookup3_algorithm.h */ /* begin file src/generic/stage1/json_structural_indexer.h */ // This file contains the common code every implementation uses in stage1 // It is intended to be included multiple times and compiled multiple times @@ -4432,7 +4476,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const src, } // we over-decrement by one when there is a '.' digit_count -= int(start - start_digits); - if (unlikely(digit_count >= 19)) { + if (digit_count >= 19) { // Ok, chances are good that we had an overflow! // this is almost never going to get called!!! // we start anew, going slowly!!! @@ -4442,7 +4486,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const src, // bool success = slow_float_parsing((const char *) src, writer); // The number was already written, but we made a copy of the writer - // when we passed it to the parse_large_integer() function, so + // when we passed it to the parse_large_integer() function, so writer.skip_double(); return success; } @@ -4481,7 +4525,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const src, // need to recover: we parse the whole thing again. bool success = parse_large_integer(src, writer, found_minus); // The number was already written, but we made a copy of the writer - // when we passed it to the parse_large_integer() function, so + // when we passed it to the parse_large_integer() function, so writer.skip_large_integer(); return success; } @@ -6525,7 +6569,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const src, } // we over-decrement by one when there is a '.' digit_count -= int(start - start_digits); - if (unlikely(digit_count >= 19)) { + if (digit_count >= 19) { // Ok, chances are good that we had an overflow! // this is almost never going to get called!!! // we start anew, going slowly!!! @@ -6535,7 +6579,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const src, // bool success = slow_float_parsing((const char *) src, writer); // The number was already written, but we made a copy of the writer - // when we passed it to the parse_large_integer() function, so + // when we passed it to the parse_large_integer() function, so writer.skip_double(); return success; } @@ -6574,7 +6618,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const src, // need to recover: we parse the whole thing again. bool success = parse_large_integer(src, writer, found_minus); // The number was already written, but we made a copy of the writer - // when we passed it to the parse_large_integer() function, so + // when we passed it to the parse_large_integer() function, so writer.skip_large_integer(); return success; } @@ -8119,6 +8163,14 @@ really_inline simd8 must_be_continuation(simd8 prev1, simd8(is_second_byte | is_third_byte | is_fourth_byte) > int8_t(0); } +really_inline simd8 must_be_2_3_continuation(simd8 prev2, simd8 prev3) { + simd8 is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0 + simd8 is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0 + // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine. + return simd8(is_third_byte | is_fourth_byte) > int8_t(0); +} + + /* begin file src/generic/stage1/buf_block_reader.h */ // Walks through a buffer in block-sized increments, loading the last part with spaces template @@ -8246,7 +8298,9 @@ public: really_inline error_code finish(bool streaming); private: + // Intended to be defined by the implementation really_inline uint64_t find_escaped(uint64_t escape); + really_inline uint64_t find_escaped_branchless(uint64_t escape); // Whether the last iteration was still inside a string (all 1's = true, all 0's = false). uint64_t prev_in_string = 0ULL; @@ -8281,7 +8335,7 @@ private: // desired | x | x x x x x x x x | // text | \\\ | \\\"\\\" \\\" \\"\\" | // -really_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { +really_inline uint64_t json_string_scanner::find_escaped_branchless(uint64_t backslash) { // If there was overflow, pretend the first character isn't a backslash backslash &= ~prev_escaped; uint64_t follows_escape = backslash << 1 | prev_escaped; @@ -8310,13 +8364,23 @@ really_inline json_string_block json_string_scanner::next(const simd::simd8x64(in_string) >> 63); + // Use ^ to turn the beginning quote off, and the end quote on. return { backslash, @@ -8442,6 +8506,13 @@ really_inline error_code json_scanner::finish(bool streaming) { } // namespace stage1 /* end file src/generic/stage1/json_scanner.h */ +namespace stage1 { +really_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { + if (!backslash) { uint64_t escaped = prev_escaped; prev_escaped = 0; return escaped; } + return find_escaped_branchless(backslash); +} +} + /* begin file src/generic/stage1/json_minifier.h */ // This file contains the common code every implementation uses in stage1 // It is intended to be included multiple times and compiled multiple times @@ -8613,7 +8684,7 @@ really_inline static size_t trim_partial_utf8(const uint8_t *buf, size_t len) { return len; } /* end file src/generic/stage1/find_next_document_index.h */ -/* begin file src/generic/stage1/utf8_lookup2_algorithm.h */ +/* begin file src/generic/stage1/utf8_lookup3_algorithm.h */ // // Detect Unicode errors. // @@ -8705,67 +8776,79 @@ namespace utf8_validation { static const int TOO_LARGE = 0x10; // 11110100 (1001|101_)____ static const int TOO_LARGE_2 = 0x20; // 1111(1___|011_|0101) 10______ + // New with lookup3. We want to catch the case where an non-continuation + // follows a leading byte + static const int TOO_SHORT_2_3_4 = 0x40; // (110_|1110|1111) ____ (0___|110_|1111) ____ + // We also want to catch a continuation that is preceded by an ASCII byte + static const int LONELY_CONTINUATION = 0x80; // 0___ ____ 01__ ____ + // After processing the rest of byte 1 (the low bits), we're still not done--we have to check // byte 2 to be sure which things are errors and which aren't. // Since high_bits is byte 5, byte 2 is high_bits.prev<3> static const int CARRY = OVERLONG_2 | TOO_LARGE_2; const simd8 byte_2_high = input.shr<4>().lookup_16( // ASCII: ________ [0___]____ - CARRY, CARRY, CARRY, CARRY, + CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4, + CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4, // ASCII: ________ [0___]____ - CARRY, CARRY, CARRY, CARRY, + CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4, + CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4, // Continuations: ________ [10__]____ - CARRY | OVERLONG_3 | OVERLONG_4, // ________ [1000]____ - CARRY | OVERLONG_3 | TOO_LARGE, // ________ [1001]____ - CARRY | TOO_LARGE | SURROGATE, // ________ [1010]____ - CARRY | TOO_LARGE | SURROGATE, // ________ [1011]____ + CARRY | OVERLONG_3 | OVERLONG_4 | LONELY_CONTINUATION, // ________ [1000]____ + CARRY | OVERLONG_3 | TOO_LARGE | LONELY_CONTINUATION, // ________ [1001]____ + CARRY | TOO_LARGE | SURROGATE | LONELY_CONTINUATION, // ________ [1010]____ + CARRY | TOO_LARGE | SURROGATE | LONELY_CONTINUATION, // ________ [1011]____ // Multibyte Leads: ________ [11__]____ - CARRY, CARRY, CARRY, CARRY + CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4, // 110_ + CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4 ); - const simd8 byte_1_high = prev1.shr<4>().lookup_16( // [0___]____ (ASCII) - 0, 0, 0, 0, - 0, 0, 0, 0, + LONELY_CONTINUATION, LONELY_CONTINUATION, LONELY_CONTINUATION, LONELY_CONTINUATION, + LONELY_CONTINUATION, LONELY_CONTINUATION, LONELY_CONTINUATION, LONELY_CONTINUATION, // [10__]____ (continuation) 0, 0, 0, 0, // [11__]____ (2+-byte leads) - OVERLONG_2, 0, // [110_]____ (2-byte lead) - OVERLONG_3 | SURROGATE, // [1110]____ (3-byte lead) - OVERLONG_4 | TOO_LARGE | TOO_LARGE_2 // [1111]____ (4+-byte lead) + OVERLONG_2 | TOO_SHORT_2_3_4, TOO_SHORT_2_3_4, // [110_]____ (2-byte lead) + OVERLONG_3 | SURROGATE | TOO_SHORT_2_3_4, // [1110]____ (3-byte lead) + OVERLONG_4 | TOO_LARGE | TOO_LARGE_2 | TOO_SHORT_2_3_4 // [1111]____ (4+-byte lead) ); - const simd8 byte_1_low = (prev1 & 0x0F).lookup_16( // ____[00__] ________ - OVERLONG_2 | OVERLONG_3 | OVERLONG_4, // ____[0000] ________ - OVERLONG_2, // ____[0001] ________ - 0, 0, + OVERLONG_2 | OVERLONG_3 | OVERLONG_4 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[0000] ________ + OVERLONG_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[0001] ________ + TOO_SHORT_2_3_4 | LONELY_CONTINUATION, + TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[01__] ________ - TOO_LARGE, // ____[0100] ________ - TOO_LARGE_2, - TOO_LARGE_2, - TOO_LARGE_2, + TOO_LARGE | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[0100] ________ + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[10__] ________ - TOO_LARGE_2, TOO_LARGE_2, TOO_LARGE_2, TOO_LARGE_2, + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[11__] ________ - TOO_LARGE_2, - TOO_LARGE_2 | SURROGATE, // ____[1101] ________ - TOO_LARGE_2, TOO_LARGE_2 + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, + TOO_LARGE_2 | SURROGATE | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[1101] ________ + TOO_LARGE_2 | TOO_SHORT_2_3_4| LONELY_CONTINUATION, + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION ); - return byte_1_high & byte_1_low & byte_2_high; } - really_inline simd8 check_multibyte_lengths(simd8 input, simd8 prev_input, simd8 prev1) { + really_inline simd8 check_multibyte_lengths(simd8 input, simd8 prev_input, + simd8 prev1) { simd8 prev2 = input.prev<2>(prev_input); simd8 prev3 = input.prev<3>(prev_input); - - // Cont is 10000000-101111111 (-65...-128) - simd8 is_continuation = simd8(input) < int8_t(-64); - // must_be_continuation is architecture-specific because Intel doesn't have unsigned comparisons - return simd8(must_be_continuation(prev1, prev2, prev3) ^ is_continuation); + // is_2_3_continuation uses one more instruction than lookup2 + simd8 is_2_3_continuation = (simd8(input).max(simd8(prev1))) < int8_t(-64); + // must_be_2_3_continuation has two fewer instructions than lookup 2 + return simd8(must_be_2_3_continuation(prev2, prev3) ^ is_2_3_continuation); } + // // Return nonzero if there are incomplete multibyte characters at the end of the block: // e.g. if there is a 4-byte character, but it's 3 bytes from the end. @@ -8832,7 +8915,7 @@ namespace utf8_validation { } using utf8_validation::utf8_checker; -/* end file src/generic/stage1/utf8_lookup2_algorithm.h */ +/* end file src/generic/stage1/utf8_lookup3_algorithm.h */ /* begin file src/generic/stage1/json_structural_indexer.h */ // This file contains the common code every implementation uses in stage1 // It is intended to be included multiple times and compiled multiple times @@ -9762,7 +9845,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const src, } // we over-decrement by one when there is a '.' digit_count -= int(start - start_digits); - if (unlikely(digit_count >= 19)) { + if (digit_count >= 19) { // Ok, chances are good that we had an overflow! // this is almost never going to get called!!! // we start anew, going slowly!!! @@ -9772,7 +9855,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const src, // bool success = slow_float_parsing((const char *) src, writer); // The number was already written, but we made a copy of the writer - // when we passed it to the parse_large_integer() function, so + // when we passed it to the parse_large_integer() function, so writer.skip_double(); return success; } @@ -9811,7 +9894,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const src, // need to recover: we parse the whole thing again. bool success = parse_large_integer(src, writer, found_minus); // The number was already written, but we made a copy of the writer - // when we passed it to the parse_large_integer() function, so + // when we passed it to the parse_large_integer() function, so writer.skip_large_integer(); return success; } @@ -11327,6 +11410,14 @@ really_inline simd8 must_be_continuation(simd8 prev1, simd8(is_second_byte | is_third_byte | is_fourth_byte) > int8_t(0); } +really_inline simd8 must_be_2_3_continuation(simd8 prev2, simd8 prev3) { + simd8 is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0 + simd8 is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0 + // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine. + return simd8(is_third_byte | is_fourth_byte) > int8_t(0); +} + + /* begin file src/generic/stage1/buf_block_reader.h */ // Walks through a buffer in block-sized increments, loading the last part with spaces template @@ -11454,7 +11545,9 @@ public: really_inline error_code finish(bool streaming); private: + // Intended to be defined by the implementation really_inline uint64_t find_escaped(uint64_t escape); + really_inline uint64_t find_escaped_branchless(uint64_t escape); // Whether the last iteration was still inside a string (all 1's = true, all 0's = false). uint64_t prev_in_string = 0ULL; @@ -11489,7 +11582,7 @@ private: // desired | x | x x x x x x x x | // text | \\\ | \\\"\\\" \\\" \\"\\" | // -really_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { +really_inline uint64_t json_string_scanner::find_escaped_branchless(uint64_t backslash) { // If there was overflow, pretend the first character isn't a backslash backslash &= ~prev_escaped; uint64_t follows_escape = backslash << 1 | prev_escaped; @@ -11518,13 +11611,23 @@ really_inline json_string_block json_string_scanner::next(const simd::simd8x64(in_string) >> 63); + // Use ^ to turn the beginning quote off, and the end quote on. return { backslash, @@ -11650,6 +11753,13 @@ really_inline error_code json_scanner::finish(bool streaming) { } // namespace stage1 /* end file src/generic/stage1/json_scanner.h */ +namespace stage1 { +really_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { + if (!backslash) { uint64_t escaped = prev_escaped; prev_escaped = 0; return escaped; } + return find_escaped_branchless(backslash); +} +} + /* begin file src/generic/stage1/json_minifier.h */ // This file contains the common code every implementation uses in stage1 // It is intended to be included multiple times and compiled multiple times @@ -11821,7 +11931,7 @@ really_inline static size_t trim_partial_utf8(const uint8_t *buf, size_t len) { return len; } /* end file src/generic/stage1/find_next_document_index.h */ -/* begin file src/generic/stage1/utf8_lookup2_algorithm.h */ +/* begin file src/generic/stage1/utf8_lookup3_algorithm.h */ // // Detect Unicode errors. // @@ -11913,67 +12023,79 @@ namespace utf8_validation { static const int TOO_LARGE = 0x10; // 11110100 (1001|101_)____ static const int TOO_LARGE_2 = 0x20; // 1111(1___|011_|0101) 10______ + // New with lookup3. We want to catch the case where an non-continuation + // follows a leading byte + static const int TOO_SHORT_2_3_4 = 0x40; // (110_|1110|1111) ____ (0___|110_|1111) ____ + // We also want to catch a continuation that is preceded by an ASCII byte + static const int LONELY_CONTINUATION = 0x80; // 0___ ____ 01__ ____ + // After processing the rest of byte 1 (the low bits), we're still not done--we have to check // byte 2 to be sure which things are errors and which aren't. // Since high_bits is byte 5, byte 2 is high_bits.prev<3> static const int CARRY = OVERLONG_2 | TOO_LARGE_2; const simd8 byte_2_high = input.shr<4>().lookup_16( // ASCII: ________ [0___]____ - CARRY, CARRY, CARRY, CARRY, + CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4, + CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4, // ASCII: ________ [0___]____ - CARRY, CARRY, CARRY, CARRY, + CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4, + CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4, // Continuations: ________ [10__]____ - CARRY | OVERLONG_3 | OVERLONG_4, // ________ [1000]____ - CARRY | OVERLONG_3 | TOO_LARGE, // ________ [1001]____ - CARRY | TOO_LARGE | SURROGATE, // ________ [1010]____ - CARRY | TOO_LARGE | SURROGATE, // ________ [1011]____ + CARRY | OVERLONG_3 | OVERLONG_4 | LONELY_CONTINUATION, // ________ [1000]____ + CARRY | OVERLONG_3 | TOO_LARGE | LONELY_CONTINUATION, // ________ [1001]____ + CARRY | TOO_LARGE | SURROGATE | LONELY_CONTINUATION, // ________ [1010]____ + CARRY | TOO_LARGE | SURROGATE | LONELY_CONTINUATION, // ________ [1011]____ // Multibyte Leads: ________ [11__]____ - CARRY, CARRY, CARRY, CARRY + CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4, // 110_ + CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4 ); - const simd8 byte_1_high = prev1.shr<4>().lookup_16( // [0___]____ (ASCII) - 0, 0, 0, 0, - 0, 0, 0, 0, + LONELY_CONTINUATION, LONELY_CONTINUATION, LONELY_CONTINUATION, LONELY_CONTINUATION, + LONELY_CONTINUATION, LONELY_CONTINUATION, LONELY_CONTINUATION, LONELY_CONTINUATION, // [10__]____ (continuation) 0, 0, 0, 0, // [11__]____ (2+-byte leads) - OVERLONG_2, 0, // [110_]____ (2-byte lead) - OVERLONG_3 | SURROGATE, // [1110]____ (3-byte lead) - OVERLONG_4 | TOO_LARGE | TOO_LARGE_2 // [1111]____ (4+-byte lead) + OVERLONG_2 | TOO_SHORT_2_3_4, TOO_SHORT_2_3_4, // [110_]____ (2-byte lead) + OVERLONG_3 | SURROGATE | TOO_SHORT_2_3_4, // [1110]____ (3-byte lead) + OVERLONG_4 | TOO_LARGE | TOO_LARGE_2 | TOO_SHORT_2_3_4 // [1111]____ (4+-byte lead) ); - const simd8 byte_1_low = (prev1 & 0x0F).lookup_16( // ____[00__] ________ - OVERLONG_2 | OVERLONG_3 | OVERLONG_4, // ____[0000] ________ - OVERLONG_2, // ____[0001] ________ - 0, 0, + OVERLONG_2 | OVERLONG_3 | OVERLONG_4 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[0000] ________ + OVERLONG_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[0001] ________ + TOO_SHORT_2_3_4 | LONELY_CONTINUATION, + TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[01__] ________ - TOO_LARGE, // ____[0100] ________ - TOO_LARGE_2, - TOO_LARGE_2, - TOO_LARGE_2, + TOO_LARGE | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[0100] ________ + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[10__] ________ - TOO_LARGE_2, TOO_LARGE_2, TOO_LARGE_2, TOO_LARGE_2, + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[11__] ________ - TOO_LARGE_2, - TOO_LARGE_2 | SURROGATE, // ____[1101] ________ - TOO_LARGE_2, TOO_LARGE_2 + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, + TOO_LARGE_2 | SURROGATE | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[1101] ________ + TOO_LARGE_2 | TOO_SHORT_2_3_4| LONELY_CONTINUATION, + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION ); - return byte_1_high & byte_1_low & byte_2_high; } - really_inline simd8 check_multibyte_lengths(simd8 input, simd8 prev_input, simd8 prev1) { + really_inline simd8 check_multibyte_lengths(simd8 input, simd8 prev_input, + simd8 prev1) { simd8 prev2 = input.prev<2>(prev_input); simd8 prev3 = input.prev<3>(prev_input); - - // Cont is 10000000-101111111 (-65...-128) - simd8 is_continuation = simd8(input) < int8_t(-64); - // must_be_continuation is architecture-specific because Intel doesn't have unsigned comparisons - return simd8(must_be_continuation(prev1, prev2, prev3) ^ is_continuation); + // is_2_3_continuation uses one more instruction than lookup2 + simd8 is_2_3_continuation = (simd8(input).max(simd8(prev1))) < int8_t(-64); + // must_be_2_3_continuation has two fewer instructions than lookup 2 + return simd8(must_be_2_3_continuation(prev2, prev3) ^ is_2_3_continuation); } + // // Return nonzero if there are incomplete multibyte characters at the end of the block: // e.g. if there is a 4-byte character, but it's 3 bytes from the end. @@ -12040,7 +12162,7 @@ namespace utf8_validation { } using utf8_validation::utf8_checker; -/* end file src/generic/stage1/utf8_lookup2_algorithm.h */ +/* end file src/generic/stage1/utf8_lookup3_algorithm.h */ /* begin file src/generic/stage1/json_structural_indexer.h */ // This file contains the common code every implementation uses in stage1 // It is intended to be included multiple times and compiled multiple times @@ -12973,7 +13095,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const src, } // we over-decrement by one when there is a '.' digit_count -= int(start - start_digits); - if (unlikely(digit_count >= 19)) { + if (digit_count >= 19) { // Ok, chances are good that we had an overflow! // this is almost never going to get called!!! // we start anew, going slowly!!! @@ -12983,7 +13105,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const src, // bool success = slow_float_parsing((const char *) src, writer); // The number was already written, but we made a copy of the writer - // when we passed it to the parse_large_integer() function, so + // when we passed it to the parse_large_integer() function, so writer.skip_double(); return success; } @@ -13022,7 +13144,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const src, // need to recover: we parse the whole thing again. bool success = parse_large_integer(src, writer, found_minus); // The number was already written, but we made a copy of the writer - // when we passed it to the parse_large_integer() function, so + // when we passed it to the parse_large_integer() function, so writer.skip_large_integer(); return success; } diff --git a/singleheader/simdjson.h b/singleheader/simdjson.h index 21efa8e49..63a8f14ec 100644 --- a/singleheader/simdjson.h +++ b/singleheader/simdjson.h @@ -1,4 +1,4 @@ -/* auto-generated on Fri 12 Jun 2020 13:09:36 EDT. Do not edit! */ +/* auto-generated on Sun Jun 21 11:49:12 PDT 2020. Do not edit! */ /* begin file include/simdjson.h */ #ifndef SIMDJSON_H #define SIMDJSON_H @@ -169,12 +169,12 @@ compiling for a known 64-bit platform." #define TARGET_WESTMERE TARGET_REGION("sse4.2,pclmul") #define TARGET_ARM64 -// Threading is disabled -#undef SIMDJSON_THREADS_ENABLED // Is threading enabled? #if defined(BOOST_HAS_THREADS) || defined(_REENTRANT) || defined(_MT) +#ifndef SIMDJSON_THREADS_ENABLED #define SIMDJSON_THREADS_ENABLED #endif +#endif // workaround for large stack sizes under -O0. @@ -183,7 +183,9 @@ compiling for a known 64-bit platform." #ifndef __OPTIMIZE__ // Apple systems have small stack sizes in secondary threads. // Lack of compiler optimization may generate high stack usage. -// So we are disabling multithreaded support for safety. +// Users may want to disable threads for safety, but only when +// in debug mode which we detect by the fact that the __OPTIMIZE__ +// macro is not defined. #undef SIMDJSON_THREADS_ENABLED #endif #endif @@ -251,6 +253,25 @@ static inline void aligned_free(void *mem_block) { static inline void aligned_free_char(char *mem_block) { aligned_free((void *)mem_block); } + +#ifdef NDEBUG + +#ifdef SIMDJSON_VISUAL_STUDIO +#define SIMDJSON_UNREACHABLE() __assume(0) +#define SIMDJSON_ASSUME(COND) __assume(COND) +#else +#define SIMDJSON_UNREACHABLE() __builtin_unreachable(); +#define SIMDJSON_ASSUME(COND) do { if (!(COND)) __builtin_unreachable(); } while (0) +#endif + +#else // NDEBUG + +#include +#define SIMDJSON_UNREACHABLE() assert(0); +#define SIMDJSON_ASSUME(COND) assert(COND) + +#endif + } // namespace simdjson #endif // SIMDJSON_PORTABILITY_H /* end file include/simdjson/portability.h */ @@ -2138,9 +2159,19 @@ struct simdjson_result_base : public std::pair { /** * Move the value and the error to the provided variables. + * + * @param value The variable to assign the value to. May not be set if there is an error. + * @param error The variable to assign the error to. Set to SUCCESS if there is no error. */ really_inline void tie(T &value, error_code &error) && noexcept; + /** + * Move the value to the provided variable. + * + * @param value The variable to assign the value to. May not be set if there is an error. + */ + really_inline error_code get(T &value) && noexcept; + /** * The error. */ @@ -2200,8 +2231,18 @@ struct simdjson_result : public internal::simdjson_result_base { /** * Move the value and the error to the provided variables. + * + * @param value The variable to assign the value to. May not be set if there is an error. + * @param error The variable to assign the error to. Set to SUCCESS if there is no error. */ - really_inline void tie(T& t, error_code & e) && noexcept; + really_inline void tie(T &value, error_code &error) && noexcept; + + /** + * Move the value to the provided variable. + * + * @param value The variable to assign the value to. May not be set if there is an error. + */ + WARN_UNUSED really_inline error_code get(T &value) && noexcept; /** * The error. @@ -2658,11 +2699,11 @@ public: /** * @private For internal implementation use * - * Run a full document parse (ensure_capacity, stage1 and stage2). + * Minify the input string assuming that it represents a JSON string, does not parse or validate. * * Overridden by each implementation. * - * @param buf the json document to parse. *MUST* be allocated up to len + SIMDJSON_PADDING bytes. + * @param buf the json document to minify. * @param len the length of the json document. * @param dst the buffer to write the minified document to. *MUST* be allocated up to len + SIMDJSON_PADDING bytes. * @param dst_len the number of bytes written. Output only. @@ -2884,6 +2925,23 @@ public: namespace simdjson { + + +/** + * + * Minify the input string assuming that it represents a JSON string, does not parse or validate. + * This function is much faster than parsing a JSON string and then writing a minified version of it. + * However, it does not validate the input. + * + * + * @param buf the json document to minify. + * @param len the length of the json document. + * @param dst the buffer to write the minified document to. *MUST* be allocated up to len + SIMDJSON_PADDING bytes. + * @param dst_len the number of bytes written. Output only. + * @return the error code, or SUCCESS if there was no error. + */ +WARN_UNUSED error_code minify(const char *buf, size_t len, char *dst, size_t &dst_len) noexcept; + /** * Minifies a JSON element or document, printing the smallest possible valid JSON. * @@ -2893,14 +2951,14 @@ namespace simdjson { * */ template -class minify { +class minifier { public: /** * Create a new minifier. * * @param _value The document or element to minify. */ - inline minify(const T &_value) noexcept : value{_value} {} + inline minifier(const T &_value) noexcept : value{_value} {} /** * Minify JSON to a string. @@ -2915,6 +2973,9 @@ private: const T &value; }; +template +inline minifier minify(const T &value) noexcept { return minifier(value); } + /** * Minify JSON to an output stream. * @@ -2923,7 +2984,7 @@ private: * @throw if there is an error with the underlying output stream. simdjson itself will not throw. */ template -inline std::ostream& operator<<(std::ostream& out, minify formatter) { return formatter.print(out); } +inline std::ostream& operator<<(std::ostream& out, minifier formatter) { return formatter.print(out); } } // namespace simdjson @@ -2940,12 +3001,12 @@ class element; /** * JSON array. */ -class array : protected internal::tape_ref { +class array { public: /** Create a new, invalid array */ really_inline array() noexcept; - class iterator : protected internal::tape_ref { + class iterator { public: /** * Get the actual value @@ -2965,7 +3026,8 @@ public: */ inline bool operator!=(const iterator& other) const noexcept; private: - really_inline iterator(const document *doc, size_t json_index) noexcept; + really_inline iterator(const internal::tape_ref &tape) noexcept; + internal::tape_ref tape; friend class array; }; @@ -3004,19 +3066,30 @@ public: inline simdjson_result at(const std::string_view &json_pointer) const noexcept; /** - * Get the value at the given index. + * Get the value at the given index. This function has linear-time complexity and + * is equivalent to the following: + * + * size_t i=0; + * for (auto element : *this) { + * if (i == index) { return element; } + * i++; + * } + * return INDEX_OUT_OF_BOUNDS; * + * Avoid calling the at() function repeatedly. + * * @return The value at the given index, or: * - INDEX_OUT_OF_BOUNDS if the array index is larger than an array length */ inline simdjson_result at(size_t index) const noexcept; private: - really_inline array(const document *doc, size_t json_index) noexcept; + really_inline array(const internal::tape_ref &tape) noexcept; + internal::tape_ref tape; friend class element; friend struct simdjson_result; template - friend class simdjson::minify; + friend class simdjson::minifier; }; /** @@ -3146,7 +3219,7 @@ public: private: inline error_code allocate(size_t len) noexcept; template - friend class simdjson::minify; + friend class simdjson::minifier; friend class parser; }; // class document @@ -3305,6 +3378,10 @@ public: * documents that consist of an object or array may omit the whitespace between them, concatenating * with no separator. documents that consist of a single primitive (i.e. documents that are not * arrays or objects) MUST be separated with whitespace. + * + * The documents must not exceed batch_size bytes (by default 1MB) or they will fail to parse. + * Setting batch_size to excessively large or excesively small values may impact negatively the + * performance. * * ### Error Handling * @@ -3335,20 +3412,19 @@ public: * spot is cache-related: small enough to fit in cache, yet big enough to * parse as many documents as possible in one tight loop. * Defaults to 10MB, which has been a reasonable sweet spot in our tests. - * @return The stream. If there is an error, it will be returned during iteration. An empty input - * will yield 0 documents rather than an EMPTY error. Errors: + * @return The stream, or an error. An empty input will yield 0 documents rather than an EMPTY error. Errors: * - IO_ERROR if there was an error opening or reading the file. * - MEMALLOC if the parser does not have enough capacity and memory allocation fails. * - CAPACITY if the parser does not have enough capacity and batch_size > max_capacity. * - other json errors if parsing fails. */ - inline document_stream load_many(const std::string &path, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept; + inline simdjson_result load_many(const std::string &path, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept; /** * Parse a buffer containing many JSON documents. * * dom::parser parser; - * for (const element doc : parser.parse_many(buf, len)) { + * for (element doc : parser.parse_many(buf, len)) { * cout << std::string(doc["title"]) << endl; * } * @@ -3362,6 +3438,10 @@ public: * documents that consist of an object or array may omit the whitespace between them, concatenating * with no separator. documents that consist of a single primitive (i.e. documents that are not * arrays or objects) MUST be separated with whitespace. + * + * The documents must not exceed batch_size bytes (by default 1MB) or they will fail to parse. + * Setting batch_size to excessively large or excesively small values may impact negatively the + * performance. * * ### Error Handling * @@ -3398,22 +3478,21 @@ public: * spot is cache-related: small enough to fit in cache, yet big enough to * parse as many documents as possible in one tight loop. * Defaults to 10MB, which has been a reasonable sweet spot in our tests. - * @return The stream. If there is an error, it will be returned during iteration. An empty input - * will yield 0 documents rather than an EMPTY error. Errors: + * @return The stream, or an error. An empty input will yield 0 documents rather than an EMPTY error. Errors: * - MEMALLOC if the parser does not have enough capacity and memory allocation fails * - CAPACITY if the parser does not have enough capacity and batch_size > max_capacity. * - other json errors if parsing fails. */ - inline document_stream parse_many(const uint8_t *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept; + inline simdjson_result parse_many(const uint8_t *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept; /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ - inline document_stream parse_many(const char *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept; + inline simdjson_result parse_many(const char *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept; /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ - inline document_stream parse_many(const std::string &s, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept; + inline simdjson_result parse_many(const std::string &s, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept; /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ - inline document_stream parse_many(const padded_string &s, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept; + inline simdjson_result parse_many(const padded_string &s, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept; /** @private We do not want to allow implicit conversion from C string to std::string. */ - really_inline simdjson_result parse_many(const char *buf, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept = delete; + simdjson_result parse_many(const char *buf, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept = delete; /** * Ensure this parser has enough memory to process JSON documents up to `capacity` bytes in length @@ -3562,11 +3641,64 @@ private: /* end file include/simdjson/dom/document.h */ #ifdef SIMDJSON_THREADS_ENABLED #include +#include +#include #endif namespace simdjson { namespace dom { + +#ifdef SIMDJSON_THREADS_ENABLED +/** @private Custom worker class **/ +struct stage1_worker { + stage1_worker() noexcept = default; + stage1_worker(const stage1_worker&) = delete; + stage1_worker(stage1_worker&&) = delete; + stage1_worker operator=(const stage1_worker&) = delete; + ~stage1_worker(); + /** + * We only start the thread when it is needed, not at object construction, this may throw. + * You should only call this once. + **/ + void start_thread(); + /** + * Start a stage 1 job. You should first call 'run', then 'finish'. + * You must call start_thread once before. + */ + void run(document_stream * ds, dom::parser * stage1, size_t next_batch_start); + /** Wait for the run to finish (blocking). You should first call 'run', then 'finish'. **/ + void finish(); + +private: + + /** + * Normally, we would never stop the thread. But we do in the destructor. + * This function is only safe assuming that you are not waiting for results. You + * should have called run, then finish, and be done. + **/ + void stop_thread(); + + std::thread thread{}; + /** These three variables define the work done by the thread. **/ + dom::parser * stage1_thread_parser{}; + size_t _next_batch_start{}; + document_stream * owner{}; + /** + * We have two state variables. This could be streamlined to one variable in the future but + * we use two for clarity. + */ + bool has_work{false}; + bool can_work{true}; + + /** + * We lock using a mutex. + */ + std::mutex locking_mutex{}; + std::condition_variable cond_var{}; +}; +#endif + /** * A forward-only stream of documents. * @@ -3575,8 +3707,20 @@ namespace dom { */ class document_stream { public: + /** + * Construct an uninitialized document_stream. + * + * ```c++ + * document_stream docs; + * error = parser.parse_many(json).get(docs); + * ``` + */ + really_inline document_stream() noexcept; /** Move one document_stream to another. */ - really_inline document_stream(document_stream && other) noexcept = default; + really_inline document_stream(document_stream &&other) noexcept = default; + /** Move one document_stream to another. */ + really_inline document_stream &operator=(document_stream &&other) noexcept = default; + really_inline ~document_stream() noexcept; /** @@ -3597,7 +3741,22 @@ public: * @param other the end iterator to compare to. */ really_inline bool operator!=(const iterator &other) const noexcept; - + /** + * @private + * + * Gives the current index in the input document in bytes. + * + * document_stream stream = parser.parse_many(json,window); + * for(auto i = stream.begin(); i != stream.end(); ++i) { + * auto doc = *i; + * size_t index = i.current_index(); + * } + * + * This function (current_index()) is experimental and the usage + * may change in future versions of simdjson: we find the API somewhat + * awkward and we would like to offer something friendlier. + */ + really_inline size_t current_index() noexcept; private: really_inline iterator(document_stream &s, bool finished) noexcept; /** The document_stream we're iterating through. */ @@ -3619,8 +3778,7 @@ public: private: document_stream &operator=(const document_stream &) = delete; // Disallow copying - - document_stream(document_stream &other) = delete; // Disallow copying + document_stream(const document_stream &other) = delete; // Disallow copying /** * Construct a document_stream. Does not allocate or parse anything until the iterator is @@ -3630,8 +3788,7 @@ private: dom::parser &parser, const uint8_t *buf, size_t len, - size_t batch_size, - error_code error = SUCCESS + size_t batch_size ) noexcept; /** @@ -3678,13 +3835,14 @@ private: /** Pass the next batch through stage 1 with the given parser. */ inline error_code run_stage1(dom::parser &p, size_t batch_start) noexcept; - dom::parser &parser; + dom::parser *parser; const uint8_t *buf; - const size_t len; - const size_t batch_size; - size_t batch_start{0}; + size_t len; + size_t batch_size; /** The error (or lack thereof) from the current document. */ error_code error; + size_t batch_start{0}; + size_t doc_index{}; #ifdef SIMDJSON_THREADS_ENABLED inline void load_from_stage1_thread() noexcept; @@ -3698,8 +3856,8 @@ private: /** The error returned from the stage 1 thread. */ error_code stage1_thread_error{UNINITIALIZED}; /** The thread used to run stage 1 against the next batch in the background. */ - std::thread stage1_thread{}; - + friend struct stage1_worker; + std::unique_ptr worker{new(std::nothrow) stage1_worker()}; /** * The parser used to run stage 1 in the background. Will be swapped * with the regular parser when finished. @@ -3708,9 +3866,31 @@ private: #endif // SIMDJSON_THREADS_ENABLED friend class dom::parser; + friend struct simdjson_result; + friend struct internal::simdjson_result_base; + }; // class document_stream } // namespace dom + +template<> +struct simdjson_result : public internal::simdjson_result_base { +public: + really_inline simdjson_result() noexcept; ///< @private + really_inline simdjson_result(error_code error) noexcept; ///< @private + really_inline simdjson_result(dom::document_stream &&value) noexcept; ///< @private + +#if SIMDJSON_EXCEPTIONS + really_inline dom::document_stream::iterator begin() noexcept(false); + really_inline dom::document_stream::iterator end() noexcept(false); +#else // SIMDJSON_EXCEPTIONS + [[deprecated("parse_many() and load_many() may return errors. Use document_stream stream; error = parser.parse_many().get(doc); instead.")]] + really_inline dom::document_stream::iterator begin() noexcept; + [[deprecated("parse_many() and load_many() may return errors. Use document_stream stream; error = parser.parse_many().get(doc); instead.")]] + really_inline dom::document_stream::iterator end() noexcept; +#endif // SIMDJSON_EXCEPTIONS +}; // struct simdjson_result + } // namespace simdjson #endif // SIMDJSON_DOCUMENT_STREAM_H @@ -3749,7 +3929,7 @@ enum class element_type { * References an element in a JSON document, representing a JSON null, boolean, string, number, * array or object. */ -class element : protected internal::tape_ref { +class element { public: /** Create a new, invalid element. */ really_inline element() noexcept; @@ -3757,8 +3937,135 @@ public: /** The type of this element. */ really_inline element_type type() const noexcept; - /** Whether this element is a json `null`. */ - really_inline bool is_null() const noexcept; + /** + * Cast this element to an array. + * + * Equivalent to get(). + * + * @returns An object that can be used to iterate the array, or: + * INCORRECT_TYPE if the JSON element is not an array. + */ + inline simdjson_result get_array() const noexcept; + /** + * Cast this element to an object. + * + * Equivalent to get(). + * + * @returns An object that can be used to look up or iterate the object's fields, or: + * INCORRECT_TYPE if the JSON element is not an object. + */ + inline simdjson_result get_object() const noexcept; + /** + * Cast this element to a string. + * + * Equivalent to get(). + * + * @returns An pointer to a null-terminated string. This string is stored in the parser and will + * be invalidated the next time it parses a document or when it is destroyed. + * Returns INCORRECT_TYPE if the JSON element is not a string. + */ + inline simdjson_result get_c_str() const noexcept; + /** + * Cast this element to a string. + * + * Equivalent to get(). + * + * @returns A string. The string is stored in the parser and will be invalidated the next time it + * parses a document or when it is destroyed. + * Returns INCORRECT_TYPE if the JSON element is not a string. + */ + inline simdjson_result get_string() const noexcept; + /** + * Cast this element to a signed integer. + * + * Equivalent to get(). + * + * @returns A signed 64-bit integer. + * Returns INCORRECT_TYPE if the JSON element is not an integer, or NUMBER_OUT_OF_RANGE + * if it is negative. + */ + inline simdjson_result get_int64_t() const noexcept; + /** + * Cast this element to an unsigned integer. + * + * Equivalent to get(). + * + * @returns An unsigned 64-bit integer. + * Returns INCORRECT_TYPE if the JSON element is not an integer, or NUMBER_OUT_OF_RANGE + * if it is too large. + */ + inline simdjson_result get_uint64_t() const noexcept; + /** + * Cast this element to an double floating-point. + * + * Equivalent to get(). + * + * @returns A double value. + * Returns INCORRECT_TYPE if the JSON element is not a number. + */ + inline simdjson_result get_double() const noexcept; + /** + * Cast this element to a bool. + * + * Equivalent to get(). + * + * @returns A bool value. + * Returns INCORRECT_TYPE if the JSON element is not a boolean. + */ + inline simdjson_result get_bool() const noexcept; + + /** + * Whether this element is a json array. + * + * Equivalent to is(). + */ + inline bool is_array() const noexcept; + /** + * Whether this element is a json object. + * + * Equivalent to is(). + */ + inline bool is_object() const noexcept; + /** + * Whether this element is a json string. + * + * Equivalent to is() or is(). + */ + inline bool is_string() const noexcept; + /** + * Whether this element is a json number that fits in a signed 64-bit integer. + * + * Equivalent to is(). + */ + inline bool is_int64_t() const noexcept; + /** + * Whether this element is a json number that fits in an unsigned 64-bit integer. + * + * Equivalent to is(). + */ + inline bool is_uint64_t() const noexcept; + /** + * Whether this element is a json number that fits in a double. + * + * Equivalent to is(). + */ + inline bool is_double() const noexcept; + /** + * Whether this element is a json number. + * + * Both integers and floating points will return true. + */ + inline bool is_number() const noexcept; + /** + * Whether this element is a json `true` or `false`. + * + * Equivalent to is(). + */ + inline bool is_bool() const noexcept; + /** + * Whether this element is a json `null`. + */ + inline bool is_null() const noexcept; /** * Tell whether the value can be cast to provided type (T). @@ -3791,7 +4098,44 @@ public: * INCORRECT_TYPE if the value cannot be cast to the given type. */ template - really_inline simdjson_result get() const noexcept; + inline simdjson_result get() const noexcept; + + /** + * Get the value as the provided type (T). + * + * Supported types: + * - Boolean: bool + * - Number: double, uint64_t, int64_t + * - String: std::string_view, const char * + * - Array: dom::array + * - Object: dom::object + * + * @tparam T bool, double, uint64_t, int64_t, std::string_view, const char *, dom::array, dom::object + * + * @param value The variable to set to the value. May not be set if there is an error. + * + * @returns The error that occurred, or SUCCESS if there was no error. + */ + template + WARN_UNUSED really_inline error_code get(T &value) const noexcept; + + /** + * Get the value as the provided type (T), setting error if it's not the given type. + * + * Supported types: + * - Boolean: bool + * - Number: double, uint64_t, int64_t + * - String: std::string_view, const char * + * - Array: dom::array + * - Object: dom::object + * + * @tparam T bool, double, uint64_t, int64_t, std::string_view, const char *, dom::array, dom::object + * + * @param value The variable to set to the given type. value is undefined if there is an error. + * @param error The variable to store the error. error is set to error_code::SUCCEED if there is an error. + */ + template + inline void tie(T &value, error_code &error) && noexcept; #if SIMDJSON_EXCEPTIONS /** @@ -3963,13 +4307,14 @@ public: inline bool dump_raw_tape(std::ostream &out) const noexcept; private: - really_inline element(const document *doc, size_t json_index) noexcept; + really_inline element(const internal::tape_ref &tape) noexcept; + internal::tape_ref tape; friend class document; friend class object; friend class array; friend struct simdjson_result; template - friend class simdjson::minify; + friend class simdjson::minifier; }; /** @@ -4002,32 +4347,51 @@ public: really_inline simdjson_result(dom::element &&value) noexcept; ///< @private really_inline simdjson_result(error_code error) noexcept; ///< @private - inline simdjson_result type() const noexcept; - inline simdjson_result is_null() const noexcept; + really_inline simdjson_result type() const noexcept; template - inline simdjson_result is() const noexcept; + really_inline simdjson_result is() const noexcept; template - inline simdjson_result get() const noexcept; + really_inline simdjson_result get() const noexcept; + template + WARN_UNUSED really_inline error_code get(T &value) const noexcept; - inline simdjson_result operator[](const std::string_view &key) const noexcept; - inline simdjson_result operator[](const char *key) const noexcept; - inline simdjson_result at(const std::string_view &json_pointer) const noexcept; - inline simdjson_result at(size_t index) const noexcept; - inline simdjson_result at_key(const std::string_view &key) const noexcept; - inline simdjson_result at_key_case_insensitive(const std::string_view &key) const noexcept; + really_inline simdjson_result get_array() const noexcept; + really_inline simdjson_result get_object() const noexcept; + really_inline simdjson_result get_c_str() const noexcept; + really_inline simdjson_result get_string() const noexcept; + really_inline simdjson_result get_int64_t() const noexcept; + really_inline simdjson_result get_uint64_t() const noexcept; + really_inline simdjson_result get_double() const noexcept; + really_inline simdjson_result get_bool() const noexcept; + + really_inline simdjson_result is_array() const noexcept; + really_inline simdjson_result is_object() const noexcept; + really_inline simdjson_result is_string() const noexcept; + really_inline simdjson_result is_int64_t() const noexcept; + really_inline simdjson_result is_uint64_t() const noexcept; + really_inline simdjson_result is_double() const noexcept; + really_inline simdjson_result is_bool() const noexcept; + really_inline simdjson_result is_null() const noexcept; + + really_inline simdjson_result operator[](const std::string_view &key) const noexcept; + really_inline simdjson_result operator[](const char *key) const noexcept; + really_inline simdjson_result at(const std::string_view &json_pointer) const noexcept; + really_inline simdjson_result at(size_t index) const noexcept; + really_inline simdjson_result at_key(const std::string_view &key) const noexcept; + really_inline simdjson_result at_key_case_insensitive(const std::string_view &key) const noexcept; #if SIMDJSON_EXCEPTIONS - inline operator bool() const noexcept(false); - inline explicit operator const char*() const noexcept(false); - inline operator std::string_view() const noexcept(false); - inline operator uint64_t() const noexcept(false); - inline operator int64_t() const noexcept(false); - inline operator double() const noexcept(false); - inline operator dom::array() const noexcept(false); - inline operator dom::object() const noexcept(false); + really_inline operator bool() const noexcept(false); + really_inline explicit operator const char*() const noexcept(false); + really_inline operator std::string_view() const noexcept(false); + really_inline operator uint64_t() const noexcept(false); + really_inline operator int64_t() const noexcept(false); + really_inline operator double() const noexcept(false); + really_inline operator dom::array() const noexcept(false); + really_inline operator dom::object() const noexcept(false); - inline dom::array::iterator begin() const noexcept(false); - inline dom::array::iterator end() const noexcept(false); + really_inline dom::array::iterator begin() const noexcept(false); + really_inline dom::array::iterator end() const noexcept(false); #endif // SIMDJSON_EXCEPTIONS }; @@ -4043,7 +4407,7 @@ public: * underlying output stream, that error will be propagated (simdjson_error will not be * thrown). */ -inline std::ostream& operator<<(std::ostream& out, const simdjson_result &value) noexcept(false); +really_inline std::ostream& operator<<(std::ostream& out, const simdjson_result &value) noexcept(false); #endif } // namespace simdjson @@ -4066,12 +4430,12 @@ class key_value_pair; /** * JSON object. */ -class object : protected internal::tape_ref { +class object { public: /** Create a new, invalid object */ really_inline object() noexcept; - class iterator : protected internal::tape_ref { + class iterator { public: /** * Get the actual key/value pair @@ -4119,7 +4483,10 @@ public: */ inline element value() const noexcept; private: - really_inline iterator(const document *doc, size_t json_index) noexcept; + really_inline iterator(const internal::tape_ref &tape) noexcept; + + internal::tape_ref tape; + friend class object; }; @@ -4150,6 +4517,8 @@ public: * parser.parse(R"({ "a\n": 1 })")["a\n"].get().value == 1 * parser.parse(R"({ "a\n": 1 })")["a\\n"].get().error == NO_SUCH_FIELD * + * This function has linear-time complexity: the keys are checked one by one. + * * @return The value associated with this field, or: * - NO_SUCH_FIELD if the field does not exist in the object * - INCORRECT_TYPE if this is not an object @@ -4165,6 +4534,8 @@ public: * parser.parse(R"({ "a\n": 1 })")["a\n"].get().value == 1 * parser.parse(R"({ "a\n": 1 })")["a\\n"].get().error == NO_SUCH_FIELD * + * This function has linear-time complexity: the keys are checked one by one. + * * @return The value associated with this field, or: * - NO_SUCH_FIELD if the field does not exist in the object * - INCORRECT_TYPE if this is not an object @@ -4196,6 +4567,8 @@ public: * parser.parse(R"({ "a\n": 1 })")["a\n"].get().value == 1 * parser.parse(R"({ "a\n": 1 })")["a\\n"].get().error == NO_SUCH_FIELD * + * This function has linear-time complexity: the keys are checked one by one. + * * @return The value associated with this field, or: * - NO_SUCH_FIELD if the field does not exist in the object */ @@ -4207,17 +4580,22 @@ public: * * Note: The key will be matched against **unescaped** JSON. * + * This function has linear-time complexity: the keys are checked one by one. + * * @return The value associated with this field, or: * - NO_SUCH_FIELD if the field does not exist in the object */ inline simdjson_result at_key_case_insensitive(const std::string_view &key) const noexcept; private: - really_inline object(const document *doc, size_t json_index) noexcept; + really_inline object(const internal::tape_ref &tape) noexcept; + + internal::tape_ref tape; + friend class element; friend struct simdjson_result; template - friend class simdjson::minify; + friend class simdjson::minifier; }; /** @@ -4840,16 +5218,16 @@ namespace dom { // // array inline implementation // -really_inline array::array() noexcept : internal::tape_ref() {} -really_inline array::array(const document *_doc, size_t _json_index) noexcept : internal::tape_ref(_doc, _json_index) {} +really_inline array::array() noexcept : tape{} {} +really_inline array::array(const internal::tape_ref &_tape) noexcept : tape{_tape} {} inline array::iterator array::begin() const noexcept { - return iterator(doc, json_index + 1); + return internal::tape_ref(tape.doc, tape.json_index + 1); } inline array::iterator array::end() const noexcept { - return iterator(doc, after_element() - 1); + return internal::tape_ref(tape.doc, tape.after_element() - 1); } inline size_t array::size() const noexcept { - return scope_count(); + return tape.scope_count(); } inline simdjson_result array::at(const std::string_view &json_pointer) const noexcept { // - means "the append position" or "the element after the end of the array" @@ -4873,7 +5251,7 @@ inline simdjson_result array::at(const std::string_view &json_pointer) if (i == 0) { return INVALID_JSON_POINTER; } // "Empty string in JSON pointer array index" // Get the child - auto child = array(doc, json_index).at(array_index); + auto child = array(tape).at(array_index); // If there is a /, we're not done yet, call recursively. if (i < json_pointer.length()) { child = child.at(json_pointer.substr(i+1)); @@ -4892,15 +5270,15 @@ inline simdjson_result array::at(size_t index) const noexcept { // // array::iterator inline implementation // -really_inline array::iterator::iterator(const document *_doc, size_t _json_index) noexcept : internal::tape_ref(_doc, _json_index) { } +really_inline array::iterator::iterator(const internal::tape_ref &_tape) noexcept : tape{_tape} { } inline element array::iterator::operator*() const noexcept { - return element(doc, json_index); + return element(tape); } inline bool array::iterator::operator!=(const array::iterator& other) const noexcept { - return json_index != other.json_index; + return tape.json_index != other.tape.json_index; } inline array::iterator& array::iterator::operator++() noexcept { - json_index = after_element(); + tape.json_index = tape.after_element(); return *this; } @@ -4911,7 +5289,7 @@ inline std::ostream& operator<<(std::ostream& out, const array &value) { } // namespace dom template<> -inline std::ostream& minify::print(std::ostream& out) { +inline std::ostream& minifier::print(std::ostream& out) { out << '['; auto iter = value.begin(); auto end = value.end(); @@ -4927,7 +5305,7 @@ inline std::ostream& minify::print(std::ostream& out) { #if SIMDJSON_EXCEPTIONS template<> -inline std::ostream& minify>::print(std::ostream& out) { +inline std::ostream& minifier>::print(std::ostream& out) { if (value.error()) { throw simdjson_error(value.error()); } return out << minify(value.first); } @@ -4949,34 +5327,95 @@ inline std::ostream& operator<<(std::ostream& out, const simdjson_result #include #include - namespace simdjson { namespace dom { +#ifdef SIMDJSON_THREADS_ENABLED +inline void stage1_worker::finish() { + std::unique_lock lock(locking_mutex); + cond_var.wait(lock, [this]{return has_work == false;}); +} + +inline stage1_worker::~stage1_worker() { + stop_thread(); +} + +inline void stage1_worker::start_thread() { + std::unique_lock lock(locking_mutex); + if(thread.joinable()) { + return; // This should never happen but we never want to create more than one thread. + } + thread = std::thread([this]{ + while(can_work) { + std::unique_lock thread_lock(locking_mutex); + cond_var.wait(thread_lock, [this]{return has_work || !can_work;}); + if(!can_work) { + break; + } + this->owner->stage1_thread_error = this->owner->run_stage1(*this->stage1_thread_parser, + this->_next_batch_start); + this->has_work = false; + thread_lock.unlock(); + cond_var.notify_one(); // will notify "finish" + } + } + ); +} + + +inline void stage1_worker::stop_thread() { + std::unique_lock lock(locking_mutex); + // We have to make sure that all locks can be released. + can_work = false; + has_work = false; + lock.unlock(); + cond_var.notify_all(); + if(thread.joinable()) { + thread.join(); + } +} + +inline void stage1_worker::run(document_stream * ds, dom::parser * stage1, size_t next_batch_start) { + std::unique_lock lock(locking_mutex); + owner = ds; + _next_batch_start = next_batch_start; + stage1_thread_parser = stage1; + has_work = true; + lock.unlock(); + cond_var.notify_one();// will notify the thread lock +} +#endif + really_inline document_stream::document_stream( dom::parser &_parser, const uint8_t *_buf, size_t _len, - size_t _batch_size, - error_code _error + size_t _batch_size ) noexcept - : parser{_parser}, + : parser{&_parser}, buf{_buf}, len{_len}, batch_size{_batch_size}, - error{_error} + error{SUCCESS} { -} - -inline document_stream::~document_stream() noexcept { #ifdef SIMDJSON_THREADS_ENABLED - // TODO kill the thread, why should people have to wait for a non-side-effecting operation to complete - if (stage1_thread.joinable()) { - stage1_thread.join(); + if(worker.get() == nullptr) { + error = MEMALLOC; } #endif } +really_inline document_stream::document_stream() noexcept + : parser{nullptr}, + buf{nullptr}, + len{0}, + batch_size{0}, + error{UNINITIALIZED} { +} + +really_inline document_stream::~document_stream() noexcept { +} + really_inline document_stream::iterator document_stream::begin() noexcept { start(); // If there are no documents, we're finished. @@ -4994,7 +5433,7 @@ really_inline document_stream::iterator::iterator(document_stream& _stream, bool really_inline simdjson_result document_stream::iterator::operator*() noexcept { // Once we have yielded any errors, we're finished. if (stream.error) { finished = true; return stream.error; } - return stream.parser.doc.root(); + return stream.parser->doc.root(); } really_inline document_stream::iterator& document_stream::iterator::operator++() noexcept { @@ -5011,12 +5450,12 @@ really_inline bool document_stream::iterator::operator!=(const document_stream:: inline void document_stream::start() noexcept { if (error) { return; } - error = parser.ensure_capacity(batch_size); + error = parser->ensure_capacity(batch_size); if (error) { return; } // Always run the first stage 1 parse immediately batch_start = 0; - error = run_stage1(parser, batch_start); + error = run_stage1(*parser, batch_start); if (error) { return; } #ifdef SIMDJSON_THREADS_ENABLED @@ -5024,6 +5463,7 @@ inline void document_stream::start() noexcept { // Kick off the first thread if needed error = stage1_thread_parser.ensure_capacity(batch_size); if (error) { return; } + worker->start_thread(); start_stage1_thread(); if (error) { return; } } @@ -5032,12 +5472,15 @@ inline void document_stream::start() noexcept { next(); } +really_inline size_t document_stream::iterator::current_index() noexcept { + return stream.doc_index; +} inline void document_stream::next() noexcept { if (error) { return; } // Load the next document from the batch - error = parser.implementation->stage2_next(parser.doc); - + doc_index = batch_start + parser->implementation->structural_indexes[parser->implementation->next_structural_index]; + error = parser->implementation->stage2_next(parser->doc); // If that was the last document in the batch, load another batch (if available) while (error == EMPTY) { batch_start = next_batch_start(); @@ -5046,17 +5489,17 @@ inline void document_stream::next() noexcept { #ifdef SIMDJSON_THREADS_ENABLED load_from_stage1_thread(); #else - error = run_stage1(parser, batch_start); + error = run_stage1(*parser, batch_start); #endif if (error) { continue; } // If the error was EMPTY, we may want to load another batch. - // Run stage 2 on the first document in the batch - error = parser.implementation->stage2_next(parser.doc); + doc_index = batch_start + parser->implementation->structural_indexes[parser->implementation->next_structural_index]; + error = parser->implementation->stage2_next(parser->doc); } } inline size_t document_stream::next_batch_start() const noexcept { - return batch_start + parser.implementation->structural_indexes[parser.implementation->n_structural_indexes]; + return batch_start + parser->implementation->structural_indexes[parser->implementation->n_structural_indexes]; } inline error_code document_stream::run_stage1(dom::parser &p, size_t _batch_start) noexcept { @@ -5072,11 +5515,10 @@ inline error_code document_stream::run_stage1(dom::parser &p, size_t _batch_star #ifdef SIMDJSON_THREADS_ENABLED inline void document_stream::load_from_stage1_thread() noexcept { - stage1_thread.join(); - + worker->finish(); // Swap to the parser that was loaded up in the thread. Make sure the parser has // enough memory to swap to, as well. - std::swap(parser, stage1_thread_parser); + std::swap(*parser, stage1_thread_parser); error = stage1_thread_error; if (error) { return; } @@ -5093,14 +5535,44 @@ inline void document_stream::start_stage1_thread() noexcept { // TODO this is NOT exception-safe. this->stage1_thread_error = UNINITIALIZED; // In case something goes wrong, make sure it's an error size_t _next_batch_start = this->next_batch_start(); - stage1_thread = std::thread([this, _next_batch_start] { - this->stage1_thread_error = run_stage1(this->stage1_thread_parser, _next_batch_start); - }); + + worker->run(this, & this->stage1_thread_parser, _next_batch_start); } #endif // SIMDJSON_THREADS_ENABLED } // namespace dom + +really_inline simdjson_result::simdjson_result() noexcept + : simdjson_result_base() { +} +really_inline simdjson_result::simdjson_result(error_code error) noexcept + : simdjson_result_base(error) { +} +really_inline simdjson_result::simdjson_result(dom::document_stream &&value) noexcept + : simdjson_result_base(std::forward(value)) { +} + +#if SIMDJSON_EXCEPTIONS +really_inline dom::document_stream::iterator simdjson_result::begin() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first.begin(); +} +really_inline dom::document_stream::iterator simdjson_result::end() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first.end(); +} +#else // SIMDJSON_EXCEPTIONS +really_inline dom::document_stream::iterator simdjson_result::begin() noexcept { + first.error = error(); + return first.begin(); +} +really_inline dom::document_stream::iterator simdjson_result::end() noexcept { + first.error = error(); + return first.end(); +} +#endif // SIMDJSON_EXCEPTIONS + } // namespace simdjson #endif // SIMDJSON_INLINE_DOCUMENT_STREAM_H /* end file include/simdjson/inline/document_stream.h */ @@ -5120,7 +5592,7 @@ namespace dom { // document inline implementation // inline element document::root() const noexcept { - return element(this, 1); + return element(internal::tape_ref(this, 1)); } WARN_UNUSED @@ -5265,133 +5737,195 @@ inline simdjson_result simdjson_result::type() if (error()) { return error(); } return first.type(); } -inline simdjson_result simdjson_result::is_null() const noexcept { - if (error()) { return error(); } - return first.is_null(); -} + template -inline simdjson_result simdjson_result::is() const noexcept { +really_inline simdjson_result simdjson_result::is() const noexcept { if (error()) { return error(); } return first.is(); } template -inline simdjson_result simdjson_result::get() const noexcept { +really_inline simdjson_result simdjson_result::get() const noexcept { if (error()) { return error(); } return first.get(); } +template +WARN_UNUSED really_inline error_code simdjson_result::get(T &value) const noexcept { + if (error()) { return error(); } + return first.get(value); +} -inline simdjson_result simdjson_result::operator[](const std::string_view &key) const noexcept { +really_inline simdjson_result simdjson_result::get_array() const noexcept { + if (error()) { return error(); } + return first.get_array(); +} +really_inline simdjson_result simdjson_result::get_object() const noexcept { + if (error()) { return error(); } + return first.get_object(); +} +really_inline simdjson_result simdjson_result::get_c_str() const noexcept { + if (error()) { return error(); } + return first.get_c_str(); +} +really_inline simdjson_result simdjson_result::get_string() const noexcept { + if (error()) { return error(); } + return first.get_string(); +} +really_inline simdjson_result simdjson_result::get_int64_t() const noexcept { + if (error()) { return error(); } + return first.get_int64_t(); +} +really_inline simdjson_result simdjson_result::get_uint64_t() const noexcept { + if (error()) { return error(); } + return first.get_uint64_t(); +} +really_inline simdjson_result simdjson_result::get_double() const noexcept { + if (error()) { return error(); } + return first.get_double(); +} +really_inline simdjson_result simdjson_result::get_bool() const noexcept { + if (error()) { return error(); } + return first.get_bool(); +} + +really_inline simdjson_result simdjson_result::is_array() const noexcept { + if (error()) { return error(); } + return first.is_array(); +} +really_inline simdjson_result simdjson_result::is_object() const noexcept { + if (error()) { return error(); } + return first.is_object(); +} +really_inline simdjson_result simdjson_result::is_string() const noexcept { + if (error()) { return error(); } + return first.is_string(); +} +really_inline simdjson_result simdjson_result::is_int64_t() const noexcept { + if (error()) { return error(); } + return first.is_int64_t(); +} +really_inline simdjson_result simdjson_result::is_uint64_t() const noexcept { + if (error()) { return error(); } + return first.is_uint64_t(); +} +really_inline simdjson_result simdjson_result::is_double() const noexcept { + if (error()) { return error(); } + return first.is_double(); +} +really_inline simdjson_result simdjson_result::is_bool() const noexcept { + if (error()) { return error(); } + return first.is_bool(); +} + +really_inline simdjson_result simdjson_result::is_null() const noexcept { + if (error()) { return error(); } + return first.is_null(); +} + +really_inline simdjson_result simdjson_result::operator[](const std::string_view &key) const noexcept { if (error()) { return error(); } return first[key]; } -inline simdjson_result simdjson_result::operator[](const char *key) const noexcept { +really_inline simdjson_result simdjson_result::operator[](const char *key) const noexcept { if (error()) { return error(); } return first[key]; } -inline simdjson_result simdjson_result::at(const std::string_view &json_pointer) const noexcept { +really_inline simdjson_result simdjson_result::at(const std::string_view &json_pointer) const noexcept { if (error()) { return error(); } return first.at(json_pointer); } -inline simdjson_result simdjson_result::at(size_t index) const noexcept { +really_inline simdjson_result simdjson_result::at(size_t index) const noexcept { if (error()) { return error(); } return first.at(index); } -inline simdjson_result simdjson_result::at_key(const std::string_view &key) const noexcept { +really_inline simdjson_result simdjson_result::at_key(const std::string_view &key) const noexcept { if (error()) { return error(); } return first.at_key(key); } -inline simdjson_result simdjson_result::at_key_case_insensitive(const std::string_view &key) const noexcept { +really_inline simdjson_result simdjson_result::at_key_case_insensitive(const std::string_view &key) const noexcept { if (error()) { return error(); } return first.at_key_case_insensitive(key); } #if SIMDJSON_EXCEPTIONS -inline simdjson_result::operator bool() const noexcept(false) { +really_inline simdjson_result::operator bool() const noexcept(false) { return get(); } -inline simdjson_result::operator const char *() const noexcept(false) { +really_inline simdjson_result::operator const char *() const noexcept(false) { return get(); } -inline simdjson_result::operator std::string_view() const noexcept(false) { +really_inline simdjson_result::operator std::string_view() const noexcept(false) { return get(); } -inline simdjson_result::operator uint64_t() const noexcept(false) { +really_inline simdjson_result::operator uint64_t() const noexcept(false) { return get(); } -inline simdjson_result::operator int64_t() const noexcept(false) { +really_inline simdjson_result::operator int64_t() const noexcept(false) { return get(); } -inline simdjson_result::operator double() const noexcept(false) { +really_inline simdjson_result::operator double() const noexcept(false) { return get(); } -inline simdjson_result::operator dom::array() const noexcept(false) { +really_inline simdjson_result::operator dom::array() const noexcept(false) { return get(); } -inline simdjson_result::operator dom::object() const noexcept(false) { +really_inline simdjson_result::operator dom::object() const noexcept(false) { return get(); } -inline dom::array::iterator simdjson_result::begin() const noexcept(false) { +really_inline dom::array::iterator simdjson_result::begin() const noexcept(false) { if (error()) { throw simdjson_error(error()); } return first.begin(); } -inline dom::array::iterator simdjson_result::end() const noexcept(false) { +really_inline dom::array::iterator simdjson_result::end() const noexcept(false) { if (error()) { throw simdjson_error(error()); } return first.end(); } -#endif +#endif // SIMDJSON_EXCEPTIONS namespace dom { // // element inline implementation // -really_inline element::element() noexcept : internal::tape_ref() {} -really_inline element::element(const document *_doc, size_t _json_index) noexcept : internal::tape_ref(_doc, _json_index) { } +really_inline element::element() noexcept : tape{} {} +really_inline element::element(const internal::tape_ref &_tape) noexcept : tape{_tape} { } inline element_type element::type() const noexcept { - auto tape_type = tape_ref_type(); + auto tape_type = tape.tape_ref_type(); return tape_type == internal::tape_type::FALSE_VALUE ? element_type::BOOL : static_cast(tape_type); } -really_inline bool element::is_null() const noexcept { - return is_null_on_tape(); -} -template<> -inline simdjson_result element::get() const noexcept { - if(is_true()) { +inline simdjson_result element::get_bool() const noexcept { + if(tape.is_true()) { return true; - } else if(is_false()) { + } else if(tape.is_false()) { return false; } return INCORRECT_TYPE; } -template<> -inline simdjson_result element::get() const noexcept { - switch (tape_ref_type()) { +inline simdjson_result element::get_c_str() const noexcept { + switch (tape.tape_ref_type()) { case internal::tape_type::STRING: { - return get_c_str(); + return tape.get_c_str(); } default: return INCORRECT_TYPE; } } -template<> -inline simdjson_result element::get() const noexcept { - switch (tape_ref_type()) { +inline simdjson_result element::get_string() const noexcept { + switch (tape.tape_ref_type()) { case internal::tape_type::STRING: - return get_string_view(); + return tape.get_string_view(); default: return INCORRECT_TYPE; } } -template<> -inline simdjson_result element::get() const noexcept { - if(unlikely(!is_uint64())) { // branch rarely taken - if(is_int64()) { - int64_t result = next_tape_value(); +inline simdjson_result element::get_uint64_t() const noexcept { + if(unlikely(!tape.is_uint64())) { // branch rarely taken + if(tape.is_int64()) { + int64_t result = tape.next_tape_value(); if (result < 0) { return NUMBER_OUT_OF_RANGE; } @@ -5399,13 +5933,12 @@ inline simdjson_result element::get() const noexcept { } return INCORRECT_TYPE; } - return next_tape_value(); + return tape.next_tape_value(); } -template<> -inline simdjson_result element::get() const noexcept { - if(unlikely(!is_int64())) { // branch rarely taken - if(is_uint64()) { - uint64_t result = next_tape_value(); +inline simdjson_result element::get_int64_t() const noexcept { + if(unlikely(!tape.is_int64())) { // branch rarely taken + if(tape.is_uint64()) { + uint64_t result = tape.next_tape_value(); // Wrapping max in parens to handle Windows issue: https://stackoverflow.com/questions/11544073/how-do-i-deal-with-the-max-macro-in-windows-h-colliding-with-max-in-std if (result > uint64_t((std::numeric_limits::max)())) { return NUMBER_OUT_OF_RANGE; @@ -5414,10 +5947,9 @@ inline simdjson_result element::get() const noexcept { } return INCORRECT_TYPE; } - return next_tape_value(); + return tape.next_tape_value(); } -template<> -inline simdjson_result element::get() const noexcept { +inline simdjson_result element::get_double() const noexcept { // Performance considerations: // 1. Querying tape_ref_type() implies doing a shift, it is fast to just do a straight // comparison. @@ -5427,42 +5959,72 @@ inline simdjson_result element::get() const noexcept { // We can expect get to refer to a double type almost all the time. // It is important to craft the code accordingly so that the compiler can use this // information. (This could also be solved with profile-guided optimization.) - if(unlikely(!is_double())) { // branch rarely taken - if(is_uint64()) { - return double(next_tape_value()); - } else if(is_int64()) { - return double(next_tape_value()); + if(unlikely(!tape.is_double())) { // branch rarely taken + if(tape.is_uint64()) { + return double(tape.next_tape_value()); + } else if(tape.is_int64()) { + return double(tape.next_tape_value()); } return INCORRECT_TYPE; } // this is common: - return next_tape_value(); + return tape.next_tape_value(); } -template<> -inline simdjson_result element::get() const noexcept { - switch (tape_ref_type()) { +inline simdjson_result element::get_array() const noexcept { + switch (tape.tape_ref_type()) { case internal::tape_type::START_ARRAY: - return array(doc, json_index); + return array(tape); default: return INCORRECT_TYPE; } } -template<> -inline simdjson_result element::get() const noexcept { - switch (tape_ref_type()) { +inline simdjson_result element::get_object() const noexcept { + switch (tape.tape_ref_type()) { case internal::tape_type::START_OBJECT: - return object(doc, json_index); + return object(tape); default: return INCORRECT_TYPE; } } +template +WARN_UNUSED really_inline error_code element::get(T &value) const noexcept { + return get().get(value); +} +// An element-specific version prevents recursion with simdjson_result::get(value) +template<> +WARN_UNUSED really_inline error_code element::get(element &value) const noexcept { + value = element(tape); + return SUCCESS; +} + template really_inline bool element::is() const noexcept { auto result = get(); return !result.error(); } +template<> inline simdjson_result element::get() const noexcept { return get_array(); } +template<> inline simdjson_result element::get() const noexcept { return get_object(); } +template<> inline simdjson_result element::get() const noexcept { return get_c_str(); } +template<> inline simdjson_result element::get() const noexcept { return get_string(); } +template<> inline simdjson_result element::get() const noexcept { return get_int64_t(); } +template<> inline simdjson_result element::get() const noexcept { return get_uint64_t(); } +template<> inline simdjson_result element::get() const noexcept { return get_double(); } +template<> inline simdjson_result element::get() const noexcept { return get_bool(); } + +inline bool element::is_array() const noexcept { return is(); } +inline bool element::is_object() const noexcept { return is(); } +inline bool element::is_string() const noexcept { return is(); } +inline bool element::is_int64_t() const noexcept { return is(); } +inline bool element::is_uint64_t() const noexcept { return is(); } +inline bool element::is_double() const noexcept { return is(); } +inline bool element::is_bool() const noexcept { return is(); } + +inline bool element::is_null() const noexcept { + return tape.is_null_on_tape(); +} + #if SIMDJSON_EXCEPTIONS inline element::operator bool() const noexcept(false) { return get(); } @@ -5490,11 +6052,11 @@ inline simdjson_result element::operator[](const char *key) const noexc return at_key(key); } inline simdjson_result element::at(const std::string_view &json_pointer) const noexcept { - switch (tape_ref_type()) { + switch (tape.tape_ref_type()) { case internal::tape_type::START_OBJECT: - return object(doc, json_index).at(json_pointer); + return object(tape).at(json_pointer); case internal::tape_type::START_ARRAY: - return array(doc, json_index).at(json_pointer); + return array(tape).at(json_pointer); default: return INCORRECT_TYPE; } @@ -5510,7 +6072,7 @@ inline simdjson_result element::at_key_case_insensitive(const std::stri } inline bool element::dump_raw_tape(std::ostream &out) const noexcept { - return doc->dump_raw_tape(out); + return tape.doc->dump_raw_tape(out); } inline std::ostream& operator<<(std::ostream& out, const element &value) { @@ -5543,7 +6105,7 @@ inline std::ostream& operator<<(std::ostream& out, element_type type) { } // namespace dom template<> -inline std::ostream& minify::print(std::ostream& out) { +inline std::ostream& minifier::print(std::ostream& out) { using tape_type=internal::tape_type; size_t depth = 0; constexpr size_t MAX_DEPTH = 16; @@ -5551,7 +6113,7 @@ inline std::ostream& minify::print(std::ostream& out) { is_object[0] = false; bool after_value = false; - internal::tape_ref iter(value); + internal::tape_ref iter(value.tape); do { // print commas after each value if (after_value) { @@ -5569,7 +6131,7 @@ inline std::ostream& minify::print(std::ostream& out) { // If we're too deep, we need to recurse to go deeper. depth++; if (unlikely(depth >= MAX_DEPTH)) { - out << minify(dom::array(iter.doc, iter.json_index)); + out << minify(dom::array(iter)); iter.json_index = iter.matching_brace_index() - 1; // Jump to the ] depth--; break; @@ -5596,7 +6158,7 @@ inline std::ostream& minify::print(std::ostream& out) { // If we're too deep, we need to recurse to go deeper. depth++; if (unlikely(depth >= MAX_DEPTH)) { - out << minify(dom::object(iter.doc, iter.json_index)); + out << minify(dom::object(iter)); iter.json_index = iter.matching_brace_index() - 1; // Jump to the } depth--; break; @@ -5669,12 +6231,12 @@ inline std::ostream& minify::print(std::ostream& out) { #if SIMDJSON_EXCEPTIONS template<> -inline std::ostream& minify>::print(std::ostream& out) { +really_inline std::ostream& minifier>::print(std::ostream& out) { if (value.error()) { throw simdjson_error(value.error()); } return out << minify(value.first); } -inline std::ostream& operator<<(std::ostream& out, const simdjson_result &value) noexcept(false) { +really_inline std::ostream& operator<<(std::ostream& out, const simdjson_result &value) noexcept(false) { return out << minify>(value); } #endif @@ -5730,8 +6292,17 @@ really_inline void simdjson_result_base::tie(T &value, error_code &error) && // on the clang compiler that comes with current macOS (Apple clang version 11.0.0), // tie(width, error) = size["w"].get(); // fails with "error: no viable overloaded '='"" - value = std::forward>(*this).first; error = this->second; + if (!error) { + value = std::forward>(*this).first; + } +} + +template +WARN_UNUSED really_inline error_code simdjson_result_base::get(T &value) && noexcept { + error_code error; + std::forward>(*this).tie(value, error); + return error; } template @@ -5784,6 +6355,11 @@ really_inline void simdjson_result::tie(T &value, error_code &error) && noexc std::forward>(*this).tie(value, error); } +template +WARN_UNUSED really_inline error_code simdjson_result::get(T &value) && noexcept { + return std::forward>(*this).get(value); +} + template really_inline error_code simdjson_result::error() const noexcept { return internal::simdjson_result_base::error(); @@ -5887,16 +6463,16 @@ namespace dom { // // object inline implementation // -really_inline object::object() noexcept : internal::tape_ref() {} -really_inline object::object(const document *_doc, size_t _json_index) noexcept : internal::tape_ref(_doc, _json_index) { } +really_inline object::object() noexcept : tape{} {} +really_inline object::object(const internal::tape_ref &_tape) noexcept : tape{_tape} { } inline object::iterator object::begin() const noexcept { - return iterator(doc, json_index + 1); + return internal::tape_ref(tape.doc, tape.json_index + 1); } inline object::iterator object::end() const noexcept { - return iterator(doc, after_element() - 1); + return internal::tape_ref(tape.doc, tape.after_element() - 1); } inline size_t object::size() const noexcept { - return scope_count(); + return tape.scope_count(); } inline simdjson_result object::operator[](const std::string_view &key) const noexcept { @@ -5967,29 +6543,29 @@ inline simdjson_result object::at_key_case_insensitive(const std::strin // // object::iterator inline implementation // -really_inline object::iterator::iterator(const document *_doc, size_t _json_index) noexcept : internal::tape_ref(_doc, _json_index) { } +really_inline object::iterator::iterator(const internal::tape_ref &_tape) noexcept : tape{_tape} { } inline const key_value_pair object::iterator::operator*() const noexcept { return key_value_pair(key(), value()); } inline bool object::iterator::operator!=(const object::iterator& other) const noexcept { - return json_index != other.json_index; + return tape.json_index != other.tape.json_index; } inline object::iterator& object::iterator::operator++() noexcept { - json_index++; - json_index = after_element(); + tape.json_index++; + tape.json_index = tape.after_element(); return *this; } inline std::string_view object::iterator::key() const noexcept { - return get_string_view(); + return tape.get_string_view(); } inline uint32_t object::iterator::key_length() const noexcept { - return get_string_length(); + return tape.get_string_length(); } inline const char* object::iterator::key_c_str() const noexcept { - return reinterpret_cast(&doc->string_buf[size_t(tape_value()) + sizeof(uint32_t)]); + return reinterpret_cast(&tape.doc->string_buf[size_t(tape.tape_value()) + sizeof(uint32_t)]); } inline element object::iterator::value() const noexcept { - return element(doc, json_index + 1); + return element(internal::tape_ref(tape.doc, tape.json_index + 1)); } /** @@ -6044,7 +6620,7 @@ inline std::ostream& operator<<(std::ostream& out, const key_value_pair &value) } // namespace dom template<> -inline std::ostream& minify::print(std::ostream& out) { +inline std::ostream& minifier::print(std::ostream& out) { out << '{'; auto pair = value.begin(); auto end = value.end(); @@ -6058,14 +6634,14 @@ inline std::ostream& minify::print(std::ostream& out) { } template<> -inline std::ostream& minify::print(std::ostream& out) { +inline std::ostream& minifier::print(std::ostream& out) { return out << '"' << internal::escape_json_string(value.key) << "\":" << value.value; } #if SIMDJSON_EXCEPTIONS template<> -inline std::ostream& minify>::print(std::ostream& out) { +inline std::ostream& minifier>::print(std::ostream& out) { if (value.error()) { throw simdjson_error(value.error()); } return out << minify(value.first); } @@ -6786,23 +7362,21 @@ inline simdjson_result parser::read_file(const std::string &path) noexce inline simdjson_result parser::load(const std::string &path) & noexcept { size_t len; - error_code code; - read_file(path).tie(len, code); - if (code) { return code; } - + auto _error = read_file(path).get(len); + if (_error) { return _error; } return parse(loaded_bytes.get(), len, false); } -inline document_stream parser::load_many(const std::string &path, size_t batch_size) noexcept { +inline simdjson_result parser::load_many(const std::string &path, size_t batch_size) noexcept { size_t len; - error_code code; - read_file(path).tie(len, code); - return document_stream(*this, (const uint8_t*)loaded_bytes.get(), len, batch_size, code); + auto _error = read_file(path).get(len); + if (_error) { return _error; } + return document_stream(*this, (const uint8_t*)loaded_bytes.get(), len, batch_size); } inline simdjson_result parser::parse(const uint8_t *buf, size_t len, bool realloc_if_needed) & noexcept { - error_code code = ensure_capacity(len); - if (code) { return code; } + error_code _error = ensure_capacity(len); + if (_error) { return _error; } if (realloc_if_needed) { const uint8_t *tmp_buf = buf; @@ -6812,11 +7386,11 @@ inline simdjson_result parser::parse(const uint8_t *buf, size_t len, bo memcpy((void *)buf, tmp_buf, len); } - code = implementation->parse(buf, len, doc); + _error = implementation->parse(buf, len, doc); if (realloc_if_needed) { aligned_free((void *)buf); // must free before we exit } - if (code) { return code; } + if (_error) { return _error; } return doc.root(); } @@ -6830,16 +7404,16 @@ really_inline simdjson_result parser::parse(const padded_string &s) & n return parse(s.data(), s.length(), false); } -inline document_stream parser::parse_many(const uint8_t *buf, size_t len, size_t batch_size) noexcept { +inline simdjson_result parser::parse_many(const uint8_t *buf, size_t len, size_t batch_size) noexcept { return document_stream(*this, buf, len, batch_size); } -inline document_stream parser::parse_many(const char *buf, size_t len, size_t batch_size) noexcept { +inline simdjson_result parser::parse_many(const char *buf, size_t len, size_t batch_size) noexcept { return parse_many((const uint8_t *)buf, len, batch_size); } -inline document_stream parser::parse_many(const std::string &s, size_t batch_size) noexcept { +inline simdjson_result parser::parse_many(const std::string &s, size_t batch_size) noexcept { return parse_many(s.data(), s.length(), batch_size); } -inline document_stream parser::parse_many(const padded_string &s, size_t batch_size) noexcept { +inline simdjson_result parser::parse_many(const padded_string &s, size_t batch_size) noexcept { return parse_many(s.data(), s.length(), batch_size); } @@ -6859,15 +7433,22 @@ inline error_code parser::allocate(size_t capacity, size_t max_depth) noexcept { // Reallocate implementation and document if needed // error_code err; + // + // It is possible that we change max_depth without touching capacity, in + // which case, we do not want to reallocate the document buffers. + // + bool need_doc_allocation{false}; if (implementation) { + need_doc_allocation = implementation->capacity() != capacity || !doc.tape; err = implementation->allocate(capacity, max_depth); } else { + need_doc_allocation = true; err = simdjson::active_implementation->create_dom_parser_implementation(capacity, max_depth, implementation); } if (err) { return err; } - - if (implementation->capacity() != capacity || !doc.tape) { - return doc.allocate(capacity); + if (need_doc_allocation) { + err = doc.allocate(capacity); + if (err) { return err; } } return SUCCESS; } diff --git a/tests/allparserscheckfile.cpp b/tests/allparserscheckfile.cpp index 6784a0603..959268e63 100644 --- a/tests/allparserscheckfile.cpp +++ b/tests/allparserscheckfile.cpp @@ -63,9 +63,10 @@ int main(int argc, char *argv[]) { exit(1); } const char *filename = argv[optind]; - auto [p, loaderr] = simdjson::padded_string::load(filename); - if (loaderr) { - std::cerr << "Could not load the file " << filename << ": " << loaderr << std::endl; + simdjson::padded_string p; + auto error = simdjson::padded_string::load(filename).get(p); + if (error) { + std::cerr << "Could not load the file " << filename << ": " << error << std::endl; return EXIT_FAILURE; } if (verbose) { @@ -79,7 +80,7 @@ int main(int argc, char *argv[]) { std::cout << std::endl; } simdjson::dom::parser parser; - auto err = parser.parse(p).error(); + error = parser.parse(p).error(); rapidjson::Document d; @@ -95,19 +96,19 @@ int main(int argc, char *argv[]) { .is_valid(); if (just_favorites) { printf("our parser : %s \n", - (err == simdjson::error_code::SUCCESS) ? "correct" : "invalid"); + (error == simdjson::error_code::SUCCESS) ? "correct" : "invalid"); printf("rapid (check encoding) : %s \n", rapid_correct_checkencoding ? "correct" : "invalid"); printf("sajson : %s \n", sajson_correct ? "correct" : "invalid"); - if (err == simdjson::DEPTH_ERROR) { + if (error == simdjson::DEPTH_ERROR) { printf("simdjson encountered a DEPTH_ERROR, it was parametrized to " "reject documents with depth exceeding %zu.\n", parser.max_depth()); } - if (((err == simdjson::error_code::SUCCESS) != rapid_correct_checkencoding) || + if (((error == simdjson::error_code::SUCCESS) != rapid_correct_checkencoding) || (rapid_correct_checkencoding != sajson_correct) || - ((err == simdjson::SUCCESS) != sajson_correct)) { + ((error == simdjson::SUCCESS) != sajson_correct)) { printf("WARNING: THEY DISAGREE\n\n"); return EXIT_FAILURE; } diff --git a/tests/basictests.cpp b/tests/basictests.cpp index 222643857..2d5663d7a 100644 --- a/tests/basictests.cpp +++ b/tests/basictests.cpp @@ -17,6 +17,8 @@ #include "cast_tester.h" #include "test_macros.h" +const size_t AMAZON_CELLPHONES_NDJSON_DOC_COUNT = 793; + namespace number_tests { // ulp distance @@ -39,8 +41,8 @@ namespace number_tests { for (int m = 10; m < 20; m++) { for (int i = -1024; i < 1024; i++) { auto str = std::to_string(i); - auto [actual, error] = parser.parse(str).get(); - if (error) { std::cerr << error << std::endl; return false; } + int64_t actual; + ASSERT_SUCCESS(parser.parse(str).get(actual)); if (actual != i) { std::cerr << "JSON '" << str << "' parsed to " << actual << " instead of " << i << std::endl; return false; @@ -60,7 +62,8 @@ namespace number_tests { size_t n = snprintf(buf, sizeof(buf), "%.*e", std::numeric_limits::max_digits10 - 1, expected); if (n >= sizeof(buf)) { abort(); } fflush(NULL); - auto [actual, error] = parser.parse(buf, n).get(); + double actual; + auto error = parser.parse(buf, n).get(actual); if (error) { std::cerr << error << std::endl; return false; } uint64_t ulp = f64_ulp_dist(actual,expected); if(ulp > maxulp) maxulp = ulp; @@ -154,7 +157,8 @@ namespace number_tests { if (n >= sizeof(buf)) { abort(); } fflush(NULL); - auto [actual, error] = parser.parse(buf, n).get(); + double actual; + auto error = parser.parse(buf, n).get(actual); if (error) { std::cerr << error << std::endl; return false; } double expected = ((i >= -307) ? testing_power_of_ten[i + 307]: std::pow(10, i)); int ulp = (int) f64_ulp_dist(actual, expected); @@ -174,17 +178,14 @@ namespace number_tests { } namespace document_tests { - int issue938() { + bool issue938() { std::vector json_strings{"[true,false]", "[1,2,3,null]", R"({"yay":"json!"})"}; simdjson::dom::parser parser1; for (simdjson::padded_string str : json_strings) { - auto [element, error] = parser1.parse(str); - if(error) { - std::cerr << error << std::endl; - } else { - std::cout << element << std::endl; - } + simdjson::dom::element element; + ASSERT_SUCCESS( parser1.parse(str).get(element) ); + std::cout << element << std::endl; } std::vector file_paths{ ADVERSARIAL_JSON, FLATADVERSARIAL_JSON, DEMO_JSON, @@ -192,23 +193,17 @@ namespace document_tests { TRUENULL_JSON}; for (auto path : file_paths) { simdjson::dom::parser parser2; + simdjson::dom::element element; std::cout << "file: " << path << std::endl; - auto [element, error] = parser2.load(path); - if(error) { - std::cerr << error << std::endl; - } else { - std::cout << element.type() << std::endl; - } + ASSERT_SUCCESS( parser2.load(path).get(element) ); + std::cout << element.type() << std::endl; } simdjson::dom::parser parser3; for (auto path : file_paths) { + simdjson::dom::element element; std::cout << "file: " << path << std::endl; - auto [element, error] = parser3.load(path); - if(error) { - std::cerr << error << std::endl; - } else { - std::cout << element.type() << std::endl; - } + ASSERT_SUCCESS( parser3.load(path).get(element) ); + std::cout << element.type() << std::endl; } return true; } @@ -218,52 +213,34 @@ namespace document_tests { std::cout << __func__ << std::endl; simdjson::padded_string badjson = "[7,7,7,7,6,7,7,7,6,7,7,6,[7,7,7,7,6,7,7,7,6,7,7,6,7,7,7,7,7,7,6"_padded; simdjson::dom::parser parser; - auto error = parser.parse(badjson).error(); - if (!error) { - printf("This json should not be valid %s.\n", badjson.data()); - return false; - } + ASSERT_ERROR( parser.parse(badjson), simdjson::TAPE_ERROR ); return true; } bool count_array_example() { std::cout << __func__ << std::endl; simdjson::padded_string smalljson = "[1,2,3]"_padded; simdjson::dom::parser parser; - auto [doc, error] = parser.parse(smalljson).get(); - if (error) { - printf("This json should be valid %s.\n", smalljson.data()); - return false; - } - if(doc.size() != 3) { - printf("This json should have size three but found %zu : %s.\n", doc.size(), smalljson.data()); - return false; - } + simdjson::dom::array array; + ASSERT_SUCCESS( parser.parse(smalljson).get(array) ); + ASSERT_EQUAL( array.size(), 3 ); return true; } bool count_object_example() { std::cout << __func__ << std::endl; simdjson::padded_string smalljson = "{\"1\":1,\"2\":1,\"3\":1}"_padded; simdjson::dom::parser parser; - auto [doc, error] = parser.parse(smalljson).get(); - if (error) { - printf("This json should be valid %s.\n", smalljson.data()); - return false; - } - if(doc.size() != 3) { - printf("This json should have size three but found %zu : %s.\n", doc.size(), smalljson.data()); - return false; - } + simdjson::dom::object object; + ASSERT_SUCCESS( parser.parse(smalljson).get(object) ); + ASSERT_EQUAL( object.size(), 3 ); return true; } bool padded_with_open_bracket() { std::cout << __func__ << std::endl; simdjson::dom::parser parser; // This is an invalid document padded with open braces. - auto error1 = parser.parse("[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[", 2, false).error(); - if (!error1) { std::cerr << "We expected an error but got: " << error1 << std::endl; return false; } + ASSERT_ERROR( parser.parse("[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[", 2, false), simdjson::TAPE_ERROR); // This is a valid document padded with open braces. - auto error2 = parser.parse("[][[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[", 2, false).error(); - if (error2) { std::cerr << "Error: " << error2 << std::endl; return false; } + ASSERT_SUCCESS( parser.parse("[][[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[", 2, false) ); return true; } // returns true if successful @@ -385,7 +362,9 @@ namespace document_tests { namespace document_stream_tests { static simdjson::dom::document_stream parse_many_stream_return(simdjson::dom::parser &parser, simdjson::padded_string &str) { - return parser.parse_many(str); + simdjson::dom::document_stream stream; + UNUSED auto error = parser.parse_many(str).get(stream); + return stream; } // this is a compilation test UNUSED static void parse_many_stream_assign() { @@ -402,15 +381,13 @@ namespace document_stream_tests { } simdjson::dom::parser parser; const size_t window = 32; // deliberately small - auto stream = parser.parse_many(json,window); + simdjson::dom::document_stream stream; + ASSERT_SUCCESS( parser.parse_many(json,window).get(stream) ); auto i = stream.begin(); size_t count = 0; for(; i != stream.end(); ++i) { auto doc = *i; - if (doc.error()) { - std::cerr << doc.error() << std::endl; - return false; - } + ASSERT_SUCCESS(doc); if( i.current_index() != count) { std::cout << "index:" << i.current_index() << std::endl; std::cout << "expected index:" << count << std::endl; @@ -426,7 +403,9 @@ namespace document_stream_tests { simdjson::dom::parser parser; size_t count = 0; size_t window_size = 10; // deliberately too small - for (auto doc : parser.parse_many(json, window_size)) { + simdjson::dom::document_stream stream; + ASSERT_SUCCESS( parser.parse_many(json, window_size).get(stream) ); + for (auto doc : stream) { if (!doc.error()) { std::cerr << "Expected a capacity error " << doc.error() << std::endl; return false; @@ -447,7 +426,9 @@ namespace document_stream_tests { simdjson::dom::parser parser; size_t count = 0; uint64_t window_size{17179869184}; // deliberately too big - for (auto doc : parser.parse_many(json, size_t(window_size))) { + simdjson::dom::document_stream stream; + ASSERT_SUCCESS( parser.parse_many(json, size_t(window_size)).get(stream) ); + for (auto doc : stream) { if (!doc.error()) { std::cerr << "I expected a failure (too big) but got " << doc.error() << std::endl; return false; @@ -460,7 +441,9 @@ namespace document_stream_tests { static bool parse_json_message_issue467(simdjson::padded_string &json, size_t expectedcount) { simdjson::dom::parser parser; size_t count = 0; - for (auto doc : parser.parse_many(json)) { + simdjson::dom::document_stream stream; + ASSERT_SUCCESS( parser.parse_many(json).get(stream) ); + for (auto doc : stream) { if (doc.error()) { std::cerr << "Failed with simdjson error= " << doc.error() << std::endl; return false; @@ -510,22 +493,12 @@ namespace document_stream_tests { simdjson::padded_string str(data); simdjson::dom::parser parser; size_t count = 0; - for (auto [doc, error] : parser.parse_many(str, batch_size)) { - if (error) { - printf("Error at on document %zd at batch size %zu: %s\n", count, batch_size, simdjson::error_message(error)); - return false; - } - - auto [keyid, error2] = doc["id"].get(); - if (error2) { - printf("Error getting id as int64 on document %zd at batch size %zu: %s\n", count, batch_size, simdjson::error_message(error2)); - return false; - } - - if (keyid != int64_t(count)) { - printf("key does not match %" PRId64 ", expected %zd on document %zd at batch size %zu\n", keyid, count, count, batch_size); - return false; - } + simdjson::dom::document_stream stream; + ASSERT_SUCCESS( parser.parse_many(str, batch_size).get(stream) ); + for (auto doc : stream) { + int64_t keyid; + ASSERT_SUCCESS( doc["id"].get(keyid) ); + ASSERT_EQUAL( keyid, int64_t(count) ); count++; } @@ -560,29 +533,16 @@ namespace document_stream_tests { simdjson::padded_string str(data); simdjson::dom::parser parser; size_t count = 0; - for (auto [doc, error] : parser.parse_many(str, batch_size)) { - if (error) { - printf("Error at on document %zd at batch size %zu: %s\n", count, batch_size, simdjson::error_message(error)); - return false; - } - - auto [keyid, error2] = doc["id"].get(); - if (error2) { - printf("Error getting id as int64 on document %zd at batch size %zu: %s\n", count, batch_size, simdjson::error_message(error2)); - return false; - } - - if (keyid != int64_t(count)) { - printf("key does not match %" PRId64 ", expected %zd on document %zd at batch size %zu\n", keyid, count, count, batch_size); - return false; - } + simdjson::dom::document_stream stream; + ASSERT_SUCCESS( parser.parse_many(str, batch_size).get(stream) ); + for (auto doc : stream) { + int64_t keyid; + ASSERT_SUCCESS( doc["id"].get(keyid) ); + ASSERT_EQUAL( keyid, int64_t(count) ); count++; } - if(count != n_records) { - printf("Found wrong number of documents %zd, expected %zd at batch size %zu\n", count, n_records, batch_size); - return false; - } + ASSERT_EQUAL( count, n_records ) } printf("ok\n"); return true; @@ -610,32 +570,52 @@ namespace parse_api_tests { bool parser_parse() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; - auto [doc, error] = parser.parse(BASIC_JSON); - if (error) { cerr << error << endl; return false; } - if (!doc.is()) { cerr << "Document did not parse as an array" << endl; return false; } + dom::element doc; + ASSERT_SUCCESS( parser.parse(BASIC_JSON).get(doc) ); + ASSERT_EQUAL( doc.is(), true ); return true; } bool parser_parse_many() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; int count = 0; - for (auto [doc, error] : parser.parse_many(BASIC_NDJSON)) { - if (error) { cerr << "Error in parse_many: " << endl; return false; } - if (!doc.is()) { cerr << "Document did not parse as an array" << endl; return false; } + simdjson::dom::document_stream stream; + ASSERT_SUCCESS( parser.parse_many(BASIC_NDJSON).get(stream) ); + for (auto doc : stream) { + UNUSED dom::array array; + ASSERT_SUCCESS( doc.get(array) ); count++; } - if (count != 2) { cerr << "parse_many returned " << count << " documents, expected 2" << endl; return false; } + ASSERT_EQUAL(count, 2); return true; } + + SIMDJSON_PUSH_DISABLE_WARNINGS + SIMDJSON_DISABLE_DEPRECATED_WARNING + bool parser_parse_many_deprecated() { + std::cout << "Running " << __func__ << std::endl; + dom::parser parser; + int count = 0; + for (auto doc : parser.parse_many(BASIC_NDJSON)) { + UNUSED dom::array array; + ASSERT_SUCCESS( doc.get(array) ); + count++; + } + ASSERT_EQUAL(count, 2); + return true; + } + SIMDJSON_POP_DISABLE_WARNINGS bool parser_parse_many_empty() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; int count = 0; - for (auto doc : parser.parse_many(EMPTY_NDJSON)) { - if (doc.error()) { cerr << "Error in parse_many: " << doc.error() << endl; return false; } + simdjson::dom::document_stream stream; + ASSERT_SUCCESS( parser.parse_many(EMPTY_NDJSON).get(stream) ); + for (auto doc : stream) { + ASSERT_SUCCESS( doc ); count++; } - if (count != 0) { cerr << "parse_many returned " << count << " documents, expected 0" << endl; return false; } + ASSERT_EQUAL(count, 0); return true; } @@ -649,92 +629,106 @@ namespace parse_api_tests { memcpy(&empty_batches_ndjson[BATCH_SIZE*3+2], "1", 1); memcpy(&empty_batches_ndjson[BATCH_SIZE*10+4], "2", 1); memcpy(&empty_batches_ndjson[BATCH_SIZE*11+6], "3", 1); - for (auto [doc, error] : parser.parse_many(empty_batches_ndjson, BATCH_SIZE*16)) { - if (error) { cerr << "Error in parse_many: " << error << endl; return false; } + simdjson::dom::document_stream stream; + ASSERT_SUCCESS( parser.parse_many(empty_batches_ndjson, BATCH_SIZE*16).get(stream) ); + for (auto doc : stream) { count++; - auto [val, val_error] = doc.get(); - if (val_error) { cerr << "Document is not an unsigned int: " << val_error << endl; return false; } - if (val != count) { cerr << "Expected document #" << count << " to equal " << count << ", but got " << val << " instead!" << endl; return false; } + uint64_t val; + ASSERT_SUCCESS( doc.get(val) ); + ASSERT_EQUAL( val, count ); } - if (count != 3) { cerr << "parse_many returned " << count << " documents, expected 0" << endl; return false; } + ASSERT_EQUAL(count, 3); return true; } bool parser_load() { std::cout << "Running " << __func__ << " on " << TWITTER_JSON << std::endl; dom::parser parser; - auto [doc, error] = parser.load(TWITTER_JSON); - if (error) { cerr << error << endl; return false; } - if (!doc.is()) { cerr << "Document did not parse as an object" << endl; return false; } + dom::object object; + ASSERT_SUCCESS( parser.load(TWITTER_JSON).get(object) ); return true; } bool parser_load_many() { std::cout << "Running " << __func__ << " on " << AMAZON_CELLPHONES_NDJSON << std::endl; dom::parser parser; int count = 0; - for (auto [doc, error] : parser.load_many(AMAZON_CELLPHONES_NDJSON)) { - if (error) { cerr << error << endl; return false; } - + simdjson::dom::document_stream stream; + ASSERT_SUCCESS( parser.load_many(AMAZON_CELLPHONES_NDJSON).get(stream) ); + for (auto doc : stream) { dom::array arr; - error = doc.get(arr); // let us get the array - if (error) { cerr << error << endl; return false; } + ASSERT_SUCCESS( doc.get(arr) ); // let us get the array + ASSERT_EQUAL( arr.size(), 9 ); - if(arr.size() != 9) { cerr << "bad array size"<< endl; return false; } - - size_t c = 0; - for(auto v : arr) { c++; (void)v; } - if(c != 9) { cerr << "mismatched array size"<< endl; return false; } + size_t arr_count = 0; + for (auto v : arr) { arr_count++; (void)v; } + ASSERT_EQUAL( arr_count, 9 ); count++; } - if (count != 793) { cerr << "Expected 793 documents, but load_many loaded " << count << " documents." << endl; return false; } + ASSERT_EQUAL(count, AMAZON_CELLPHONES_NDJSON_DOC_COUNT); return true; } + SIMDJSON_PUSH_DISABLE_WARNINGS + SIMDJSON_DISABLE_DEPRECATED_WARNING + bool parser_load_many_deprecated() { + std::cout << "Running " << __func__ << " on " << AMAZON_CELLPHONES_NDJSON << std::endl; + dom::parser parser; + int count = 0; + for (auto doc : parser.load_many(AMAZON_CELLPHONES_NDJSON)) { + dom::array arr; + ASSERT_SUCCESS( doc.get(arr) ); + ASSERT_EQUAL( arr.size(), 9 ); + + size_t arr_count = 0; + for (auto v : arr) { arr_count++; (void)v; } + ASSERT_EQUAL( arr_count, 9 ); + + count++; + } + ASSERT_EQUAL( count, AMAZON_CELLPHONES_NDJSON_DOC_COUNT ); + return true; + } + SIMDJSON_POP_DISABLE_WARNINGS + #if SIMDJSON_EXCEPTIONS bool parser_parse_exception() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; - element doc = parser.parse(BASIC_JSON); - if (!doc.is()) { cerr << "Document did not parse as an array" << endl; return false; } + UNUSED dom::array array = parser.parse(BASIC_JSON); return true; } bool parser_parse_many_exception() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; int count = 0; - for (const element doc : parser.parse_many(BASIC_NDJSON)) { - if (!doc.is()) { cerr << "Document did not parse as an array" << endl; return false; } + for (UNUSED dom::array doc : parser.parse_many(BASIC_NDJSON)) { count++; } - if (count != 2) { cerr << "parse_many returned " << count << " documents, expected 2" << endl; return false; } + ASSERT_EQUAL(count, 2); return true; } bool parser_load_exception() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; - const element doc = parser.load(TWITTER_JSON); - if (!doc.is()) { cerr << "Document did not parse as an object" << endl; return false; } - size_t c = 0; - dom::object obj = doc.get().value(); // let us get the object - for (auto x : obj) { - c++; - (void) x; + size_t count = 0; + dom::object object = parser.load(TWITTER_JSON); + for (UNUSED auto field : object) { + count++; } - if(c != obj.size()) { cerr << "Mismatched size" << endl; return false; } + ASSERT_EQUAL( count, object.size() ); return true; } bool parser_load_many_exception() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; int count = 0; - for (const element doc : parser.load_many(AMAZON_CELLPHONES_NDJSON)) { - if (!doc.is()) { cerr << "Document did not parse as an array" << endl; return false; } + for (UNUSED dom::array doc : parser.load_many(AMAZON_CELLPHONES_NDJSON)) { count++; } - if (count != 793) { cerr << "Expected 1 document, but load_many loaded " << count << " documents." << endl; return false; } + ASSERT_EQUAL( count, AMAZON_CELLPHONES_NDJSON_DOC_COUNT ); return true; } #endif @@ -742,10 +736,12 @@ namespace parse_api_tests { bool run() { return parser_parse() && parser_parse_many() && + parser_parse_many_deprecated() && parser_parse_many_empty() && parser_parse_many_empty_batches() && parser_load() && parser_load_many() && + parser_load_many_deprecated() && #if SIMDJSON_EXCEPTIONS parser_parse_exception() && parser_parse_many_exception() && @@ -871,16 +867,17 @@ namespace dom_api_tests { string json(R"({ "a": 1, "b": 2, "c": 3 })"); const char* expected_key[] = { "a", "b", "c" }; uint64_t expected_value[] = { 1, 2, 3 }; - int i = 0; dom::parser parser; - auto [object, error] = parser.parse(json).get(); - if (error) { cerr << "Error: " << error << endl; return false; } + dom::object object; + ASSERT_SUCCESS( parser.parse(json).get(object) ); + int i = 0; for (auto [key, value] : object) { - if (key != expected_key[i] || value.get().first != expected_value[i]) { cerr << "Expected " << expected_key[i] << " = " << expected_value[i] << ", got " << key << "=" << value << endl; return false; } + ASSERT_EQUAL( key, expected_key[i] ); + ASSERT_EQUAL( value.get().first, expected_value[i] ); i++; } - if (i*sizeof(uint64_t) != sizeof(expected_value)) { cout << "Expected " << sizeof(expected_value) << " values, got " << i << endl; return false; } + ASSERT_EQUAL( i*sizeof(uint64_t), sizeof(expected_value) ); return true; } @@ -888,16 +885,18 @@ namespace dom_api_tests { std::cout << "Running " << __func__ << std::endl; string json(R"([ 1, 10, 100 ])"); uint64_t expected_value[] = { 1, 10, 100 }; - int i=0; dom::parser parser; - auto [array, error] = parser.parse(json).get(); - if (error) { cerr << "Error: " << error << endl; return false; } + dom::array array; + ASSERT_SUCCESS( parser.parse(json).get(array) ); + int i=0; for (auto value : array) { - if (value.get().first != expected_value[i]) { cerr << "Expected " << expected_value[i] << ", got " << value << endl; return false; } + uint64_t v; + ASSERT_SUCCESS( value.get(v) ); + ASSERT_EQUAL( v, expected_value[i] ); i++; } - if (i*sizeof(uint64_t) != sizeof(expected_value)) { cout << "Expected " << sizeof(expected_value) << " values, got " << i << endl; return false; } + ASSERT_EQUAL( i*sizeof(uint64_t), sizeof(expected_value) ); return true; } @@ -907,13 +906,13 @@ namespace dom_api_tests { int i = 0; dom::parser parser; - auto [object, error] = parser.parse(json).get(); - if (error) { cerr << "Error: " << error << endl; return false; } - for (auto [key, value] : object) { - cout << "Unexpected " << key << " = " << value << endl; + dom::object object; + ASSERT_SUCCESS( parser.parse(json).get(object) ); + for (UNUSED auto field : object) { + TEST_FAIL("Unexpected field"); i++; } - if (i > 0) { cout << "Expected 0 values, got " << i << endl; return false; } + ASSERT_EQUAL(i, 0); return true; } @@ -923,13 +922,13 @@ namespace dom_api_tests { int i=0; dom::parser parser; - auto [array, error] = parser.parse(json).get(); - if (error) { cerr << "Error: " << error << endl; return false; } - for (auto value : array) { - cout << "Unexpected value " << value << endl; + dom::array array; + ASSERT_SUCCESS( parser.parse(json).get(array) ); + for (UNUSED auto value : array) { + TEST_FAIL("Unexpected value"); i++; } - if (i > 0) { cout << "Expected 0 values, got " << i << endl; return false; } + ASSERT_EQUAL(i, 0); return true; } @@ -937,13 +936,18 @@ namespace dom_api_tests { std::cout << "Running " << __func__ << std::endl; string json(R"([ "hi", "has backslash\\" ])"); dom::parser parser; - auto [array, error] = parser.parse(json).get(); - if (error) { cerr << "Error: " << error << endl; return false; } - auto val = array.begin(); + dom::array array; + ASSERT_SUCCESS( parser.parse(json).get(array) ); + + auto iter = array.begin(); + std::string_view val; + ASSERT_SUCCESS( (*iter).get(val) ); + ASSERT_EQUAL( val, "hi" ); + + ++iter; + ASSERT_SUCCESS( (*iter).get(val) ); + ASSERT_EQUAL( val, "has backslash\\" ); - if ((*val).get().first != "hi") { cerr << "Expected value to be \"hi\", was " << (*val).get().first << endl; return false; } - ++val; - if ((*val).get().first != "has backslash\\") { cerr << "Expected string_view(\"has backslash\\\\\") to be \"has backslash\\\", was " << (*val).get().first << endl; return false; } return true; } @@ -951,22 +955,22 @@ namespace dom_api_tests { std::cout << "Running " << __func__ << std::endl; string json(R"([ 0, 1, -1, 1.1 ])"); dom::parser parser; - auto [array, error] = parser.parse(json).get(); - if (error) { cerr << "Error: " << error << endl; return false; } - auto val = array.begin(); + dom::array array; + ASSERT_SUCCESS( parser.parse(json).get(array) ); - if ((*val).get().first != 0) { cerr << "Expected uint64_t(0) to be 0, was " << (*val) << endl; return false; } - if ((*val).get().first != 0) { cerr << "Expected int64_t(0) to be 0, was " << (*val).get().first << endl; return false; } - if ((*val).get().first != 0) { cerr << "Expected double(0) to be 0, was " << (*val).get().first << endl; return false; } - ++val; - if ((*val).get().first != 1) { cerr << "Expected uint64_t(1) to be 1, was " << (*val) << endl; return false; } - if ((*val).get().first != 1) { cerr << "Expected int64_t(1) to be 1, was " << (*val).get().first << endl; return false; } - if ((*val).get().first != 1) { cerr << "Expected double(1) to be 1, was " << (*val).get().first << endl; return false; } - ++val; - if ((*val).get().first != -1) { cerr << "Expected int64_t(-1) to be -1, was " << (*val).get().first << endl; return false; } - if ((*val).get().first != -1) { cerr << "Expected double(-1) to be -1, was " << (*val).get().first << endl; return false; } - ++val; - if ((*val).get().first != 1.1) { cerr << "Expected double(1.1) to be 1.1, was " << (*val).get().first << endl; return false; } + auto iter = array.begin(); + ASSERT_EQUAL( (*iter).get().first, 0 ); + ASSERT_EQUAL( (*iter).get().first, 0 ); + ASSERT_EQUAL( (*iter).get().first, 0 ); + ++iter; + ASSERT_EQUAL( (*iter).get().first, 1 ); + ASSERT_EQUAL( (*iter).get().first, 1 ); + ASSERT_EQUAL( (*iter).get().first, 1 ); + ++iter; + ASSERT_EQUAL( (*iter).get().first, -1 ); + ASSERT_EQUAL( (*iter).get().first, -1 ); + ++iter; + ASSERT_EQUAL( (*iter).get().first, 1.1 ); return true; } @@ -974,13 +978,13 @@ namespace dom_api_tests { std::cout << "Running " << __func__ << std::endl; string json(R"([ true, false ])"); dom::parser parser; - auto [array, error] = parser.parse(json).get(); - if (error) { cerr << "Error: " << error << endl; return false; } - auto val = array.begin(); + dom::array array; + ASSERT_SUCCESS( parser.parse(json).get(array) ); - if ((*val).get().first != true) { cerr << "Expected bool(true) to be true, was " << (*val) << endl; return false; } + auto val = array.begin(); + ASSERT_EQUAL( (*val).get().first, true ); ++val; - if ((*val).get().first != false) { cerr << "Expected bool(false) to be false, was " << (*val) << endl; return false; } + ASSERT_EQUAL( (*val).get().first, false ); return true; } @@ -988,10 +992,11 @@ namespace dom_api_tests { std::cout << "Running " << __func__ << std::endl; string json(R"([ null ])"); dom::parser parser; - auto [array, error] = parser.parse(json).get(); - if (error) { cerr << "Error: " << error << endl; return false; } + dom::array array; + ASSERT_SUCCESS( parser.parse(json).get(array) ); + auto val = array.begin(); - if (!(*val).is_null()) { cerr << "Expected null to be null!" << endl; return false; } + ASSERT_EQUAL( !(*val).is_null(), 0 ); return true; } @@ -999,27 +1004,29 @@ namespace dom_api_tests { std::cout << "Running " << __func__ << std::endl; string json(R"({ "a": 1, "b": 2, "c/d": 3})"); dom::parser parser; - auto [doc, error] = parser.parse(json); - if (doc["a"].get().first != 1) { cerr << "Expected uint64_t(doc[\"a\"]) to be 1, was " << doc["a"].first << endl; return false; } - if (doc["b"].get().first != 2) { cerr << "Expected uint64_t(doc[\"b\"]) to be 2, was " << doc["b"].first << endl; return false; } - if (doc["c/d"].get().first != 3) { cerr << "Expected uint64_t(doc[\"c/d\"]) to be 3, was " << doc["c"].first << endl; return false; } + dom::object object; + ASSERT_SUCCESS( parser.parse(json).get(object) ); + ASSERT_EQUAL( object["a"].get().first, 1 ); + ASSERT_EQUAL( object["b"].get().first, 2 ); + ASSERT_EQUAL( object["c/d"].get().first, 3 ); // Check all three again in backwards order, to ensure we can go backwards - if (doc["c/d"].get().first != 3) { cerr << "Expected uint64_t(doc[\"c/d\"]) to be 3, was " << doc["c"].first << endl; return false; } - if (doc["b"].get().first != 2) { cerr << "Expected uint64_t(doc[\"b\"]) to be 2, was " << doc["b"].first << endl; return false; } - if (doc["a"].get().first != 1) { cerr << "Expected uint64_t(doc[\"a\"]) to be 1, was " << doc["a"].first << endl; return false; } + ASSERT_EQUAL( object["c/d"].get().first, 3 ); + ASSERT_EQUAL( object["b"].get().first, 2 ); + ASSERT_EQUAL( object["a"].get().first, 1 ); + simdjson::error_code error; UNUSED element val; #ifndef _LIBCPP_VERSION // should work everywhere but with libc++, must include the header. - std::tie(val,error) = doc["d"]; - if (error != simdjson::NO_SUCH_FIELD) { cerr << "Expected NO_SUCH_FIELD error for uint64_t(doc[\"d\"]), got " << error << endl; return false; } - std::tie(std::ignore,error) = doc["d"]; - if (error != simdjson::NO_SUCH_FIELD) { cerr << "Expected NO_SUCH_FIELD error for uint64_t(doc[\"d\"]), got " << error << endl; return false; } + std::tie(val,error) = object["d"]; + ASSERT_ERROR( error, NO_SUCH_FIELD ); + std::tie(std::ignore,error) = object["d"]; + ASSERT_ERROR( error, NO_SUCH_FIELD ); #endif - // tie(val, error) = doc["d"]; fails with "no viable overloaded '='" on Apple clang version 11.0.0 tie(val, error) = doc["d"]; - doc["d"].tie(val, error); - if (error != simdjson::NO_SUCH_FIELD) { cerr << "Expected NO_SUCH_FIELD error for uint64_t(doc[\"d\"]), got " << error << endl; return false; } - if (doc["d"].get(val) != simdjson::NO_SUCH_FIELD) { cerr << "Expected NO_SUCH_FIELD error for uint64_t(doc[\"d\"]), got " << error << endl; return false; } - if (doc["d"].error() != simdjson::NO_SUCH_FIELD) { cerr << "Expected NO_SUCH_FIELD error for uint64_t(doc[\"d\"]), got " << error << endl; return false; } + // tie(val, error) = object["d"]; fails with "no viable overloaded '='" on Apple clang version 11.0.0 tie(val, error) = doc["d"]; + object["d"].tie(val, error); + ASSERT_ERROR( error, NO_SUCH_FIELD ); + ASSERT_ERROR( object["d"].get(val), NO_SUCH_FIELD ); + ASSERT_ERROR( object["d"], NO_SUCH_FIELD ); return true; } @@ -1027,26 +1034,25 @@ namespace dom_api_tests { std::cout << "Running " << __func__ << std::endl; string json(R"({ "obj": { "a": 1, "b": 2, "c/d": 3 } })"); dom::parser parser; - auto [doc, error] = parser.parse(json); - if (error) { cerr << "Error: " << error << endl; return false; } - if (doc["obj"]["a"].get().first != 1) { cerr << "Expected uint64_t(doc[\"obj\"][\"a\"]) to be 1, was " << doc["obj"]["a"].first << endl; return false; } + dom::element doc; + ASSERT_SUCCESS( parser.parse(json).get(doc) ); + ASSERT_EQUAL( doc["obj"]["a"].get().first, 1); object obj; - error = doc.get(obj); // tie(...) = fails with "no viable overloaded '='" on Apple clang version 11.0.0 - if (error) { cerr << "Error: " << error << endl; return false; } - if (obj["obj"]["a"].get().first != 1) { cerr << "Expected uint64_t(doc[\"obj\"][\"a\"]) to be 1, was " << doc["obj"]["a"].first << endl; return false; } + ASSERT_SUCCESS( doc.get(obj) ); + ASSERT_EQUAL( obj["obj"]["a"].get().first, 1); - error = obj["obj"].get(obj); // tie(...) = fails with "no viable overloaded '='" on Apple clang version 11.0.0 - if (obj["a"].get().first != 1) { cerr << "Expected uint64_t(obj[\"a\"]) to be 1, was " << obj["a"].first << endl; return false; } - if (obj["b"].get().first != 2) { cerr << "Expected uint64_t(obj[\"b\"]) to be 2, was " << obj["b"].first << endl; return false; } - if (obj["c/d"].get().first != 3) { cerr << "Expected uint64_t(obj[\"c\"]) to be 3, was " << obj["c"].first << endl; return false; } + ASSERT_SUCCESS( obj["obj"].get(obj) ); + ASSERT_EQUAL( obj["a"].get().first, 1 ); + ASSERT_EQUAL( obj["b"].get().first, 2 ); + ASSERT_EQUAL( obj["c/d"].get().first, 3 ); // Check all three again in backwards order, to ensure we can go backwards - if (obj["c/d"].get().first != 3) { cerr << "Expected uint64_t(obj[\"c\"]) to be 3, was " << obj["c"].first << endl; return false; } - if (obj["b"].get().first != 2) { cerr << "Expected uint64_t(obj[\"b\"]) to be 2, was " << obj["b"].first << endl; return false; } - if (obj["a"].get().first != 1) { cerr << "Expected uint64_t(obj[\"a\"]) to be 1, was " << obj["a"].first << endl; return false; } + ASSERT_EQUAL( obj["c/d"].get().first, 3 ); + ASSERT_EQUAL( obj["b"].get().first, 2 ); + ASSERT_EQUAL( obj["a"].get().first, 1 ); UNUSED element val; - if (doc["d"].get(val) != simdjson::NO_SUCH_FIELD) { cerr << "Expected NO_SUCH_FIELD error for uint64_t(obj[\"d\"]), got " << error << endl; return false; } + ASSERT_ERROR( doc["d"].get(val), NO_SUCH_FIELD); return true; } @@ -1054,9 +1060,9 @@ namespace dom_api_tests { std::cout << "Running " << __func__ << std::endl; // Prints the number of results in twitter.json dom::parser parser; - auto [result_count, error] = parser.load(TWITTER_JSON)["search_metadata"]["count"].get(); - if (error) { cerr << "Error: " << error << endl; return false; } - if (result_count != 100) { cerr << "Expected twitter.json[metadata_count][count] = 100, got " << result_count << endl; return false; } + uint64_t result_count; + ASSERT_SUCCESS( parser.load(TWITTER_JSON)["search_metadata"]["count"].get(result_count) ); + ASSERT_EQUAL( result_count, 100 ); return true; } @@ -1065,23 +1071,20 @@ namespace dom_api_tests { // Print users with a default profile. set default_users; dom::parser parser; - auto [tweets, error] = parser.load(TWITTER_JSON)["statuses"].get(); - if (error) { cerr << "Error: " << error << endl; return false; } + dom::array tweets; + ASSERT_SUCCESS( parser.load(TWITTER_JSON)["statuses"].get(tweets) ); for (auto tweet : tweets) { object user; - error = tweet["user"].get(user); // tie(...) = fails with "no viable overloaded '='" on Apple clang version 11.0.0; - if (error) { cerr << "Error: " << error << endl; return false; } + ASSERT_SUCCESS( tweet["user"].get(user) ); bool default_profile; - error = user["default_profile"].get(default_profile); // tie(...) = fails with "no viable overloaded '='" on Apple clang version 11.0.0; - if (error) { cerr << "Error: " << error << endl; return false; } + ASSERT_SUCCESS( user["default_profile"].get(default_profile) ); if (default_profile) { std::string_view screen_name; - error = user["screen_name"].get(screen_name); // tie(...) = fails with "no viable overloaded '='" on Apple clang version 11.0.0; - if (error) { cerr << "Error: " << error << endl; return false; } + ASSERT_SUCCESS( user["screen_name"].get(screen_name) ); default_users.insert(screen_name); } } - if (default_users.size() != 86) { cerr << "Expected twitter.json[statuses][user] to contain 86 default_profile users, got " << default_users.size() << endl; return false; } + ASSERT_EQUAL( default_users.size(), 86 ); return true; } @@ -1089,28 +1092,26 @@ namespace dom_api_tests { std::cout << "Running " << __func__ << std::endl; // Print image names and sizes set> image_sizes; + simdjson::error_code error; dom::parser parser; - auto [tweets, error] = parser.load(TWITTER_JSON)["statuses"].get(); - if (error) { cerr << "Error: " << error << endl; return false; } + dom::array tweets; + ASSERT_SUCCESS( parser.load(TWITTER_JSON)["statuses"].get(tweets) ); for (auto tweet : tweets) { - auto [media, not_found] = tweet["entities"]["media"].get(); - if (!not_found) { + dom::array media; + if (not (error = tweet["entities"]["media"].get(media))) { for (auto image : media) { object sizes; - error = image["sizes"].get(sizes); // tie(...) = fails with "no viable overloaded '='" on Apple clang version 11.0.0; - if (error) { cerr << "Error: " << error << endl; return false; } + ASSERT_SUCCESS( image["sizes"].get(sizes) ); for (auto size : sizes) { uint64_t width, height; - error = size.value["w"].get(width); // tie(...) = fails with "no viable overloaded '='" on Apple clang version 11.0.0; - if (error) { cerr << "Error: " << error << endl; return false; } - error = size.value["h"].get(height); // tie(...) = fails with "no viable overloaded '='" on Apple clang version 11.0.0; - if (error) { cerr << "Error: " << error << endl; return false; } + ASSERT_SUCCESS( size.value["w"].get(width) ); + ASSERT_SUCCESS( size.value["h"].get(height) ); image_sizes.insert(make_pair(width, height)); } } } } - if (image_sizes.size() != 15) { cerr << "Expected twitter.json[statuses][entities][media][sizes] to contain 15 different sizes, got " << image_sizes.size() << endl; return false; } + ASSERT_EQUAL( image_sizes.size(), 15 ); return true; } @@ -1124,12 +1125,12 @@ namespace dom_api_tests { int i = 0; dom::parser parser; - element doc = parser.parse(json); - for (auto [key, value] : doc.get()) { - if (key != expected_key[i] || uint64_t(value) != expected_value[i]) { cerr << "Expected " << expected_key[i] << " = " << expected_value[i] << ", got " << key << "=" << uint64_t(value) << endl; return false; } + for (auto [key, value] : dom::object(parser.parse(json))) { + ASSERT_EQUAL( key, expected_key[i]); + ASSERT_EQUAL( uint64_t(value), expected_value[i] ); i++; } - if (i*sizeof(uint64_t) != sizeof(expected_value)) { cout << "Expected " << sizeof(expected_value) << " values, got " << i << endl; return false; } + ASSERT_EQUAL( i*sizeof(uint64_t), sizeof(expected_value) ); return true; } @@ -1140,69 +1141,61 @@ namespace dom_api_tests { int i=0; dom::parser parser; - element doc = parser.parse(json); - for (uint64_t value : doc.get()) { - if (value != expected_value[i]) { cerr << "Expected " << expected_value[i] << ", got " << value << endl; return false; } + for (uint64_t value : parser.parse(json)) { + ASSERT_EQUAL( value, expected_value[i] ); i++; } - if (i*sizeof(uint64_t) != sizeof(expected_value)) { cout << "Expected " << sizeof(expected_value) << " values, got " << i << endl; return false; } + ASSERT_EQUAL( i*sizeof(uint64_t), sizeof(expected_value) ); return true; } bool string_value_exception() { std::cout << "Running " << __func__ << std::endl; - string json(R"([ "hi", "has backslash\\" ])"); dom::parser parser; - auto val = parser.parse(json).get().begin(); - - if (strcmp((const char*)*val, "hi")) { cerr << "Expected const char*(\"hi\") to be \"hi\", was " << (const char*)*val << endl; return false; } - if (string_view(*val) != "hi") { cerr << "Expected string_view(\"hi\") to be \"hi\", was " << string_view(*val) << endl; return false; } - ++val; - if (strcmp((const char*)*val, "has backslash\\")) { cerr << "Expected const char*(\"has backslash\\\\\") to be \"has backslash\\\", was " << (const char*)*val << endl; return false; } - if (string_view(*val) != "has backslash\\") { cerr << "Expected string_view(\"has backslash\\\\\") to be \"has backslash\\\", was " << string_view(*val) << endl; return false; } + ASSERT_EQUAL( (const char *)parser.parse(R"("hi")"_padded), "hi" ); + ASSERT_EQUAL( string_view(parser.parse(R"("hi")"_padded)), "hi" ); + ASSERT_EQUAL( (const char *)parser.parse(R"("has backslash\\")"_padded), "has backslash\\"); + ASSERT_EQUAL( string_view(parser.parse(R"("has backslash\\")"_padded)), "has backslash\\" ); return true; } bool numeric_values_exception() { std::cout << "Running " << __func__ << std::endl; - string json(R"([ 0, 1, -1, 1.1 ])"); dom::parser parser; - auto val = parser.parse(json).get().begin(); - if (uint64_t(*val) != 0) { cerr << "Expected uint64_t(0) to be 0, was " << uint64_t(*val) << endl; return false; } - if (int64_t(*val) != 0) { cerr << "Expected int64_t(0) to be 0, was " << int64_t(*val) << endl; return false; } - if (double(*val) != 0) { cerr << "Expected double(0) to be 0, was " << double(*val) << endl; return false; } - ++val; - if (uint64_t(*val) != 1) { cerr << "Expected uint64_t(1) to be 1, was " << uint64_t(*val) << endl; return false; } - if (int64_t(*val) != 1) { cerr << "Expected int64_t(1) to be 1, was " << int64_t(*val) << endl; return false; } - if (double(*val) != 1) { cerr << "Expected double(1) to be 1, was " << double(*val) << endl; return false; } - ++val; - if (int64_t(*val) != -1) { cerr << "Expected int64_t(-1) to be -1, was " << int64_t(*val) << endl; return false; } - if (double(*val) != -1) { cerr << "Expected double(-1) to be -1, was " << double(*val) << endl; return false; } - ++val; - if (double(*val) != 1.1) { cerr << "Expected double(1.1) to be 1.1, was " << double(*val) << endl; return false; } + ASSERT_EQUAL( uint64_t(parser.parse("0"_padded)), 0); + ASSERT_EQUAL( int64_t(parser.parse("0"_padded)), 0); + ASSERT_EQUAL( double(parser.parse("0"_padded)), 0); + + ASSERT_EQUAL( uint64_t(parser.parse("1"_padded)), 1); + ASSERT_EQUAL( int64_t(parser.parse("1"_padded)), 1); + ASSERT_EQUAL( double(parser.parse("1"_padded)), 1); + + ASSERT_EQUAL( int64_t(parser.parse("-1"_padded)), -1); + ASSERT_EQUAL( double(parser.parse("-1"_padded)), -1); + + ASSERT_EQUAL( double(parser.parse("1.1"_padded)), 1.1); + return true; } bool boolean_values_exception() { std::cout << "Running " << __func__ << std::endl; - string json(R"([ true, false ])"); dom::parser parser; - auto val = parser.parse(json).get().begin(); - if (bool(*val) != true) { cerr << "Expected bool(true) to be true, was " << bool(*val) << endl; return false; } - ++val; - if (bool(*val) != false) { cerr << "Expected bool(false) to be false, was " << bool(*val) << endl; return false; } + ASSERT_EQUAL( bool(parser.parse("true"_padded)), true); + + ASSERT_EQUAL( bool(parser.parse("false"_padded)), false); + return true; } bool null_value_exception() { std::cout << "Running " << __func__ << std::endl; - string json(R"([ null ])"); dom::parser parser; - auto val = parser.parse(json).get().begin(); - if (!(*val).is_null()) { cerr << "Expected null to be null!" << endl; return false; } + ASSERT_EQUAL( bool(parser.parse("null"_padded).is_null()), true ); + return true; } @@ -1210,8 +1203,10 @@ namespace dom_api_tests { std::cout << "Running " << __func__ << std::endl; string json(R"({ "a": 1, "b": 2, "c": 3})"); dom::parser parser; - element doc = parser.parse(json); - if (uint64_t(doc["a"]) != 1) { cerr << "Expected uint64_t(doc[\"a\"]) to be 1, was " << uint64_t(doc["a"]) << endl; return false; } + auto obj = parser.parse(json); + + ASSERT_EQUAL(uint64_t(obj["a"]), 1); + return true; } @@ -1220,7 +1215,9 @@ namespace dom_api_tests { string json(R"({ "obj": { "a": 1, "b": 2, "c": 3 } })"); dom::parser parser; object obj = parser.parse(json)["obj"]; - if (uint64_t(obj["a"]) != 1) { cerr << "Expected uint64_t(doc[\"a\"]) to be 1, was " << uint64_t(obj["a"]) << endl; return false; } + + ASSERT_EQUAL( uint64_t(obj["a"]), 1); + return true; } @@ -1255,18 +1252,17 @@ namespace dom_api_tests { // Print image names and sizes set> image_sizes; dom::parser parser; - element doc = parser.load(TWITTER_JSON); - for (object tweet : doc["statuses"].get()) { - auto [media, not_found] = tweet["entities"]["media"]; - if (!not_found) { - for (object image : media.get()) { - for (auto size : image["sizes"].get()) { + for (object tweet : parser.load(TWITTER_JSON)["statuses"]) { + auto media = tweet["entities"]["media"]; + if (!media.error()) { + for (object image : media) { + for (auto size : object(image["sizes"])) { image_sizes.insert(make_pair(size.value["w"], size.value["h"])); } } } } - if (image_sizes.size() != 15) { cerr << "Expected twitter.json[statuses][entities][media][sizes] to contain 15 different sizes, got " << image_sizes.size() << endl; return false; } + ASSERT_EQUAL( image_sizes.size(), 15 ); return true; } @@ -1343,7 +1339,9 @@ namespace type_tests { // Grab the element out and check success dom::element element = result.first; - RUN_TEST( tester.test_get(element, expected ) ); + RUN_TEST( tester.test_get_t(element, expected) ); + RUN_TEST( tester.test_get_t(result, expected) ); + RUN_TEST( tester.test_get(element, expected) ); RUN_TEST( tester.test_get(result, expected) ); // RUN_TEST( tester.test_named_get(element, expected) ); // RUN_TEST( tester.test_named_get(result, expected) ); @@ -1366,6 +1364,8 @@ namespace type_tests { // Grab the element out and check success dom::element element = result.first; + RUN_TEST( tester.test_get_t(element) ); + RUN_TEST( tester.test_get_t(result) ); RUN_TEST( tester.test_get(element) ); RUN_TEST( tester.test_get(result) ); RUN_TEST( tester.test_named_get(element) ); @@ -1441,27 +1441,9 @@ namespace type_tests { std::cout << " test_is_null() expecting " << expected_is_null << std::endl; // Grab the element out and check success dom::element element = result.first; - bool actual_is_null; - auto error = result.is_null().get(actual_is_null); - ASSERT_SUCCESS(error); - ASSERT_EQUAL(actual_is_null, expected_is_null); + ASSERT_EQUAL(result.is_null(), expected_is_null); - actual_is_null = element.is_null(); - ASSERT_EQUAL(actual_is_null, expected_is_null); - -#if SIMDJSON_EXCEPTIONS - - try { - - actual_is_null = result.is_null(); - ASSERT_EQUAL(actual_is_null, expected_is_null); - - } catch(simdjson_error &e) { - std::cerr << e.error() << std::endl; - return false; - } - -#endif // SIMDJSON_EXCEPTIONS + ASSERT_EQUAL(element.is_null(), expected_is_null); return true; } @@ -1684,21 +1666,10 @@ namespace minify_tests { return false; } size_t newlength{}; - auto error = simdjson::minify(input, length, buffer.get(), newlength); - if(error != simdjson::SUCCESS) { - std::cerr << "error " << error << std::endl; - return false; - } - // memcmp - if(newlength != expected_length) { - std::cerr << "lengths do not match " << std::endl; - return false; - } + ASSERT_SUCCESS( simdjson::minify(input, length, buffer.get(), newlength) ); + ASSERT_EQUAL( newlength, expected_length); for(size_t i = 0; i < newlength; i++) { - if(buffer.get()[i] != expected[i]) { - std::cerr << "Inputs do not match (but same length) " << std::endl; - return false; - } + ASSERT_EQUAL( buffer.get()[i], expected[i]); } return true; } @@ -1748,8 +1719,8 @@ namespace format_tests { bool print_parser_parse() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; - auto [doc, error] = parser.parse(DOCUMENT); - if (error) { cerr << error << endl; return false; } + dom::element doc; + ASSERT_SUCCESS( parser.parse(DOCUMENT).get(doc) ); ostringstream s; s << doc; return assert_minified(s); @@ -1757,8 +1728,8 @@ namespace format_tests { bool print_minify_parser_parse() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; - auto [doc, error] = parser.parse(DOCUMENT); - if (error) { cerr << error << endl; return false; } + dom::element doc; + ASSERT_SUCCESS( parser.parse(DOCUMENT).get(doc) ); ostringstream s; s << minify(doc); return assert_minified(s); @@ -1767,8 +1738,8 @@ namespace format_tests { bool print_element() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; - auto [value, error] = parser.parse(DOCUMENT)["foo"]; - if (error) { cerr << error << endl; return false; } + dom::element value; + ASSERT_SUCCESS( parser.parse(DOCUMENT)["foo"].get(value) ); ostringstream s; s << value; return assert_minified(s, "1"); @@ -1776,8 +1747,8 @@ namespace format_tests { bool print_minify_element() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; - auto [value, error] = parser.parse(DOCUMENT)["foo"]; - if (error) { cerr << error << endl; return false; } + dom::element value; + ASSERT_SUCCESS( parser.parse(DOCUMENT)["foo"].get(value) ); ostringstream s; s << minify(value); return assert_minified(s, "1"); @@ -1786,38 +1757,38 @@ namespace format_tests { bool print_array() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; - auto [value, error] = parser.parse(DOCUMENT)["bar"].get(); - if (error) { cerr << error << endl; return false; } + dom::array array; + ASSERT_SUCCESS( parser.parse(DOCUMENT)["bar"].get(array) ); ostringstream s; - s << value; + s << array; return assert_minified(s, "[1,2,3]"); } bool print_minify_array() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; - auto [value, error] = parser.parse(DOCUMENT)["bar"].get(); - if (error) { cerr << error << endl; return false; } + dom::array array; + ASSERT_SUCCESS( parser.parse(DOCUMENT)["bar"].get(array) ); ostringstream s; - s << minify(value); + s << minify(array); return assert_minified(s, "[1,2,3]"); } bool print_object() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; - auto [value, error] = parser.parse(DOCUMENT)["baz"].get(); - if (error) { cerr << error << endl; return false; } + dom::object object; + ASSERT_SUCCESS( parser.parse(DOCUMENT)["baz"].get(object) ); ostringstream s; - s << value; + s << object; return assert_minified(s, R"({"a":1,"b":2,"c":3})"); } bool print_minify_object() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; - auto [value, error] = parser.parse(DOCUMENT)["baz"].get(); - if (error) { cerr << error << endl; return false; } + dom::object object; + ASSERT_SUCCESS( parser.parse(DOCUMENT)["baz"].get(object) ); ostringstream s; - s << minify(value); + s << minify(object); return assert_minified(s, R"({"a":1,"b":2,"c":3})"); } @@ -1841,25 +1812,22 @@ namespace format_tests { bool print_element_result_exception() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; - element doc = parser.parse(DOCUMENT); ostringstream s; - s << doc["foo"]; + s << parser.parse(DOCUMENT)["foo"]; return assert_minified(s, "1"); } bool print_minify_element_result_exception() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; - element doc = parser.parse(DOCUMENT); ostringstream s; - s << minify(doc["foo"]); + s << minify(parser.parse(DOCUMENT)["foo"]); return assert_minified(s, "1"); } bool print_element_exception() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; - element doc = parser.parse(DOCUMENT); - element value = doc["foo"]; + element value = parser.parse(DOCUMENT)["foo"]; ostringstream s; s << value; return assert_minified(s, "1"); @@ -1867,8 +1835,7 @@ namespace format_tests { bool print_minify_element_exception() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; - element doc = parser.parse(DOCUMENT); - element value = doc["foo"]; + element value = parser.parse(DOCUMENT)["foo"]; ostringstream s; s << minify(value); return assert_minified(s, "1"); @@ -1877,66 +1844,64 @@ namespace format_tests { bool print_array_result_exception() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; - element doc = parser.parse(DOCUMENT); ostringstream s; - s << doc["bar"].get(); + s << parser.parse(DOCUMENT)["bar"].get(); return assert_minified(s, "[1,2,3]"); } bool print_minify_array_result_exception() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; - element doc = parser.parse(DOCUMENT); ostringstream s; - s << minify(doc["bar"].get()); + s << minify(parser.parse(DOCUMENT)["bar"].get()); return assert_minified(s, "[1,2,3]"); } bool print_object_result_exception() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; - element doc = parser.parse(DOCUMENT); ostringstream s; - s << doc["baz"].get(); + s << parser.parse(DOCUMENT)["baz"].get(); return assert_minified(s, R"({"a":1,"b":2,"c":3})"); } bool print_minify_object_result_exception() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; - element doc = parser.parse(DOCUMENT); ostringstream s; - s << minify(doc["baz"].get()); + s << minify(parser.parse(DOCUMENT)["baz"].get()); return assert_minified(s, R"({"a":1,"b":2,"c":3})"); } bool print_array_exception() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; + dom::array array = parser.parse(DOCUMENT)["bar"]; ostringstream s; - s << parser.parse(DOCUMENT)["bar"]; + s << array; return assert_minified(s, "[1,2,3]"); } bool print_minify_array_exception() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; + dom::array array = parser.parse(DOCUMENT)["bar"]; ostringstream s; - s << minify(parser.parse(DOCUMENT)["bar"]); + s << minify(array); return assert_minified(s, "[1,2,3]"); } bool print_object_exception() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; + dom::object object = parser.parse(DOCUMENT)["baz"]; ostringstream s; - s << parser.parse(DOCUMENT)["baz"]; + s << object; return assert_minified(s, R"({"a":1,"b":2,"c":3})"); } bool print_minify_object_exception() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; - element doc = parser.parse(DOCUMENT); - object value = doc["baz"]; + dom::object object = parser.parse(DOCUMENT)["baz"]; ostringstream s; - s << minify(value); + s << minify(object); return assert_minified(s, R"({"a":1,"b":2,"c":3})"); } #endif // SIMDJSON_EXCEPTIONS diff --git a/tests/cast_tester.h b/tests/cast_tester.h index 59cdc948d..55eda96ae 100644 --- a/tests/cast_tester.h +++ b/tests/cast_tester.h @@ -26,6 +26,11 @@ public: bool test_get_error(element element, error_code expected_error); bool test_get_error(simdjson_result element, error_code expected_error); + bool test_get_t(element element, T expected = {}); + bool test_get_t(simdjson_result element, T expected = {}); + bool test_get_t_error(element element, error_code expected_error); + bool test_get_t_error(simdjson_result element, error_code expected_error); + #if SIMDJSON_EXCEPTIONS bool test_implicit_cast(element element, T expected = {}); bool test_implicit_cast(simdjson_result element, T expected = {}); @@ -35,7 +40,6 @@ public: bool test_is(element element, bool expected); bool test_is(simdjson_result element, bool expected); - bool test_is_error(simdjson_result element, error_code expected_error); bool test_named_get(element element, T expected = {}); bool test_named_get(simdjson_result element, T expected = {}); @@ -44,81 +48,94 @@ public: bool test_named_is(element element, bool expected); bool test_named_is(simdjson_result element, bool expected); - bool test_named_is_error(simdjson_result element, error_code expected_error); private: simdjson_result named_get(element element); simdjson_result named_get(simdjson_result element); bool named_is(element element); - simdjson_result named_is(simdjson_result element); + bool named_is(simdjson_result element); bool assert_equal(const T& expected, const T& actual); }; template bool cast_tester::test_get(element element, T expected) { T actual; - error_code error; - error = element.get(actual); - ASSERT_SUCCESS(error); + ASSERT_SUCCESS(element.get(actual)); return assert_equal(actual, expected); } template bool cast_tester::test_get(simdjson_result element, T expected) { T actual; - error_code error; - error = element.get(actual); - ASSERT_SUCCESS(error); + ASSERT_SUCCESS(element.get(actual)); return assert_equal(actual, expected); } template bool cast_tester::test_get_error(element element, error_code expected_error) { T actual; - error_code error; - error = element.get(actual); - ASSERT_EQUAL(error, expected_error); + ASSERT_EQUAL(element.get(actual), expected_error); return true; } template bool cast_tester::test_get_error(simdjson_result element, error_code expected_error) { T actual; - error_code error; - error = element.get(actual); - ASSERT_EQUAL(error, expected_error); + ASSERT_EQUAL(element.get(actual), expected_error); + return true; +} + +template +bool cast_tester::test_get_t(element element, T expected) { + auto actual = element.get(); + ASSERT_SUCCESS(actual.error()); + return assert_equal(actual.first, expected); +} + +template +bool cast_tester::test_get_t(simdjson_result element, T expected) { + auto actual = element.get(); + ASSERT_SUCCESS(actual.error()); + return assert_equal(actual.first, expected); +} + +template +bool cast_tester::test_get_t_error(element element, error_code expected_error) { + ASSERT_EQUAL(element.get().error(), expected_error); + return true; +} + +template +bool cast_tester::test_get_t_error(simdjson_result element, error_code expected_error) { + ASSERT_EQUAL(element.get().error(), expected_error); return true; } template bool cast_tester::test_named_get(element element, T expected) { T actual; - auto error = named_get(element).get(actual); - ASSERT_SUCCESS(error); + ASSERT_SUCCESS(named_get(element).get(actual)); return assert_equal(actual, expected); } template bool cast_tester::test_named_get(simdjson_result element, T expected) { T actual; - auto error = named_get(element).get(actual); - ASSERT_SUCCESS(error); + ASSERT_SUCCESS(named_get(element).get(actual)); return assert_equal(actual, expected); } template bool cast_tester::test_named_get_error(element element, error_code expected_error) { T actual; - auto error = named_get(element).get(actual); - ASSERT_EQUAL(error, expected_error); + ASSERT_EQUAL(named_get(element).get(actual), expected_error); return true; } template bool cast_tester::test_named_get_error(simdjson_result element, error_code expected_error) { T actual; - auto error = named_get(element).get(actual); - ASSERT_EQUAL(error, expected_error); + ASSERT_EQUAL(named_get(element).get(actual), expected_error); return true; } @@ -187,18 +204,7 @@ bool cast_tester::test_is(element element, bool expected) { template bool cast_tester::test_is(simdjson_result element, bool expected) { - bool actual; - auto error = element.is().get(actual); - ASSERT_SUCCESS(error); - ASSERT_EQUAL(actual, expected); - return true; -} - -template -bool cast_tester::test_is_error(simdjson_result element, error_code expected_error) { - UNUSED bool actual; - auto error = element.is().get(actual); - ASSERT_EQUAL(error, expected_error); + ASSERT_EQUAL(element.is(), expected); return true; } @@ -210,18 +216,7 @@ bool cast_tester::test_named_is(element element, bool expected) { template bool cast_tester::test_named_is(simdjson_result element, bool expected) { - bool actual; - auto error = named_is(element).get(actual); - ASSERT_SUCCESS(error); - ASSERT_EQUAL(actual, expected); - return true; -} - -template -bool cast_tester::test_named_is_error(simdjson_result element, error_code expected_error) { - bool actual; - auto error = named_is(element).get(actual); - ASSERT_EQUAL(error, expected_error); + ASSERT_EQUAL(named_is(element), expected); return true; } @@ -252,14 +247,14 @@ template<> bool cast_tester::named_is(element element) { return element template<> bool cast_tester::named_is(element element) { return element.is_double(); } template<> bool cast_tester::named_is(element element) { return element.is_bool(); } -template<> simdjson_result cast_tester::named_is(simdjson_result element) { return element.is_array(); } -template<> simdjson_result cast_tester::named_is(simdjson_result element) { return element.is_object(); } -template<> simdjson_result cast_tester::named_is(simdjson_result element) { return element.is_string(); } -template<> simdjson_result cast_tester::named_is(simdjson_result element) { return element.is_string(); } -template<> simdjson_result cast_tester::named_is(simdjson_result element) { return element.is_uint64_t(); } -template<> simdjson_result cast_tester::named_is(simdjson_result element) { return element.is_int64_t(); } -template<> simdjson_result cast_tester::named_is(simdjson_result element) { return element.is_double(); } -template<> simdjson_result cast_tester::named_is(simdjson_result element) { return element.is_bool(); } +template<> bool cast_tester::named_is(simdjson_result element) { return element.is_array(); } +template<> bool cast_tester::named_is(simdjson_result element) { return element.is_object(); } +template<> bool cast_tester::named_is(simdjson_result element) { return element.is_string(); } +template<> bool cast_tester::named_is(simdjson_result element) { return element.is_string(); } +template<> bool cast_tester::named_is(simdjson_result element) { return element.is_uint64_t(); } +template<> bool cast_tester::named_is(simdjson_result element) { return element.is_int64_t(); } +template<> bool cast_tester::named_is(simdjson_result element) { return element.is_double(); } +template<> bool cast_tester::named_is(simdjson_result element) { return element.is_bool(); } template bool cast_tester::assert_equal(const T& expected, const T& actual) { ASSERT_EQUAL(expected, actual); diff --git a/tests/errortests.cpp b/tests/errortests.cpp index 27f9bfc8a..f2050d5af 100644 --- a/tests/errortests.cpp +++ b/tests/errortests.cpp @@ -14,15 +14,8 @@ using namespace simdjson; using namespace std; -#ifndef SIMDJSON_BENCHMARK_DATA_DIR -#define SIMDJSON_BENCHMARK_DATA_DIR "jsonexamples/" -#endif -const char *TWITTER_JSON = SIMDJSON_BENCHMARK_DATA_DIR "twitter.json"; +#include "test_macros.h" -#define TEST_START() { cout << "Running " << __func__ << " ..." << endl; } -#define ASSERT_ERROR(ACTUAL, EXPECTED) if ((ACTUAL) != (EXPECTED)) { cerr << "FAIL: Unexpected error \"" << (ACTUAL) << "\" (expected \"" << (EXPECTED) << "\")" << endl; return false; } -#define TEST_FAIL(MESSAGE) { cerr << "FAIL: " << (MESSAGE) << endl; return false; } -#define TEST_SUCCEED() { return true; } namespace parser_load { const char * NONEXISTENT_FILE = "this_file_does_not_exist.json"; bool parser_load_capacity() { @@ -35,7 +28,9 @@ namespace parser_load { bool parser_load_many_capacity() { TEST_START(); dom::parser parser(1); // 1 byte max capacity - for (auto doc : parser.load_many(TWITTER_JSON)) { + dom::document_stream docs; + ASSERT_SUCCESS(parser.load_many(TWITTER_JSON).get(docs)); + for (auto doc : docs) { ASSERT_ERROR(doc.error(), CAPACITY); TEST_SUCCEED(); } @@ -47,17 +42,20 @@ namespace parser_load { const padded_string DOC = "1 2 [} 3"_padded; size_t count = 0; dom::parser parser; - for (auto doc : parser.parse_many(DOC)) { + dom::document_stream docs; + ASSERT_SUCCESS(parser.parse_many(DOC).get(docs)); + for (auto doc : docs) { count++; - auto [val, error] = doc.get(); + uint64_t val; + auto error = doc.get(val); if (count == 3) { ASSERT_ERROR(error, TAPE_ERROR); } else { - if (error) { TEST_FAIL(error); } - if (val != count) { cerr << "FAIL: expected " << count << ", got " << val << endl; return false; } + ASSERT_SUCCESS(error); + ASSERT_EQUAL(val, count); } } - if (count != 3) { cerr << "FAIL: expected 2 documents and 1 error, got " << count << " total things" << endl; return false; } + ASSERT_EQUAL(count, 3); TEST_SUCCEED(); } @@ -66,11 +64,13 @@ namespace parser_load { const padded_string DOC = "["_padded; size_t count = 0; dom::parser parser; - for (auto doc : parser.parse_many(DOC)) { + dom::document_stream docs; + ASSERT_SUCCESS(parser.parse_many(DOC).get(docs)); + for (auto doc : docs) { count++; ASSERT_ERROR(doc.error(), TAPE_ERROR); } - if (count != 1) { cerr << "FAIL: expected no documents and 1 error, got " << count << " total things" << endl; return false; } + ASSERT_EQUAL(count, 1); TEST_SUCCEED(); } @@ -79,60 +79,56 @@ namespace parser_load { const padded_string DOC = "1 2 ["_padded; size_t count = 0; dom::parser parser; - for (auto doc : parser.parse_many(DOC)) { + dom::document_stream docs; + ASSERT_SUCCESS(parser.parse_many(DOC).get(docs)); + for (auto doc : docs) { count++; - auto [val, error] = doc.get(); + uint64_t val; + auto error = doc.get(val); if (count == 3) { ASSERT_ERROR(error, TAPE_ERROR); } else { - if (error) { TEST_FAIL(error); } - if (val != count) { cerr << "FAIL: expected " << count << ", got " << val << endl; return false; } + ASSERT_SUCCESS(error); + ASSERT_EQUAL(val, count); } } - if (count != 3) { cerr << "FAIL: expected 2 documents and 1 error, got " << count << " total things" << endl; return false; } + ASSERT_EQUAL(count, 3); TEST_SUCCEED(); } bool parser_load_nonexistent() { TEST_START(); dom::parser parser; - auto error = parser.load(NONEXISTENT_FILE).error(); - ASSERT_ERROR(error, IO_ERROR); + ASSERT_ERROR( parser.load(NONEXISTENT_FILE).error(), IO_ERROR ); TEST_SUCCEED(); } bool parser_load_many_nonexistent() { TEST_START(); dom::parser parser; - for (auto doc : parser.load_many(NONEXISTENT_FILE)) { - ASSERT_ERROR(doc.error(), IO_ERROR); - TEST_SUCCEED(); - } - TEST_FAIL("No documents returned"); + ASSERT_ERROR( parser.load_many(NONEXISTENT_FILE).error(), IO_ERROR ); + TEST_SUCCEED(); } bool padded_string_load_nonexistent() { TEST_START(); - auto error = padded_string::load(NONEXISTENT_FILE).error(); - ASSERT_ERROR(error, IO_ERROR); + ASSERT_ERROR(padded_string::load(NONEXISTENT_FILE).error(), IO_ERROR); TEST_SUCCEED(); } bool parser_load_chain() { TEST_START(); dom::parser parser; - auto error = parser.load(NONEXISTENT_FILE)["foo"].get().error(); - ASSERT_ERROR(error, IO_ERROR); + UNUSED uint64_t foo; + ASSERT_ERROR( parser.load(NONEXISTENT_FILE)["foo"].get(foo), IO_ERROR); TEST_SUCCEED(); } bool parser_load_many_chain() { TEST_START(); dom::parser parser; - for (auto doc : parser.load_many(NONEXISTENT_FILE)) { - auto error = doc["foo"].get().error(); - ASSERT_ERROR(error, IO_ERROR); - TEST_SUCCEED(); - } - TEST_FAIL("No documents returned"); + UNUSED dom::document_stream stream; + ASSERT_ERROR( parser.load_many(NONEXISTENT_FILE).get(stream), IO_ERROR ); + TEST_SUCCEED(); } + bool run() { return true && parser_load_capacity() diff --git a/tests/integer_tests.cpp b/tests/integer_tests.cpp index c072ac05f..c35d717c8 100644 --- a/tests/integer_tests.cpp +++ b/tests/integer_tests.cpp @@ -3,6 +3,7 @@ #include #include "simdjson.h" +#include "test_macros.h" // we define our own asserts to get around NDEBUG #ifndef ASSERT @@ -29,44 +30,43 @@ template static const std::string make_json(T value) { } template -static void parse_and_validate(const std::string src, T expected) { +static bool parse_and_validate(const std::string src, T expected) { std::cout << "src: " << src << ", "; const padded_string pstr{src}; simdjson::dom::parser parser; - bool result; if constexpr (std::is_same::value) { - auto [actual, error] = parser.parse(pstr).get()["key"].get(); - if (error) { std::cerr << error << std::endl; abort(); } - result = (expected == actual); + int64_t actual; + ASSERT_SUCCESS( parser.parse(pstr)["key"].get(actual) ); + std::cout << std::boolalpha << "test: " << (expected == actual) << std::endl; + ASSERT_EQUAL( expected, actual ); } else { - auto [actual, error] = parser.parse(pstr).get()["key"].get(); - if (error) { std::cerr << error << std::endl; abort(); } - result = (expected == actual); - } - std::cout << std::boolalpha << "test: " << result << std::endl; - if(!result) { - std::cerr << "bug detected" << std::endl; - exit(EXIT_FAILURE); + uint64_t actual; + ASSERT_SUCCESS( parser.parse(pstr)["key"].get(actual) ); + std::cout << std::boolalpha << "test: " << (expected == actual) << std::endl; + ASSERT_EQUAL( expected, actual ); } + return true; } static bool parse_and_check_signed(const std::string src) { std::cout << "src: " << src << ", expecting signed" << std::endl; const padded_string pstr{src}; simdjson::dom::parser parser; - auto [value, error] = parser.parse(pstr).get()["key"]; - if (error) { std::cerr << error << std::endl; abort(); } - return value.is(); + simdjson::dom::element value; + ASSERT_SUCCESS( parser.parse(pstr).get()["key"].get(value) ); + ASSERT_EQUAL( value.is(), true ); + return true; } static bool parse_and_check_unsigned(const std::string src) { std::cout << "src: " << src << ", expecting signed" << std::endl; const padded_string pstr{src}; simdjson::dom::parser parser; - auto [value, error] = parser.parse(pstr).get()["key"]; - if (error) { std::cerr << error << std::endl; abort(); } - return value.is(); + simdjson::dom::element value; + ASSERT_SUCCESS( parser.parse(pstr).get()["key"].get(value) ); + ASSERT_EQUAL( value.is(), true ); + return true; } int main() { @@ -75,21 +75,21 @@ int main() { constexpr auto int64_min = numeric_limits::lowest(); constexpr auto uint64_max = numeric_limits::max(); constexpr auto uint64_min = numeric_limits::lowest(); - parse_and_validate(make_json(int64_max), int64_max); - parse_and_validate(make_json(int64_min), int64_min); - parse_and_validate(make_json(uint64_max), uint64_max); - parse_and_validate(make_json(uint64_min), uint64_min); constexpr auto int64_max_plus1 = static_cast(int64_max) + 1; - parse_and_validate(make_json(int64_max_plus1), int64_max_plus1); - if(!parse_and_check_signed(make_json(int64_max))) { - std::cerr << "bug: large signed integers should be represented as signed integers" << std::endl; - return EXIT_FAILURE; + if (true + && parse_and_validate(make_json(int64_max), int64_max) + && parse_and_validate(make_json(uint64_max), uint64_max) + && parse_and_validate(make_json(uint64_min), uint64_min) + && parse_and_validate(make_json(int64_min), int64_min) + && parse_and_validate(make_json(uint64_max), uint64_max) + && parse_and_validate(make_json(uint64_min), uint64_min) + && parse_and_validate(make_json(int64_max_plus1), int64_max_plus1) + && parse_and_check_signed(make_json(int64_max)) + && parse_and_check_unsigned(make_json(uint64_max)) + ) { + std::cout << "All ok." << std::endl; + return EXIT_SUCCESS; } - if(!parse_and_check_unsigned(make_json(uint64_max))) { - std::cerr << "bug: a large unsigned integers is not represented as an unsigned integer" << std::endl; - return EXIT_FAILURE; - } - std::cout << "All ok." << std::endl; - return EXIT_SUCCESS; + return EXIT_FAILURE; } diff --git a/tests/jsoncheck.cpp b/tests/jsoncheck.cpp index 55c734e69..35001c7ce 100644 --- a/tests/jsoncheck.cpp +++ b/tests/jsoncheck.cpp @@ -60,7 +60,8 @@ bool validate(const char *dirname) { char *fullpath = static_cast(malloc(fullpathlen)); snprintf(fullpath, fullpathlen, "%s%s%s", dirname, needsep ? "/" : "", name); - auto [p, error] = simdjson::padded_string::load(fullpath); + simdjson::padded_string p; + auto error = simdjson::padded_string::load(fullpath).get(p); if (error) { std::cerr << "Could not load the file " << fullpath << std::endl; return EXIT_FAILURE; diff --git a/tests/numberparsingcheck.cpp b/tests/numberparsingcheck.cpp index 92ab25f37..dbcc30477 100644 --- a/tests/numberparsingcheck.cpp +++ b/tests/numberparsingcheck.cpp @@ -172,7 +172,8 @@ bool validate(const char *dirname) { } else { strcpy(fullpath + dirlen, name); } - auto [p, error] = simdjson::padded_string::load(fullpath); + simdjson::padded_string p; + auto error = simdjson::padded_string::load(fullpath).get(p); if (error) { std::cerr << "Could not load the file " << fullpath << std::endl; return EXIT_FAILURE; diff --git a/tests/parse_many_test.cpp b/tests/parse_many_test.cpp index 66fba3a4e..9e3ab6f43 100644 --- a/tests/parse_many_test.cpp +++ b/tests/parse_many_test.cpp @@ -69,13 +69,16 @@ bool validate(const char *dirname) { snprintf(fullpath, fullpathlen, "%s%s%s", dirname, needsep ? "/" : "", name); /* The actual test*/ - auto [json, error] = simdjson::padded_string::load(fullpath); + simdjson::padded_string json; + auto error = simdjson::padded_string::load(fullpath).get(json); if (!error) { simdjson::dom::parser parser; ++how_many; - for (auto result : parser.parse_many(json)) { - error = result.error(); + simdjson::dom::document_stream docs; + error = parser.parse_many(json).get(docs); + for (auto doc : docs) { + error = doc.error(); } } printf("%s\n", error ? "ok" : "invalid"); diff --git a/tests/pointercheck.cpp b/tests/pointercheck.cpp index 58e67ce3e..e0517edb5 100644 --- a/tests/pointercheck.cpp +++ b/tests/pointercheck.cpp @@ -1,6 +1,7 @@ #include #include "simdjson.h" +#include "test_macros.h" // we define our own asserts to get around NDEBUG #ifndef ASSERT @@ -35,49 +36,46 @@ const padded_string TEST_JSON = R"( bool json_pointer_success_test(const char *json_pointer, std::string_view expected_value) { std::cout << "Running successful JSON pointer test '" << json_pointer << "' ..." << std::endl; dom::parser parser; - auto [value, error] = parser.parse(TEST_JSON).at(json_pointer).get(); - if (error) { std::cerr << "Unexpected Error: " << error << std::endl; return false; } - ASSERT(value == expected_value); + std::string_view value; + ASSERT_SUCCESS( parser.parse(TEST_JSON).at(json_pointer).get(value) ); + ASSERT_EQUAL(value, expected_value); return true; } bool json_pointer_success_test(const char *json_pointer) { std::cout << "Running successful JSON pointer test '" << json_pointer << "' ..." << std::endl; dom::parser parser; - auto error = parser.parse(TEST_JSON).at(json_pointer).error(); - if (error) { std::cerr << "Unexpected Error: " << error << std::endl; return false; } + ASSERT_SUCCESS( parser.parse(TEST_JSON).at(json_pointer).error() ); return true; } -bool json_pointer_failure_test(const char *json_pointer, error_code expected_failure_test) { +bool json_pointer_failure_test(const char *json_pointer, error_code expected_error) { std::cout << "Running invalid JSON pointer test '" << json_pointer << "' ..." << std::endl; dom::parser parser; - auto error = parser.parse(TEST_JSON).at(json_pointer).error(); - ASSERT(error == expected_failure_test); + ASSERT_ERROR(parser.parse(TEST_JSON).at(json_pointer).error(), expected_error); return true; } int main() { - if ( - json_pointer_success_test("") && - json_pointer_success_test("~1~001abc") && - json_pointer_success_test("~1~001abc/1") && - json_pointer_success_test("~1~001abc/1/\\\" 0") && - json_pointer_success_test("~1~001abc/1/\\\" 0/0", "value0") && - json_pointer_success_test("~1~001abc/1/\\\" 0/1", "value1") && - json_pointer_failure_test("~1~001abc/1/\\\" 0/2", INDEX_OUT_OF_BOUNDS) && // index actually out of bounds - json_pointer_success_test("arr") && // get array - json_pointer_failure_test("arr/0", INDEX_OUT_OF_BOUNDS) && // array index 0 out of bounds on empty array - json_pointer_success_test("~1~001abc") && // get object - json_pointer_success_test("0", "0 ok") && // object index with integer-ish key - json_pointer_success_test("01", "01 ok") && // object index with key that would be an invalid integer - json_pointer_success_test("", "empty ok") && // object index with empty key - json_pointer_failure_test("~01abc", NO_SUCH_FIELD) && // Test that we don't try to compare the literal key - json_pointer_failure_test("~1~001abc/01", INVALID_JSON_POINTER) && // Leading 0 in integer index - json_pointer_failure_test("~1~001abc/", INVALID_JSON_POINTER) && // Empty index to array - json_pointer_failure_test("~1~001abc/-", INDEX_OUT_OF_BOUNDS) && // End index is always out of bounds - true + if (true + && json_pointer_success_test("") + && json_pointer_success_test("~1~001abc") + && json_pointer_success_test("~1~001abc/1") + && json_pointer_success_test("~1~001abc/1/\\\" 0") + && json_pointer_success_test("~1~001abc/1/\\\" 0/0", "value0") + && json_pointer_success_test("~1~001abc/1/\\\" 0/1", "value1") + && json_pointer_failure_test("~1~001abc/1/\\\" 0/2", INDEX_OUT_OF_BOUNDS) // index actually out of bounds + && json_pointer_success_test("arr") // get array + && json_pointer_failure_test("arr/0", INDEX_OUT_OF_BOUNDS) // array index 0 out of bounds on empty array + && json_pointer_success_test("~1~001abc") // get object + && json_pointer_success_test("0", "0 ok") // object index with integer-ish key + && json_pointer_success_test("01", "01 ok") // object index with key that would be an invalid integer + && json_pointer_success_test("", "empty ok") // object index with empty key + && json_pointer_failure_test("~01abc", NO_SUCH_FIELD) // Test that we don't try to compare the literal key + && json_pointer_failure_test("~1~001abc/01", INVALID_JSON_POINTER) // Leading 0 in integer index + && json_pointer_failure_test("~1~001abc/", INVALID_JSON_POINTER) // Empty index to array + && json_pointer_failure_test("~1~001abc/-", INDEX_OUT_OF_BOUNDS) // End index is always out of bounds ) { std::cout << "Success!" << std::endl; return 0; diff --git a/tests/readme_examples.cpp b/tests/readme_examples.cpp index 692b9492a..1469d3d2c 100644 --- a/tests/readme_examples.cpp +++ b/tests/readme_examples.cpp @@ -88,7 +88,7 @@ void basics_dom_4() { auto abstract_json = R"( { "str" : { "123" : {"abc" : 3.14 } } } )"_padded; dom::parser parser; - double v = parser.parse(abstract_json)["str"]["123"]["abc"].get(); + double v = parser.parse(abstract_json)["str"]["123"]["abc"]; cout << "number: " << v << endl; } @@ -141,9 +141,10 @@ namespace treewalk_1 { #ifdef SIMDJSON_CPLUSPLUS17 void basics_cpp17_1() { - dom::parser parser; padded_string json = R"( { "foo": 1, "bar": 2 } )"_padded; - auto [object, error] = parser.parse(json).get(); + dom::parser parser; + dom::object object; + auto error = parser.parse(json).get(object); if (error) { cerr << error << endl; return; } for (auto [key, value] : object) { cout << key << " = " << value << endl; @@ -153,11 +154,10 @@ void basics_cpp17_1() { void basics_cpp17_2() { // C++ 11 version for comparison - dom::parser parser; padded_string json = R"( { "foo": 1, "bar": 2 } )"_padded; - simdjson::error_code error; + dom::parser parser; dom::object object; - error = parser.parse(json).get(object); + auto error = parser.parse(json).get(object); if (!error) { cerr << error << endl; return; } for (dom::key_value_pair field : object) { cout << field.key << " = " << field.value << endl; @@ -216,26 +216,28 @@ SIMDJSON_PUSH_DISABLE_ALL_WARNINGS // The web_request part of this is aspirational, so we compile as much as we can here void performance_2() { dom::parser parser(1000*1000); // Never grow past documents > 1MB -// for (web_request request : listen()) { - auto [doc, error] = parser.parse("1"_padded/*request.body*/); -// // If the document was above our limit, emit 413 = payload too large + /* for (web_request request : listen()) */ { + dom::element doc; + auto error = parser.parse("1"_padded/*request.body*/).get(doc); + // If the document was above our limit, emit 413 = payload too large if (error == CAPACITY) { /* request.respond(413); continue; */ } -// // ... -// } + // ... + } } // The web_request part of this is aspirational, so we compile as much as we can here void performance_3() { dom::parser parser(0); // This parser will refuse to automatically grow capacity - simdjson::error_code allocate_error = parser.allocate(1000*1000); // This allocates enough capacity to handle documents <= 1MB - if (allocate_error) { cerr << allocate_error << endl; exit(1); } + auto error = parser.allocate(1000*1000); // This allocates enough capacity to handle documents <= 1MB + if (error) { cerr << error << endl; exit(1); } - // for (web_request request : listen()) { - auto [doc, error] = parser.parse("1"_padded/*request.body*/); + /* for (web_request request : listen()) */ { + dom::element doc; + auto error = parser.parse("1"_padded/*request.body*/).get(doc); // If the document was above our limit, emit 413 = payload too large if (error == CAPACITY) { /* request.respond(413); continue; */ } // ... - // } + } } SIMDJSON_POP_DISABLE_WARNINGS #endif diff --git a/tests/readme_examples_noexceptions.cpp b/tests/readme_examples_noexceptions.cpp index 36bf89ced..e668a0f40 100644 --- a/tests/readme_examples_noexceptions.cpp +++ b/tests/readme_examples_noexceptions.cpp @@ -10,7 +10,8 @@ void basics_error_1() { dom::parser parser; auto json = "1"_padded; - auto [doc, error] = parser.parse(json); // doc is a dom::element + dom::element doc; + auto error = parser.parse(json).get(doc); if (error) { cerr << error << endl; exit(1); } // Use document here now that we've checked for the error } @@ -18,14 +19,6 @@ SIMDJSON_POP_DISABLE_WARNINGS #endif void basics_error_2() { - dom::parser parser; - auto json = "1"_padded; - - dom::element doc; - UNUSED auto error = parser.parse(json).get(doc); // <-- Assigns to doc and error just like "auto [doc, error]"} -} - -void basics_error_3() { auto cars_json = R"( [ { "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] }, { "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] }, @@ -70,8 +63,7 @@ void basics_error_3() { } } - -void basics_error_4() { +void basics_error_3() { auto abstract_json = R"( [ { "12345" : {"a":12.34, "b":56.78, "c": 9998877} }, { "12545" : {"a":11.44, "b":12.78, "c": 11111111} } @@ -102,7 +94,7 @@ void basics_error_4() { } } -void basics_error_5() { +void basics_error_4() { auto abstract_json = R"( { "str" : { "123" : {"abc" : 3.14 } } } )"_padded; dom::parser parser; @@ -116,7 +108,7 @@ void basics_error_5() { #ifdef SIMDJSON_CPLUSPLUS17 -void basics_error_3_cpp17() { +void basics_error_2_cpp17() { auto cars_json = R"( [ { "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] }, { "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] }, @@ -201,6 +193,5 @@ int main() { basics_error_2(); basics_error_3(); basics_error_4(); - basics_error_5(); return EXIT_SUCCESS; } diff --git a/tests/singleheadertest.cpp b/tests/singleheadertest.cpp index b2d0051d8..46a0e3cb0 100644 --- a/tests/singleheadertest.cpp +++ b/tests/singleheadertest.cpp @@ -6,12 +6,13 @@ using namespace simdjson; int main() { const char *filename = SIMDJSON_BENCHMARK_DATA_DIR "/twitter.json"; - padded_string p = get_corpus(filename); dom::parser parser; - auto [doc, error] = parser.parse(p); - if(error) { + dom::element doc; + auto error = parser.load(filename).get(doc); + if (error) { std::cerr << error << std::endl; return EXIT_FAILURE; } + std::cout << doc << std::endl; return EXIT_SUCCESS; } diff --git a/tests/stringparsingcheck.cpp b/tests/stringparsingcheck.cpp index eeec5900b..cefaa637b 100644 --- a/tests/stringparsingcheck.cpp +++ b/tests/stringparsingcheck.cpp @@ -337,7 +337,8 @@ bool validate(const char *dirname) { } else { strcpy(fullpath + dirlen, name); } - auto [p, error] = simdjson::padded_string::load(fullpath); + simdjson::padded_string p; + auto error = simdjson::padded_string::load(fullpath).get(p); if (error) { std::cerr << "Could not load the file " << fullpath << std::endl; return EXIT_FAILURE; diff --git a/tests/test_macros.h b/tests/test_macros.h index 909deb10e..5c46c0765 100644 --- a/tests/test_macros.h +++ b/tests/test_macros.h @@ -18,17 +18,38 @@ const char *SMALLDEMO_JSON = SIMDJSON_BENCHMARK_SMALLDATA_DIR "smalldemo.json"; const char *TRUENULL_JSON = SIMDJSON_BENCHMARK_SMALLDATA_DIR "truenull.json"; // For the ASSERT_EQUAL macro -template -bool equals_expected(T actual, T expected) { - return actual == expected; +template +really_inline bool equals_expected(T actual, S expected) { + return actual == T(expected); } template<> -bool equals_expected(const char *actual, const char *expected) { +really_inline bool equals_expected(const char *actual, const char *expected) { return !strcmp(actual, expected); } -#define ASSERT_EQUAL(ACTUAL, EXPECTED) if (!equals_expected(ACTUAL, EXPECTED)) { std::cerr << "Expected " << #ACTUAL << " to be " << (EXPECTED) << ", got " << (ACTUAL) << " instead!" << std::endl; return false; } + +really_inline simdjson::error_code to_error_code(simdjson::error_code error) { + return error; +} +template +really_inline simdjson::error_code to_error_code(const simdjson::simdjson_result &result) { + return result.error(); +} + +#define TEST_START() { cout << "Running " << __func__ << " ..." << endl; } +#define ASSERT_EQUAL(ACTUAL, EXPECTED) \ +do { \ + auto _actual = (ACTUAL); \ + auto _expected = (EXPECTED); \ + if (!equals_expected(_actual, _expected)) { \ + std::cerr << "Expected " << (#ACTUAL) << " to be " << _expected << ", got " << _actual << " instead!" << std::endl; \ + return false; \ + } \ +} while(0); +#define ASSERT_ERROR(ACTUAL, EXPECTED) do { auto _actual = to_error_code(ACTUAL); auto _expected = to_error_code(EXPECTED); if (_actual != _expected) { std::cerr << "FAIL: Unexpected error \"" << _actual << "\" (expected \"" << _expected << "\")" << std::endl; return false; } } while (0); #define ASSERT(RESULT, MESSAGE) if (!(RESULT)) { std::cerr << MESSAGE << std::endl; return false; } #define RUN_TEST(RESULT) if (!RESULT) { return false; } -#define ASSERT_SUCCESS(ERROR) if (ERROR) { std::cerr << (ERROR) << std::endl; return false; } +#define ASSERT_SUCCESS(ERROR) do { auto _error = to_error_code(ERROR); if (_error) { std::cerr << _error << std::endl; return false; } } while(0); +#define TEST_FAIL(MESSAGE) { std::cerr << "FAIL: " << (MESSAGE) << std::endl; return false; } +#define TEST_SUCCEED() { return true; } #endif // TEST_MACROS_H \ No newline at end of file diff --git a/tools/json2json.cpp b/tools/json2json.cpp index d02ba74c5..084ad3ca2 100644 --- a/tools/json2json.cpp +++ b/tools/json2json.cpp @@ -49,10 +49,10 @@ int main(int argc, char *argv[]) { const char *filename = result["file"].as().c_str(); simdjson::dom::parser parser; - auto [doc, error] = parser.load(filename); // do the parsing, return false on error - if (error != simdjson::SUCCESS) { - std::cerr << " Parsing failed. Error is '" << simdjson::error_message(error) - << "'." << std::endl; + simdjson::dom::element doc; + auto error = parser.load(filename).get(doc); // do the parsing, return false on error + if (error) { + std::cerr << " Parsing failed. Error is '" << error << "'." << std::endl; return EXIT_FAILURE; } if(rawdump) { diff --git a/tools/jsonpointer.cpp b/tools/jsonpointer.cpp index c5f0c6f55..ca6311a1b 100644 --- a/tools/jsonpointer.cpp +++ b/tools/jsonpointer.cpp @@ -20,16 +20,17 @@ int main(int argc, char *argv[]) { const char *filename = argv[1]; simdjson::dom::parser parser; - auto [doc, error] = parser.load(filename); + simdjson::dom::element doc; + auto error = parser.load(filename).get(doc); if (error) { std::cerr << "Error parsing " << filename << ": " << error << std::endl; } std::cout << "[" << std::endl; for (int idx = 2; idx < argc; idx++) { const char *json_pointer = argv[idx]; - auto [value, pointer_error] = doc[json_pointer]; + simdjson::dom::element value; std::cout << "{\"jsonpath\": \"" << json_pointer << "\""; - if (pointer_error) { - std::cout << ",\"error\":\"" << pointer_error << "\""; + if ((error = doc[json_pointer].get(value))) { + std::cout << ",\"error\":\"" << error << "\""; } else { std::cout << ",\"value\":" << value; } diff --git a/tools/jsonstats.cpp b/tools/jsonstats.cpp index 4f2aa4208..977d6a480 100644 --- a/tools/jsonstats.cpp +++ b/tools/jsonstats.cpp @@ -166,7 +166,8 @@ void recurse(simdjson::dom::element element, stat_t &s, size_t depth) { stat_t simdjson_compute_stats(const simdjson::padded_string &p) { stat_t s{}; simdjson::dom::parser parser; - auto [doc, error] = parser.parse(p); + simdjson::dom::element doc; + auto error = parser.parse(p).get(doc); if (error) { s.valid = false; std::cerr << error << std::endl; @@ -217,7 +218,8 @@ int main(int argc, char *argv[]) { const char *filename = result["file"].as().c_str(); - auto [p, error] = simdjson::padded_string::load(filename); + simdjson::padded_string p; + auto error = simdjson::padded_string::load(filename).get(p); if (error) { std::cerr << "Could not load the file " << filename << std::endl; return EXIT_FAILURE; diff --git a/tools/minify.cpp b/tools/minify.cpp index f9f264c42..2e2acebd2 100644 --- a/tools/minify.cpp +++ b/tools/minify.cpp @@ -56,7 +56,8 @@ int main(int argc, char *argv[]) { std::string filename = result["file"].as(); - auto [p, error] = simdjson::padded_string::load(filename); + simdjson::padded_string p; + auto error = simdjson::padded_string::load(filename).get(p); if (error) { std::cerr << "Could not load the file " << filename << std::endl; return EXIT_FAILURE;