From 94440e0170d6be60c8eda02b99b49850d9037047 Mon Sep 17 00:00:00 2001 From: John Keiser Date: Sat, 20 Jun 2020 15:10:10 -0700 Subject: [PATCH 01/10] Return simdjson_result from load_many/parse_many --- include/simdjson/dom/document_stream.h | 38 +++++++++++------ include/simdjson/dom/parser.h | 18 ++++---- include/simdjson/inline/document_stream.h | 50 +++++++++++++++-------- include/simdjson/inline/parser.h | 19 ++++----- 4 files changed, 76 insertions(+), 49 deletions(-) diff --git a/include/simdjson/dom/document_stream.h b/include/simdjson/dom/document_stream.h index 947ebf8ce..805de46c9 100644 --- a/include/simdjson/dom/document_stream.h +++ b/include/simdjson/dom/document_stream.h @@ -133,7 +133,10 @@ private: document_stream &operator=(const document_stream &) = delete; // Disallow copying - document_stream(document_stream &other) = delete; // Disallow copying + document_stream(document_stream &other) = delete; // Disallow copying + + /** Construct an uninitialized document_stream **/ + really_inline document_stream() noexcept; /** * Construct a document_stream. Does not allocate or parse anything until the iterator is @@ -141,18 +144,9 @@ private: */ really_inline document_stream( dom::parser &parser, - size_t batch_size, const uint8_t *buf, - size_t len - ) noexcept; - - /** - * Construct a document_stream with an initial error. - */ - really_inline document_stream( - dom::parser &parser, - size_t batch_size, - error_code error + size_t len, + size_t batch_size ) noexcept; /** @@ -229,12 +223,32 @@ private: #endif // SIMDJSON_THREADS_ENABLED friend class dom::parser; + friend class internal::simdjson_result_base; size_t doc_index{}; }; // class document_stream } // namespace dom + +template<> +struct simdjson_result : public internal::simdjson_result_base { +public: + really_inline simdjson_result() noexcept; ///< @private + really_inline simdjson_result(error_code error) noexcept; ///< @private + really_inline simdjson_result(dom::document_stream &&value) noexcept; ///< @private + +#if SIMDJSON_EXCEPTIONS + really_inline dom::document_stream::iterator begin() noexcept(false); + really_inline dom::document_stream::iterator end() noexcept(false); +#else // SIMDJSON_EXCEPTIONS + [[deprecated("parse_many() and load_many() may return errors. Use document_stream stream; error = parser.parse_many().get(doc); instead.")]] + really_inline dom::document_stream::iterator begin() noexcept; + [[deprecated("parse_many() and load_many() may return errors. Use document_stream stream; error = parser.parse_many().get(doc); instead.")]] + really_inline dom::document_stream::iterator end() noexcept; +#endif // SIMDJSON_EXCEPTIONS +}; // struct simdjson_result + } // namespace simdjson #endif // SIMDJSON_DOCUMENT_STREAM_H diff --git a/include/simdjson/dom/parser.h b/include/simdjson/dom/parser.h index 49ca234da..d1104d35c 100644 --- a/include/simdjson/dom/parser.h +++ b/include/simdjson/dom/parser.h @@ -193,14 +193,13 @@ public: * spot is cache-related: small enough to fit in cache, yet big enough to * parse as many documents as possible in one tight loop. * Defaults to 10MB, which has been a reasonable sweet spot in our tests. - * @return The stream. If there is an error, it will be returned during iteration. An empty input - * will yield 0 documents rather than an EMPTY error. Errors: + * @return The stream, or an error. An empty input will yield 0 documents rather than an EMPTY error. Errors: * - IO_ERROR if there was an error opening or reading the file. * - MEMALLOC if the parser does not have enough capacity and memory allocation fails. * - CAPACITY if the parser does not have enough capacity and batch_size > max_capacity. * - other json errors if parsing fails. */ - inline document_stream load_many(const std::string &path, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept; + inline simdjson_result load_many(const std::string &path, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept; /** * Parse a buffer containing many JSON documents. @@ -260,22 +259,21 @@ public: * spot is cache-related: small enough to fit in cache, yet big enough to * parse as many documents as possible in one tight loop. * Defaults to 10MB, which has been a reasonable sweet spot in our tests. - * @return The stream. If there is an error, it will be returned during iteration. An empty input - * will yield 0 documents rather than an EMPTY error. Errors: + * @return The stream, or an error. An empty input will yield 0 documents rather than an EMPTY error. Errors: * - MEMALLOC if the parser does not have enough capacity and memory allocation fails * - CAPACITY if the parser does not have enough capacity and batch_size > max_capacity. * - other json errors if parsing fails. */ - inline document_stream parse_many(const uint8_t *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept; + inline simdjson_result parse_many(const uint8_t *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept; /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ - inline document_stream parse_many(const char *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept; + inline simdjson_result parse_many(const char *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept; /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ - inline document_stream parse_many(const std::string &s, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept; + inline simdjson_result parse_many(const std::string &s, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept; /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ - inline document_stream parse_many(const padded_string &s, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept; + inline simdjson_result parse_many(const padded_string &s, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept; /** @private We do not want to allow implicit conversion from C string to std::string. */ - really_inline simdjson_result parse_many(const char *buf, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept = delete; + simdjson_result parse_many(const char *buf, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept = delete; /** * Ensure this parser has enough memory to process JSON documents up to `capacity` bytes in length diff --git a/include/simdjson/inline/document_stream.h b/include/simdjson/inline/document_stream.h index 3e72305b8..1da8bf6d1 100644 --- a/include/simdjson/inline/document_stream.h +++ b/include/simdjson/inline/document_stream.h @@ -66,9 +66,9 @@ inline void stage1_worker::run(document_stream * ds, dom::parser * stage1, size_ really_inline document_stream::document_stream( dom::parser &_parser, - size_t _batch_size, const uint8_t *_buf, - size_t _len + size_t _len, + size_t _batch_size ) noexcept : parser{_parser}, buf{_buf}, @@ -83,20 +83,6 @@ really_inline document_stream::document_stream( #endif } -really_inline document_stream::document_stream( - dom::parser &_parser, - size_t _batch_size, - error_code _error -) noexcept - : parser{_parser}, - buf{nullptr}, - len{0}, - batch_size{_batch_size}, - error{_error} -{ - assert(_error); -} - inline document_stream::~document_stream() noexcept { } @@ -226,5 +212,37 @@ inline void document_stream::start_stage1_thread() noexcept { #endif // SIMDJSON_THREADS_ENABLED } // namespace dom + +really_inline simdjson_result::simdjson_result() noexcept + : simdjson_result_base() { +} +really_inline simdjson_result::simdjson_result(error_code error) noexcept + : simdjson_result_base(error) { +} +really_inline simdjson_result::simdjson_result(dom::document_stream &&value) noexcept + : simdjson_result_base(std::forward(value)) { +} + +#if SIMDJSON_EXCEPTIONS +really_inline dom::document_stream::iterator simdjson_result::begin() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first.begin(); +} +really_inline dom::document_stream::iterator simdjson_result::end() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first.end(); +} +#else // SIMDJSON_EXCEPTIONS +really_inline dom::document_stream::iterator simdjson_result::begin() noexcept { + first.error = error(); + return first.begin(); +} +really_inline dom::document_stream::iterator simdjson_result::end() noexcept { + first.error = error(); + return first.end(); +} +#endif // SIMDJSON_EXCEPTIONS + + } // namespace simdjson #endif // SIMDJSON_INLINE_DOCUMENT_STREAM_H diff --git a/include/simdjson/inline/parser.h b/include/simdjson/inline/parser.h index 16bd47c62..fdfe49e6b 100644 --- a/include/simdjson/inline/parser.h +++ b/include/simdjson/inline/parser.h @@ -80,17 +80,14 @@ inline simdjson_result parser::load(const std::string &path) & noexcept size_t len; auto _error = read_file(path).get(len); if (_error) { return _error; } - return parse(loaded_bytes.get(), len, false); } -inline document_stream parser::load_many(const std::string &path, size_t batch_size) noexcept { +inline simdjson_result parser::load_many(const std::string &path, size_t batch_size) noexcept { size_t len; auto _error = read_file(path).get(len); - if (_error) { - return document_stream(*this, batch_size, _error); - } - return document_stream(*this, batch_size, (const uint8_t*)loaded_bytes.get(), len); + if (_error) { return _error; } + return document_stream(*this, (const uint8_t*)loaded_bytes.get(), len, batch_size); } inline simdjson_result parser::parse(const uint8_t *buf, size_t len, bool realloc_if_needed) & noexcept { @@ -123,16 +120,16 @@ really_inline simdjson_result parser::parse(const padded_string &s) & n return parse(s.data(), s.length(), false); } -inline document_stream parser::parse_many(const uint8_t *buf, size_t len, size_t batch_size) noexcept { - return document_stream(*this, batch_size, buf, len); +inline simdjson_result parser::parse_many(const uint8_t *buf, size_t len, size_t batch_size) noexcept { + return document_stream(*this, buf, len, batch_size); } -inline document_stream parser::parse_many(const char *buf, size_t len, size_t batch_size) noexcept { +inline simdjson_result parser::parse_many(const char *buf, size_t len, size_t batch_size) noexcept { return parse_many((const uint8_t *)buf, len, batch_size); } -inline document_stream parser::parse_many(const std::string &s, size_t batch_size) noexcept { +inline simdjson_result parser::parse_many(const std::string &s, size_t batch_size) noexcept { return parse_many(s.data(), s.length(), batch_size); } -inline document_stream parser::parse_many(const padded_string &s, size_t batch_size) noexcept { +inline simdjson_result parser::parse_many(const padded_string &s, size_t batch_size) noexcept { return parse_many(s.data(), s.length(), batch_size); } From 9899e5021da9d08d565418a03c4ef848e66a0150 Mon Sep 17 00:00:00 2001 From: John Keiser Date: Sat, 20 Jun 2020 15:47:41 -0700 Subject: [PATCH 02/10] Allow use of document_stream with tie() --- benchmark/parse_stream.cpp | 22 +++++-- include/simdjson/dom/document_stream.h | 34 ++++++----- include/simdjson/dom/element.h | 19 ------ include/simdjson/dom/parser.h | 2 +- include/simdjson/inline/document_stream.h | 32 ++++++---- include/simdjson/inline/error.h | 4 +- tests/basictests.cpp | 42 +++++++------- tests/cast_tester.h | 71 ++++++++++++++--------- tests/errortests.cpp | 17 ++++-- tests/parse_many_test.cpp | 9 ++- tests/test_macros.h | 4 +- 11 files changed, 145 insertions(+), 111 deletions(-) diff --git a/benchmark/parse_stream.cpp b/benchmark/parse_stream.cpp index 5735d4176..d893fd1d4 100644 --- a/benchmark/parse_stream.cpp +++ b/benchmark/parse_stream.cpp @@ -87,10 +87,15 @@ int main(int argc, char *argv[]) { auto start = std::chrono::steady_clock::now(); count = 0; - for (auto result : parser.parse_many(p, i)) { + simdjson::dom::document_stream docs; + if ((error = parser.parse_many(p, i).get(docs))) { + std::wcerr << "Parsing failed with: " << error << std::endl; + exit(1); + } + for (auto result : docs) { error = result.error(); - if (error != simdjson::SUCCESS) { - std::wcerr << "Parsing failed with: " << error_message(error) << std::endl; + if (error) { + std::wcerr << "Parsing failed with: " << error << std::endl; exit(1); } count++; @@ -134,10 +139,15 @@ int main(int argc, char *argv[]) { auto start = std::chrono::steady_clock::now(); // This includes allocation of the parser - for (auto result : parser.parse_many(p, optimal_batch_size)) { + simdjson::dom::document_stream docs; + if ((error = parser.parse_many(p, optimal_batch_size).get(docs))) { + std::wcerr << "Parsing failed with: " << error << std::endl; + exit(1); + } + for (auto result : docs) { error = result.error(); - if (error != simdjson::SUCCESS) { - std::wcerr << "Parsing failed with: " << error_message(error) << std::endl; + if (error) { + std::wcerr << "Parsing failed with: " << error << std::endl; exit(1); } } diff --git a/include/simdjson/dom/document_stream.h b/include/simdjson/dom/document_stream.h index 805de46c9..47c0ab19d 100644 --- a/include/simdjson/dom/document_stream.h +++ b/include/simdjson/dom/document_stream.h @@ -72,8 +72,20 @@ private: */ class document_stream { public: + /** + * Construct an uninitialized document_stream. + * + * ```c++ + * document_stream docs; + * error = parser.parse_many(json).get(docs); + * ``` + */ + really_inline document_stream() noexcept; /** Move one document_stream to another. */ - really_inline document_stream(document_stream && other) noexcept = default; + really_inline document_stream(document_stream &&other) noexcept = default; + /** Move one document_stream to another. */ + really_inline document_stream &operator=(document_stream &&other) noexcept = default; + really_inline ~document_stream() noexcept; /** @@ -99,9 +111,8 @@ public: * * Gives the current index in the input document in bytes. * - * auto stream = parser.parse_many(json,window); - * auto i = stream.begin(); - * for(; i != stream.end(); ++i) { + * document_stream stream = parser.parse_many(json,window); + * for(auto i = stream.begin(); i != stream.end(); ++i) { * auto doc = *i; * size_t index = i.current_index(); * } @@ -135,9 +146,6 @@ private: document_stream(document_stream &other) = delete; // Disallow copying - /** Construct an uninitialized document_stream **/ - really_inline document_stream() noexcept; - /** * Construct a document_stream. Does not allocate or parse anything until the iterator is * used. @@ -193,13 +201,14 @@ private: /** Pass the next batch through stage 1 with the given parser. */ inline error_code run_stage1(dom::parser &p, size_t batch_start) noexcept; - dom::parser &parser; + dom::parser *parser; const uint8_t *buf; - const size_t len; - const size_t batch_size; - size_t batch_start{0}; + size_t len; + size_t batch_size; /** The error (or lack thereof) from the current document. */ error_code error; + size_t batch_start{0}; + size_t doc_index{}; #ifdef SIMDJSON_THREADS_ENABLED inline void load_from_stage1_thread() noexcept; @@ -223,10 +232,9 @@ private: #endif // SIMDJSON_THREADS_ENABLED friend class dom::parser; + friend class simdjson_result; friend class internal::simdjson_result_base; - size_t doc_index{}; - }; // class document_stream } // namespace dom diff --git a/include/simdjson/dom/element.h b/include/simdjson/dom/element.h index ea84d4f00..6d9af51dd 100644 --- a/include/simdjson/dom/element.h +++ b/include/simdjson/dom/element.h @@ -243,25 +243,6 @@ public: template inline void tie(T &value, error_code &error) && noexcept; - /** - * Get the value as the provided type (T). - * - * Supported types: - * - Boolean: bool - * - Number: double, uint64_t, int64_t - * - String: std::string_view, const char * - * - Array: dom::array - * - Object: dom::object - * - * @tparam T bool, double, uint64_t, int64_t, std::string_view, const char *, dom::array, dom::object - * - * @param value The variable to set to the given type. value is undefined if there is an error. - * - * @returns true if the value was able to be set, false if there was an error. - */ - template - WARN_UNUSED inline bool tie(T &value) && noexcept; - #if SIMDJSON_EXCEPTIONS /** * Read this element as a boolean. diff --git a/include/simdjson/dom/parser.h b/include/simdjson/dom/parser.h index d1104d35c..ba3d9b668 100644 --- a/include/simdjson/dom/parser.h +++ b/include/simdjson/dom/parser.h @@ -205,7 +205,7 @@ public: * Parse a buffer containing many JSON documents. * * dom::parser parser; - * for (const element doc : parser.parse_many(buf, len)) { + * for (element doc : parser.parse_many(buf, len)) { * cout << std::string(doc["title"]) << endl; * } * diff --git a/include/simdjson/inline/document_stream.h b/include/simdjson/inline/document_stream.h index 1da8bf6d1..e28a1325b 100644 --- a/include/simdjson/inline/document_stream.h +++ b/include/simdjson/inline/document_stream.h @@ -70,7 +70,7 @@ really_inline document_stream::document_stream( size_t _len, size_t _batch_size ) noexcept - : parser{_parser}, + : parser{&_parser}, buf{_buf}, len{_len}, batch_size{_batch_size}, @@ -83,7 +83,15 @@ really_inline document_stream::document_stream( #endif } -inline document_stream::~document_stream() noexcept { +really_inline document_stream::document_stream() noexcept + : parser{nullptr}, + buf{nullptr}, + len{0}, + batch_size{0}, + error{UNINITIALIZED} { +} + +really_inline document_stream::~document_stream() noexcept { } really_inline document_stream::iterator document_stream::begin() noexcept { @@ -103,7 +111,7 @@ really_inline document_stream::iterator::iterator(document_stream& _stream, bool really_inline simdjson_result document_stream::iterator::operator*() noexcept { // Once we have yielded any errors, we're finished. if (stream.error) { finished = true; return stream.error; } - return stream.parser.doc.root(); + return stream.parser->doc.root(); } really_inline document_stream::iterator& document_stream::iterator::operator++() noexcept { @@ -120,12 +128,12 @@ really_inline bool document_stream::iterator::operator!=(const document_stream:: inline void document_stream::start() noexcept { if (error) { return; } - error = parser.ensure_capacity(batch_size); + error = parser->ensure_capacity(batch_size); if (error) { return; } // Always run the first stage 1 parse immediately batch_start = 0; - error = run_stage1(parser, batch_start); + error = run_stage1(*parser, batch_start); if (error) { return; } #ifdef SIMDJSON_THREADS_ENABLED @@ -149,8 +157,8 @@ inline void document_stream::next() noexcept { if (error) { return; } // Load the next document from the batch - doc_index = batch_start + parser.implementation->structural_indexes[parser.implementation->next_structural_index]; - error = parser.implementation->stage2_next(parser.doc); + doc_index = batch_start + parser->implementation->structural_indexes[parser->implementation->next_structural_index]; + error = parser->implementation->stage2_next(parser->doc); // If that was the last document in the batch, load another batch (if available) while (error == EMPTY) { batch_start = next_batch_start(); @@ -159,17 +167,17 @@ inline void document_stream::next() noexcept { #ifdef SIMDJSON_THREADS_ENABLED load_from_stage1_thread(); #else - error = run_stage1(parser, batch_start); + error = run_stage1(*parser, batch_start); #endif if (error) { continue; } // If the error was EMPTY, we may want to load another batch. // Run stage 2 on the first document in the batch - doc_index = batch_start + parser.implementation->structural_indexes[parser.implementation->next_structural_index]; - error = parser.implementation->stage2_next(parser.doc); + doc_index = batch_start + parser->implementation->structural_indexes[parser->implementation->next_structural_index]; + error = parser->implementation->stage2_next(parser->doc); } } inline size_t document_stream::next_batch_start() const noexcept { - return batch_start + parser.implementation->structural_indexes[parser.implementation->n_structural_indexes]; + return batch_start + parser->implementation->structural_indexes[parser->implementation->n_structural_indexes]; } inline error_code document_stream::run_stage1(dom::parser &p, size_t _batch_start) noexcept { @@ -188,7 +196,7 @@ inline void document_stream::load_from_stage1_thread() noexcept { worker->finish(); // Swap to the parser that was loaded up in the thread. Make sure the parser has // enough memory to swap to, as well. - std::swap(parser, stage1_thread_parser); + std::swap(*parser, stage1_thread_parser); error = stage1_thread_error; if (error) { return; } diff --git a/include/simdjson/inline/error.h b/include/simdjson/inline/error.h index c2d884c3c..843a9b809 100644 --- a/include/simdjson/inline/error.h +++ b/include/simdjson/inline/error.h @@ -53,9 +53,7 @@ really_inline void simdjson_result_base::tie(T &value, error_code &error) && template WARN_UNUSED really_inline error_code simdjson_result_base::get(T &value) && noexcept { - error_code error; - std::forward>(*this).tie(value, error); - return error; + return std::forward>(*this).get(value); } template diff --git a/tests/basictests.cpp b/tests/basictests.cpp index 99a04d409..42c4ca417 100644 --- a/tests/basictests.cpp +++ b/tests/basictests.cpp @@ -385,7 +385,9 @@ namespace document_tests { namespace document_stream_tests { static simdjson::dom::document_stream parse_many_stream_return(simdjson::dom::parser &parser, simdjson::padded_string &str) { - return parser.parse_many(str); + simdjson::dom::document_stream stream; + UNUSED auto error = parser.parse_many(str).get(stream); + return stream; } // this is a compilation test UNUSED static void parse_many_stream_assign() { @@ -1032,11 +1034,11 @@ namespace dom_api_tests { if (doc["obj"]["a"].get().first != 1) { cerr << "Expected uint64_t(doc[\"obj\"][\"a\"]) to be 1, was " << doc["obj"]["a"].first << endl; return false; } object obj; - error = doc.get(obj); // tie(...) = fails with "no viable overloaded '='" on Apple clang version 11.0.0 + error = doc.get(obj); if (error) { cerr << "Error: " << error << endl; return false; } if (obj["obj"]["a"].get().first != 1) { cerr << "Expected uint64_t(doc[\"obj\"][\"a\"]) to be 1, was " << doc["obj"]["a"].first << endl; return false; } - error = obj["obj"].get(obj); // tie(...) = fails with "no viable overloaded '='" on Apple clang version 11.0.0 + error = obj["obj"].get(obj); if (obj["a"].get().first != 1) { cerr << "Expected uint64_t(obj[\"a\"]) to be 1, was " << obj["a"].first << endl; return false; } if (obj["b"].get().first != 2) { cerr << "Expected uint64_t(obj[\"b\"]) to be 2, was " << obj["b"].first << endl; return false; } if (obj["c/d"].get().first != 3) { cerr << "Expected uint64_t(obj[\"c\"]) to be 3, was " << obj["c"].first << endl; return false; } @@ -1065,19 +1067,17 @@ namespace dom_api_tests { // Print users with a default profile. set default_users; dom::parser parser; - auto [tweets, error] = parser.load(TWITTER_JSON)["statuses"].get(); + dom::array tweets; + auto error = parser.load(TWITTER_JSON)["statuses"].get(tweets); if (error) { cerr << "Error: " << error << endl; return false; } for (auto tweet : tweets) { object user; - error = tweet["user"].get(user); // tie(...) = fails with "no viable overloaded '='" on Apple clang version 11.0.0; - if (error) { cerr << "Error: " << error << endl; return false; } + if ((error = tweet["user"].get(user))) { cerr << "Error: " << error << endl; return false; } bool default_profile; - error = user["default_profile"].get(default_profile); // tie(...) = fails with "no viable overloaded '='" on Apple clang version 11.0.0; - if (error) { cerr << "Error: " << error << endl; return false; } + if ((error = user["default_profile"].get(default_profile))) { cerr << "Error: " << error << endl; return false; } if (default_profile) { std::string_view screen_name; - error = user["screen_name"].get(screen_name); // tie(...) = fails with "no viable overloaded '='" on Apple clang version 11.0.0; - if (error) { cerr << "Error: " << error << endl; return false; } + if ((error = user["screen_name"].get(screen_name))) { cerr << "Error: " << error << endl; return false; } default_users.insert(screen_name); } } @@ -1090,21 +1090,19 @@ namespace dom_api_tests { // Print image names and sizes set> image_sizes; dom::parser parser; - auto [tweets, error] = parser.load(TWITTER_JSON)["statuses"].get(); + dom::array tweets; + auto error = parser.load(TWITTER_JSON)["statuses"].get(tweets); if (error) { cerr << "Error: " << error << endl; return false; } for (auto tweet : tweets) { - auto [media, not_found] = tweet["entities"]["media"].get(); - if (!not_found) { + dom::array media; + if (not (error = tweet["entities"]["media"].get(media))) { for (auto image : media) { object sizes; - error = image["sizes"].get(sizes); // tie(...) = fails with "no viable overloaded '='" on Apple clang version 11.0.0; - if (error) { cerr << "Error: " << error << endl; return false; } + if ((error = image["sizes"].get(sizes))) { cerr << "Error: " << error << endl; return false; } for (auto size : sizes) { uint64_t width, height; - error = size.value["w"].get(width); // tie(...) = fails with "no viable overloaded '='" on Apple clang version 11.0.0; - if (error) { cerr << "Error: " << error << endl; return false; } - error = size.value["h"].get(height); // tie(...) = fails with "no viable overloaded '='" on Apple clang version 11.0.0; - if (error) { cerr << "Error: " << error << endl; return false; } + if ((error = size.value["w"].get(width))) { cerr << "Error: " << error << endl; return false; } + if ((error = size.value["h"].get(height))) { cerr << "Error: " << error << endl; return false; } image_sizes.insert(make_pair(width, height)); } } @@ -1343,7 +1341,9 @@ namespace type_tests { // Grab the element out and check success dom::element element = result.first; - RUN_TEST( tester.test_get(element, expected ) ); + RUN_TEST( tester.test_get_t(element, expected) ); + RUN_TEST( tester.test_get_t(result, expected) ); + RUN_TEST( tester.test_get(element, expected) ); RUN_TEST( tester.test_get(result, expected) ); // RUN_TEST( tester.test_named_get(element, expected) ); // RUN_TEST( tester.test_named_get(result, expected) ); @@ -1366,6 +1366,8 @@ namespace type_tests { // Grab the element out and check success dom::element element = result.first; + RUN_TEST( tester.test_get_t(element) ); + RUN_TEST( tester.test_get_t(result) ); RUN_TEST( tester.test_get(element) ); RUN_TEST( tester.test_get(result) ); RUN_TEST( tester.test_named_get(element) ); diff --git a/tests/cast_tester.h b/tests/cast_tester.h index 59cdc948d..00e90de74 100644 --- a/tests/cast_tester.h +++ b/tests/cast_tester.h @@ -26,6 +26,11 @@ public: bool test_get_error(element element, error_code expected_error); bool test_get_error(simdjson_result element, error_code expected_error); + bool test_get_t(element element, T expected = {}); + bool test_get_t(simdjson_result element, T expected = {}); + bool test_get_t_error(element element, error_code expected_error); + bool test_get_t_error(simdjson_result element, error_code expected_error); + #if SIMDJSON_EXCEPTIONS bool test_implicit_cast(element element, T expected = {}); bool test_implicit_cast(simdjson_result element, T expected = {}); @@ -57,68 +62,82 @@ private: template bool cast_tester::test_get(element element, T expected) { T actual; - error_code error; - error = element.get(actual); - ASSERT_SUCCESS(error); + ASSERT_SUCCESS(element.get(actual)); return assert_equal(actual, expected); } template bool cast_tester::test_get(simdjson_result element, T expected) { T actual; - error_code error; - error = element.get(actual); - ASSERT_SUCCESS(error); + ASSERT_SUCCESS(element.get(actual)); return assert_equal(actual, expected); } template bool cast_tester::test_get_error(element element, error_code expected_error) { T actual; - error_code error; - error = element.get(actual); - ASSERT_EQUAL(error, expected_error); + ASSERT_EQUAL(element.get(actual), expected_error); return true; } template bool cast_tester::test_get_error(simdjson_result element, error_code expected_error) { T actual; - error_code error; - error = element.get(actual); - ASSERT_EQUAL(error, expected_error); + ASSERT_EQUAL(element.get(actual), expected_error); + return true; +} + +template +bool cast_tester::test_get_t(element element, T expected) { + auto actual = element.get(); + ASSERT_SUCCESS(actual.error()); + return assert_equal(actual.first, expected); +} + +template +bool cast_tester::test_get_t(simdjson_result element, T expected) { + auto actual = element.get(); + ASSERT_SUCCESS(actual.error()); + return assert_equal(actual.first, expected); +} + +template +bool cast_tester::test_get_t_error(element element, error_code expected_error) { + ASSERT_EQUAL(element.get().error(), expected_error); + return true; +} + +template +bool cast_tester::test_get_t_error(simdjson_result element, error_code expected_error) { + ASSERT_EQUAL(element.get().error(), expected_error); return true; } template bool cast_tester::test_named_get(element element, T expected) { T actual; - auto error = named_get(element).get(actual); - ASSERT_SUCCESS(error); + ASSERT_SUCCESS(named_get(element).get(actual)); return assert_equal(actual, expected); } template bool cast_tester::test_named_get(simdjson_result element, T expected) { T actual; - auto error = named_get(element).get(actual); - ASSERT_SUCCESS(error); + ASSERT_SUCCESS(named_get(element).get(actual)); return assert_equal(actual, expected); } template bool cast_tester::test_named_get_error(element element, error_code expected_error) { T actual; - auto error = named_get(element).get(actual); - ASSERT_EQUAL(error, expected_error); + ASSERT_EQUAL(named_get(element).get(actual), expected_error); return true; } template bool cast_tester::test_named_get_error(simdjson_result element, error_code expected_error) { T actual; - auto error = named_get(element).get(actual); - ASSERT_EQUAL(error, expected_error); + ASSERT_EQUAL(named_get(element).get(actual), expected_error); return true; } @@ -188,8 +207,7 @@ bool cast_tester::test_is(element element, bool expected) { template bool cast_tester::test_is(simdjson_result element, bool expected) { bool actual; - auto error = element.is().get(actual); - ASSERT_SUCCESS(error); + ASSERT_SUCCESS(element.is().get(actual)); ASSERT_EQUAL(actual, expected); return true; } @@ -197,8 +215,7 @@ bool cast_tester::test_is(simdjson_result element, bool expected) { template bool cast_tester::test_is_error(simdjson_result element, error_code expected_error) { UNUSED bool actual; - auto error = element.is().get(actual); - ASSERT_EQUAL(error, expected_error); + ASSERT_EQUAL(element.is().get(actual), expected_error); return true; } @@ -211,8 +228,7 @@ bool cast_tester::test_named_is(element element, bool expected) { template bool cast_tester::test_named_is(simdjson_result element, bool expected) { bool actual; - auto error = named_is(element).get(actual); - ASSERT_SUCCESS(error); + ASSERT_SUCCESS(named_is(element).get(actual)); ASSERT_EQUAL(actual, expected); return true; } @@ -220,8 +236,7 @@ bool cast_tester::test_named_is(simdjson_result element, bool expect template bool cast_tester::test_named_is_error(simdjson_result element, error_code expected_error) { bool actual; - auto error = named_is(element).get(actual); - ASSERT_EQUAL(error, expected_error); + ASSERT_EQUAL(named_is(element).get(actual), expected_error); return true; } diff --git a/tests/errortests.cpp b/tests/errortests.cpp index 27f9bfc8a..aaaa9bcc3 100644 --- a/tests/errortests.cpp +++ b/tests/errortests.cpp @@ -21,6 +21,7 @@ const char *TWITTER_JSON = SIMDJSON_BENCHMARK_DATA_DIR "twitter.json"; #define TEST_START() { cout << "Running " << __func__ << " ..." << endl; } #define ASSERT_ERROR(ACTUAL, EXPECTED) if ((ACTUAL) != (EXPECTED)) { cerr << "FAIL: Unexpected error \"" << (ACTUAL) << "\" (expected \"" << (EXPECTED) << "\")" << endl; return false; } +#define ASSERT_SUCCESS(CODE) do { simdjson::error_code error = CODE; if (error) { cerr << "FAIL: Unexpected error " << error << endl; return false; } } while (0); #define TEST_FAIL(MESSAGE) { cerr << "FAIL: " << (MESSAGE) << endl; return false; } #define TEST_SUCCEED() { return true; } namespace parser_load { @@ -35,7 +36,9 @@ namespace parser_load { bool parser_load_many_capacity() { TEST_START(); dom::parser parser(1); // 1 byte max capacity - for (auto doc : parser.load_many(TWITTER_JSON)) { + dom::document_stream docs; + ASSERT_SUCCESS(parser.load_many(TWITTER_JSON).get(docs)); + for (auto doc : docs) { ASSERT_ERROR(doc.error(), CAPACITY); TEST_SUCCEED(); } @@ -47,7 +50,9 @@ namespace parser_load { const padded_string DOC = "1 2 [} 3"_padded; size_t count = 0; dom::parser parser; - for (auto doc : parser.parse_many(DOC)) { + dom::document_stream docs; + ASSERT_SUCCESS(parser.parse_many(DOC).get(docs)); + for (auto doc : docs) { count++; auto [val, error] = doc.get(); if (count == 3) { @@ -66,7 +71,9 @@ namespace parser_load { const padded_string DOC = "["_padded; size_t count = 0; dom::parser parser; - for (auto doc : parser.parse_many(DOC)) { + dom::document_stream docs; + ASSERT_SUCCESS(parser.parse_many(DOC).get(docs)); + for (auto doc : docs) { count++; ASSERT_ERROR(doc.error(), TAPE_ERROR); } @@ -79,7 +86,9 @@ namespace parser_load { const padded_string DOC = "1 2 ["_padded; size_t count = 0; dom::parser parser; - for (auto doc : parser.parse_many(DOC)) { + dom::document_stream docs; + ASSERT_SUCCESS(parser.parse_many(DOC).get(docs)); + for (auto doc : docs) { count++; auto [val, error] = doc.get(); if (count == 3) { diff --git a/tests/parse_many_test.cpp b/tests/parse_many_test.cpp index 66fba3a4e..9e3ab6f43 100644 --- a/tests/parse_many_test.cpp +++ b/tests/parse_many_test.cpp @@ -69,13 +69,16 @@ bool validate(const char *dirname) { snprintf(fullpath, fullpathlen, "%s%s%s", dirname, needsep ? "/" : "", name); /* The actual test*/ - auto [json, error] = simdjson::padded_string::load(fullpath); + simdjson::padded_string json; + auto error = simdjson::padded_string::load(fullpath).get(json); if (!error) { simdjson::dom::parser parser; ++how_many; - for (auto result : parser.parse_many(json)) { - error = result.error(); + simdjson::dom::document_stream docs; + error = parser.parse_many(json).get(docs); + for (auto doc : docs) { + error = doc.error(); } } printf("%s\n", error ? "ok" : "invalid"); diff --git a/tests/test_macros.h b/tests/test_macros.h index 909deb10e..b671f0b9b 100644 --- a/tests/test_macros.h +++ b/tests/test_macros.h @@ -26,9 +26,9 @@ template<> bool equals_expected(const char *actual, const char *expected) { return !strcmp(actual, expected); } -#define ASSERT_EQUAL(ACTUAL, EXPECTED) if (!equals_expected(ACTUAL, EXPECTED)) { std::cerr << "Expected " << #ACTUAL << " to be " << (EXPECTED) << ", got " << (ACTUAL) << " instead!" << std::endl; return false; } +#define ASSERT_EQUAL(ACTUAL, EXPECTED) do { auto _actual = (ACTUAL); auto _expected = (EXPECTED); if (!equals_expected(_actual, _expected)) { std::cerr << "Expected " << #ACTUAL << " to be " << _expected << ", got " << _actual << " instead!" << std::endl; return false; } } while(0); #define ASSERT(RESULT, MESSAGE) if (!(RESULT)) { std::cerr << MESSAGE << std::endl; return false; } #define RUN_TEST(RESULT) if (!RESULT) { return false; } -#define ASSERT_SUCCESS(ERROR) if (ERROR) { std::cerr << (ERROR) << std::endl; return false; } +#define ASSERT_SUCCESS(ERROR) do { auto _error = (ERROR); if (_error) { std::cerr << _error << std::endl; return false; } } while(0); #endif // TEST_MACROS_H \ No newline at end of file From ae1bd891e7003a68b109e1704bcc32f7a210bad1 Mon Sep 17 00:00:00 2001 From: John Keiser Date: Sat, 20 Jun 2020 22:03:57 -0700 Subject: [PATCH 03/10] Remove deprecated uses of parse_many --- include/simdjson/dom/document_stream.h | 4 +- include/simdjson/inline/document_stream.h | 1 - include/simdjson/inline/error.h | 4 +- singleheader/amalgamate.sh | 11 +- singleheader/amalgamate_demo.cpp | 13 +- singleheader/simdjson.cpp | 358 ++++--- singleheader/simdjson.h | 1031 ++++++++++++++++----- tests/basictests.cpp | 84 +- tests/errortests.cpp | 18 +- tests/test_macros.h | 5 + 10 files changed, 1150 insertions(+), 379 deletions(-) diff --git a/include/simdjson/dom/document_stream.h b/include/simdjson/dom/document_stream.h index 47c0ab19d..a650bf4ee 100644 --- a/include/simdjson/dom/document_stream.h +++ b/include/simdjson/dom/document_stream.h @@ -232,8 +232,8 @@ private: #endif // SIMDJSON_THREADS_ENABLED friend class dom::parser; - friend class simdjson_result; - friend class internal::simdjson_result_base; + friend struct simdjson_result; + friend struct internal::simdjson_result_base; }; // class document_stream diff --git a/include/simdjson/inline/document_stream.h b/include/simdjson/inline/document_stream.h index e28a1325b..ccd4d9e31 100644 --- a/include/simdjson/inline/document_stream.h +++ b/include/simdjson/inline/document_stream.h @@ -251,6 +251,5 @@ really_inline dom::document_stream::iterator simdjson_result::tie(T &value, error_code &error) && template WARN_UNUSED really_inline error_code simdjson_result_base::get(T &value) && noexcept { - return std::forward>(*this).get(value); + error_code error; + std::forward>(*this).tie(value, error); + return error; } template diff --git a/singleheader/amalgamate.sh b/singleheader/amalgamate.sh index d6213ba0e..901cd1467 100755 --- a/singleheader/amalgamate.sh +++ b/singleheader/amalgamate.sh @@ -135,9 +135,8 @@ int main(int argc, char *argv[]) { } const char * filename = argv[1]; simdjson::dom::parser parser; - simdjson::error_code error; UNUSED simdjson::dom::element elem; - parser.load(filename).tie(elem, error); // do the parsing + auto error = parser.load(filename).get(elem); // do the parsing if (error) { std::cout << "parse failed" << std::endl; std::cout << "error code: " << error << std::endl; @@ -152,8 +151,12 @@ int main(int argc, char *argv[]) { // parse_many const char * filename2 = argv[2]; - for (auto result : parser.load_many(filename2)) { - error = result.error(); + simdjson::dom::document_stream stream; + error = parser.load_many(filename2).get(stream); + if (!error) { + for (auto result : stream) { + error = result.error(); + } } if (error) { std::cout << "parse_many failed" << std::endl; diff --git a/singleheader/amalgamate_demo.cpp b/singleheader/amalgamate_demo.cpp index a6ab5cf4d..94f984a95 100644 --- a/singleheader/amalgamate_demo.cpp +++ b/singleheader/amalgamate_demo.cpp @@ -1,4 +1,4 @@ -/* auto-generated on Fri 12 Jun 2020 13:09:36 EDT. Do not edit! */ +/* auto-generated on Sat Jun 20 21:35:29 PDT 2020. Do not edit! */ #include #include "simdjson.h" @@ -9,9 +9,8 @@ int main(int argc, char *argv[]) { } const char * filename = argv[1]; simdjson::dom::parser parser; - simdjson::error_code error; UNUSED simdjson::dom::element elem; - parser.load(filename).tie(elem, error); // do the parsing + auto error = parser.load(filename).get(elem); // do the parsing if (error) { std::cout << "parse failed" << std::endl; std::cout << "error code: " << error << std::endl; @@ -26,8 +25,12 @@ int main(int argc, char *argv[]) { // parse_many const char * filename2 = argv[2]; - for (auto result : parser.load_many(filename2)) { - error = result.error(); + simdjson::dom::document_stream stream; + error = parser.load_many(filename2).get(stream); + if (!error) { + for (auto result : stream) { + error = result.error(); + } } if (error) { std::cout << "parse_many failed" << std::endl; diff --git a/singleheader/simdjson.cpp b/singleheader/simdjson.cpp index d99dc8b67..ab26d2ad9 100644 --- a/singleheader/simdjson.cpp +++ b/singleheader/simdjson.cpp @@ -1,4 +1,4 @@ -/* auto-generated on Fri 12 Jun 2020 13:09:36 EDT. Do not edit! */ +/* auto-generated on Sat Jun 20 21:35:29 PDT 2020. Do not edit! */ /* begin file src/simdjson.cpp */ #include "simdjson.h" @@ -586,6 +586,11 @@ const implementation *detect_best_supported_implementation_on_first_use::set_bes SIMDJSON_DLLIMPORTEXPORT const internal::available_implementation_list available_implementations{}; SIMDJSON_DLLIMPORTEXPORT internal::atomic_ptr active_implementation{&internal::detect_best_supported_implementation_on_first_use_singleton}; +WARN_UNUSED error_code minify(const char *buf, size_t len, char *dst, size_t &dst_len) noexcept { + return active_implementation->minify((const uint8_t *)buf, len, (uint8_t *)dst, dst_len); +} + + } // namespace simdjson /* end file src/fallback/implementation.h */ @@ -2794,6 +2799,12 @@ really_inline simd8 must_be_continuation(simd8 prev1, simd8 must_be_2_3_continuation(simd8 prev2, simd8 prev3) { + simd8 is_third_byte = prev2 >= uint8_t(0b11100000u); + simd8 is_fourth_byte = prev3 >= uint8_t(0b11110000u); + return is_third_byte ^ is_fourth_byte; +} + /* begin file src/generic/stage1/buf_block_reader.h */ // Walks through a buffer in block-sized increments, loading the last part with spaces template @@ -2921,7 +2932,9 @@ public: really_inline error_code finish(bool streaming); private: + // Intended to be defined by the implementation really_inline uint64_t find_escaped(uint64_t escape); + really_inline uint64_t find_escaped_branchless(uint64_t escape); // Whether the last iteration was still inside a string (all 1's = true, all 0's = false). uint64_t prev_in_string = 0ULL; @@ -2956,7 +2969,7 @@ private: // desired | x | x x x x x x x x | // text | \\\ | \\\"\\\" \\\" \\"\\" | // -really_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { +really_inline uint64_t json_string_scanner::find_escaped_branchless(uint64_t backslash) { // If there was overflow, pretend the first character isn't a backslash backslash &= ~prev_escaped; uint64_t follows_escape = backslash << 1 | prev_escaped; @@ -2985,13 +2998,23 @@ really_inline json_string_block json_string_scanner::next(const simd::simd8x64(in_string) >> 63); + // Use ^ to turn the beginning quote off, and the end quote on. return { backslash, @@ -3117,6 +3140,15 @@ really_inline error_code json_scanner::finish(bool streaming) { } // namespace stage1 /* end file src/generic/stage1/json_scanner.h */ +namespace stage1 { +really_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { + // On ARM, we don't short-circuit this if there are no backslashes, because the branch gives us no + // benefit and therefore makes things worse. + // if (!backslash) { uint64_t escaped = prev_escaped; prev_escaped = 0; return escaped; } + return find_escaped_branchless(backslash); +} +} + /* begin file src/generic/stage1/json_minifier.h */ // This file contains the common code every implementation uses in stage1 // It is intended to be included multiple times and compiled multiple times @@ -3288,7 +3320,7 @@ really_inline static size_t trim_partial_utf8(const uint8_t *buf, size_t len) { return len; } /* end file src/generic/stage1/find_next_document_index.h */ -/* begin file src/generic/stage1/utf8_lookup2_algorithm.h */ +/* begin file src/generic/stage1/utf8_lookup3_algorithm.h */ // // Detect Unicode errors. // @@ -3380,67 +3412,79 @@ namespace utf8_validation { static const int TOO_LARGE = 0x10; // 11110100 (1001|101_)____ static const int TOO_LARGE_2 = 0x20; // 1111(1___|011_|0101) 10______ + // New with lookup3. We want to catch the case where an non-continuation + // follows a leading byte + static const int TOO_SHORT_2_3_4 = 0x40; // (110_|1110|1111) ____ (0___|110_|1111) ____ + // We also want to catch a continuation that is preceded by an ASCII byte + static const int LONELY_CONTINUATION = 0x80; // 0___ ____ 01__ ____ + // After processing the rest of byte 1 (the low bits), we're still not done--we have to check // byte 2 to be sure which things are errors and which aren't. // Since high_bits is byte 5, byte 2 is high_bits.prev<3> static const int CARRY = OVERLONG_2 | TOO_LARGE_2; const simd8 byte_2_high = input.shr<4>().lookup_16( // ASCII: ________ [0___]____ - CARRY, CARRY, CARRY, CARRY, + CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4, + CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4, // ASCII: ________ [0___]____ - CARRY, CARRY, CARRY, CARRY, + CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4, + CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4, // Continuations: ________ [10__]____ - CARRY | OVERLONG_3 | OVERLONG_4, // ________ [1000]____ - CARRY | OVERLONG_3 | TOO_LARGE, // ________ [1001]____ - CARRY | TOO_LARGE | SURROGATE, // ________ [1010]____ - CARRY | TOO_LARGE | SURROGATE, // ________ [1011]____ + CARRY | OVERLONG_3 | OVERLONG_4 | LONELY_CONTINUATION, // ________ [1000]____ + CARRY | OVERLONG_3 | TOO_LARGE | LONELY_CONTINUATION, // ________ [1001]____ + CARRY | TOO_LARGE | SURROGATE | LONELY_CONTINUATION, // ________ [1010]____ + CARRY | TOO_LARGE | SURROGATE | LONELY_CONTINUATION, // ________ [1011]____ // Multibyte Leads: ________ [11__]____ - CARRY, CARRY, CARRY, CARRY + CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4, // 110_ + CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4 ); - const simd8 byte_1_high = prev1.shr<4>().lookup_16( // [0___]____ (ASCII) - 0, 0, 0, 0, - 0, 0, 0, 0, + LONELY_CONTINUATION, LONELY_CONTINUATION, LONELY_CONTINUATION, LONELY_CONTINUATION, + LONELY_CONTINUATION, LONELY_CONTINUATION, LONELY_CONTINUATION, LONELY_CONTINUATION, // [10__]____ (continuation) 0, 0, 0, 0, // [11__]____ (2+-byte leads) - OVERLONG_2, 0, // [110_]____ (2-byte lead) - OVERLONG_3 | SURROGATE, // [1110]____ (3-byte lead) - OVERLONG_4 | TOO_LARGE | TOO_LARGE_2 // [1111]____ (4+-byte lead) + OVERLONG_2 | TOO_SHORT_2_3_4, TOO_SHORT_2_3_4, // [110_]____ (2-byte lead) + OVERLONG_3 | SURROGATE | TOO_SHORT_2_3_4, // [1110]____ (3-byte lead) + OVERLONG_4 | TOO_LARGE | TOO_LARGE_2 | TOO_SHORT_2_3_4 // [1111]____ (4+-byte lead) ); - const simd8 byte_1_low = (prev1 & 0x0F).lookup_16( // ____[00__] ________ - OVERLONG_2 | OVERLONG_3 | OVERLONG_4, // ____[0000] ________ - OVERLONG_2, // ____[0001] ________ - 0, 0, + OVERLONG_2 | OVERLONG_3 | OVERLONG_4 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[0000] ________ + OVERLONG_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[0001] ________ + TOO_SHORT_2_3_4 | LONELY_CONTINUATION, + TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[01__] ________ - TOO_LARGE, // ____[0100] ________ - TOO_LARGE_2, - TOO_LARGE_2, - TOO_LARGE_2, + TOO_LARGE | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[0100] ________ + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[10__] ________ - TOO_LARGE_2, TOO_LARGE_2, TOO_LARGE_2, TOO_LARGE_2, + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[11__] ________ - TOO_LARGE_2, - TOO_LARGE_2 | SURROGATE, // ____[1101] ________ - TOO_LARGE_2, TOO_LARGE_2 + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, + TOO_LARGE_2 | SURROGATE | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[1101] ________ + TOO_LARGE_2 | TOO_SHORT_2_3_4| LONELY_CONTINUATION, + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION ); - return byte_1_high & byte_1_low & byte_2_high; } - really_inline simd8 check_multibyte_lengths(simd8 input, simd8 prev_input, simd8 prev1) { + really_inline simd8 check_multibyte_lengths(simd8 input, simd8 prev_input, + simd8 prev1) { simd8 prev2 = input.prev<2>(prev_input); simd8 prev3 = input.prev<3>(prev_input); - - // Cont is 10000000-101111111 (-65...-128) - simd8 is_continuation = simd8(input) < int8_t(-64); - // must_be_continuation is architecture-specific because Intel doesn't have unsigned comparisons - return simd8(must_be_continuation(prev1, prev2, prev3) ^ is_continuation); + // is_2_3_continuation uses one more instruction than lookup2 + simd8 is_2_3_continuation = (simd8(input).max(simd8(prev1))) < int8_t(-64); + // must_be_2_3_continuation has two fewer instructions than lookup 2 + return simd8(must_be_2_3_continuation(prev2, prev3) ^ is_2_3_continuation); } + // // Return nonzero if there are incomplete multibyte characters at the end of the block: // e.g. if there is a 4-byte character, but it's 3 bytes from the end. @@ -3507,7 +3551,7 @@ namespace utf8_validation { } using utf8_validation::utf8_checker; -/* end file src/generic/stage1/utf8_lookup2_algorithm.h */ +/* end file src/generic/stage1/utf8_lookup3_algorithm.h */ /* begin file src/generic/stage1/json_structural_indexer.h */ // This file contains the common code every implementation uses in stage1 // It is intended to be included multiple times and compiled multiple times @@ -4432,7 +4476,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const src, } // we over-decrement by one when there is a '.' digit_count -= int(start - start_digits); - if (unlikely(digit_count >= 19)) { + if (digit_count >= 19) { // Ok, chances are good that we had an overflow! // this is almost never going to get called!!! // we start anew, going slowly!!! @@ -4442,7 +4486,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const src, // bool success = slow_float_parsing((const char *) src, writer); // The number was already written, but we made a copy of the writer - // when we passed it to the parse_large_integer() function, so + // when we passed it to the parse_large_integer() function, so writer.skip_double(); return success; } @@ -4481,7 +4525,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const src, // need to recover: we parse the whole thing again. bool success = parse_large_integer(src, writer, found_minus); // The number was already written, but we made a copy of the writer - // when we passed it to the parse_large_integer() function, so + // when we passed it to the parse_large_integer() function, so writer.skip_large_integer(); return success; } @@ -6525,7 +6569,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const src, } // we over-decrement by one when there is a '.' digit_count -= int(start - start_digits); - if (unlikely(digit_count >= 19)) { + if (digit_count >= 19) { // Ok, chances are good that we had an overflow! // this is almost never going to get called!!! // we start anew, going slowly!!! @@ -6535,7 +6579,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const src, // bool success = slow_float_parsing((const char *) src, writer); // The number was already written, but we made a copy of the writer - // when we passed it to the parse_large_integer() function, so + // when we passed it to the parse_large_integer() function, so writer.skip_double(); return success; } @@ -6574,7 +6618,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const src, // need to recover: we parse the whole thing again. bool success = parse_large_integer(src, writer, found_minus); // The number was already written, but we made a copy of the writer - // when we passed it to the parse_large_integer() function, so + // when we passed it to the parse_large_integer() function, so writer.skip_large_integer(); return success; } @@ -8119,6 +8163,14 @@ really_inline simd8 must_be_continuation(simd8 prev1, simd8(is_second_byte | is_third_byte | is_fourth_byte) > int8_t(0); } +really_inline simd8 must_be_2_3_continuation(simd8 prev2, simd8 prev3) { + simd8 is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0 + simd8 is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0 + // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine. + return simd8(is_third_byte | is_fourth_byte) > int8_t(0); +} + + /* begin file src/generic/stage1/buf_block_reader.h */ // Walks through a buffer in block-sized increments, loading the last part with spaces template @@ -8246,7 +8298,9 @@ public: really_inline error_code finish(bool streaming); private: + // Intended to be defined by the implementation really_inline uint64_t find_escaped(uint64_t escape); + really_inline uint64_t find_escaped_branchless(uint64_t escape); // Whether the last iteration was still inside a string (all 1's = true, all 0's = false). uint64_t prev_in_string = 0ULL; @@ -8281,7 +8335,7 @@ private: // desired | x | x x x x x x x x | // text | \\\ | \\\"\\\" \\\" \\"\\" | // -really_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { +really_inline uint64_t json_string_scanner::find_escaped_branchless(uint64_t backslash) { // If there was overflow, pretend the first character isn't a backslash backslash &= ~prev_escaped; uint64_t follows_escape = backslash << 1 | prev_escaped; @@ -8310,13 +8364,23 @@ really_inline json_string_block json_string_scanner::next(const simd::simd8x64(in_string) >> 63); + // Use ^ to turn the beginning quote off, and the end quote on. return { backslash, @@ -8442,6 +8506,13 @@ really_inline error_code json_scanner::finish(bool streaming) { } // namespace stage1 /* end file src/generic/stage1/json_scanner.h */ +namespace stage1 { +really_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { + if (!backslash) { uint64_t escaped = prev_escaped; prev_escaped = 0; return escaped; } + return find_escaped_branchless(backslash); +} +} + /* begin file src/generic/stage1/json_minifier.h */ // This file contains the common code every implementation uses in stage1 // It is intended to be included multiple times and compiled multiple times @@ -8613,7 +8684,7 @@ really_inline static size_t trim_partial_utf8(const uint8_t *buf, size_t len) { return len; } /* end file src/generic/stage1/find_next_document_index.h */ -/* begin file src/generic/stage1/utf8_lookup2_algorithm.h */ +/* begin file src/generic/stage1/utf8_lookup3_algorithm.h */ // // Detect Unicode errors. // @@ -8705,67 +8776,79 @@ namespace utf8_validation { static const int TOO_LARGE = 0x10; // 11110100 (1001|101_)____ static const int TOO_LARGE_2 = 0x20; // 1111(1___|011_|0101) 10______ + // New with lookup3. We want to catch the case where an non-continuation + // follows a leading byte + static const int TOO_SHORT_2_3_4 = 0x40; // (110_|1110|1111) ____ (0___|110_|1111) ____ + // We also want to catch a continuation that is preceded by an ASCII byte + static const int LONELY_CONTINUATION = 0x80; // 0___ ____ 01__ ____ + // After processing the rest of byte 1 (the low bits), we're still not done--we have to check // byte 2 to be sure which things are errors and which aren't. // Since high_bits is byte 5, byte 2 is high_bits.prev<3> static const int CARRY = OVERLONG_2 | TOO_LARGE_2; const simd8 byte_2_high = input.shr<4>().lookup_16( // ASCII: ________ [0___]____ - CARRY, CARRY, CARRY, CARRY, + CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4, + CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4, // ASCII: ________ [0___]____ - CARRY, CARRY, CARRY, CARRY, + CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4, + CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4, // Continuations: ________ [10__]____ - CARRY | OVERLONG_3 | OVERLONG_4, // ________ [1000]____ - CARRY | OVERLONG_3 | TOO_LARGE, // ________ [1001]____ - CARRY | TOO_LARGE | SURROGATE, // ________ [1010]____ - CARRY | TOO_LARGE | SURROGATE, // ________ [1011]____ + CARRY | OVERLONG_3 | OVERLONG_4 | LONELY_CONTINUATION, // ________ [1000]____ + CARRY | OVERLONG_3 | TOO_LARGE | LONELY_CONTINUATION, // ________ [1001]____ + CARRY | TOO_LARGE | SURROGATE | LONELY_CONTINUATION, // ________ [1010]____ + CARRY | TOO_LARGE | SURROGATE | LONELY_CONTINUATION, // ________ [1011]____ // Multibyte Leads: ________ [11__]____ - CARRY, CARRY, CARRY, CARRY + CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4, // 110_ + CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4 ); - const simd8 byte_1_high = prev1.shr<4>().lookup_16( // [0___]____ (ASCII) - 0, 0, 0, 0, - 0, 0, 0, 0, + LONELY_CONTINUATION, LONELY_CONTINUATION, LONELY_CONTINUATION, LONELY_CONTINUATION, + LONELY_CONTINUATION, LONELY_CONTINUATION, LONELY_CONTINUATION, LONELY_CONTINUATION, // [10__]____ (continuation) 0, 0, 0, 0, // [11__]____ (2+-byte leads) - OVERLONG_2, 0, // [110_]____ (2-byte lead) - OVERLONG_3 | SURROGATE, // [1110]____ (3-byte lead) - OVERLONG_4 | TOO_LARGE | TOO_LARGE_2 // [1111]____ (4+-byte lead) + OVERLONG_2 | TOO_SHORT_2_3_4, TOO_SHORT_2_3_4, // [110_]____ (2-byte lead) + OVERLONG_3 | SURROGATE | TOO_SHORT_2_3_4, // [1110]____ (3-byte lead) + OVERLONG_4 | TOO_LARGE | TOO_LARGE_2 | TOO_SHORT_2_3_4 // [1111]____ (4+-byte lead) ); - const simd8 byte_1_low = (prev1 & 0x0F).lookup_16( // ____[00__] ________ - OVERLONG_2 | OVERLONG_3 | OVERLONG_4, // ____[0000] ________ - OVERLONG_2, // ____[0001] ________ - 0, 0, + OVERLONG_2 | OVERLONG_3 | OVERLONG_4 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[0000] ________ + OVERLONG_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[0001] ________ + TOO_SHORT_2_3_4 | LONELY_CONTINUATION, + TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[01__] ________ - TOO_LARGE, // ____[0100] ________ - TOO_LARGE_2, - TOO_LARGE_2, - TOO_LARGE_2, + TOO_LARGE | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[0100] ________ + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[10__] ________ - TOO_LARGE_2, TOO_LARGE_2, TOO_LARGE_2, TOO_LARGE_2, + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[11__] ________ - TOO_LARGE_2, - TOO_LARGE_2 | SURROGATE, // ____[1101] ________ - TOO_LARGE_2, TOO_LARGE_2 + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, + TOO_LARGE_2 | SURROGATE | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[1101] ________ + TOO_LARGE_2 | TOO_SHORT_2_3_4| LONELY_CONTINUATION, + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION ); - return byte_1_high & byte_1_low & byte_2_high; } - really_inline simd8 check_multibyte_lengths(simd8 input, simd8 prev_input, simd8 prev1) { + really_inline simd8 check_multibyte_lengths(simd8 input, simd8 prev_input, + simd8 prev1) { simd8 prev2 = input.prev<2>(prev_input); simd8 prev3 = input.prev<3>(prev_input); - - // Cont is 10000000-101111111 (-65...-128) - simd8 is_continuation = simd8(input) < int8_t(-64); - // must_be_continuation is architecture-specific because Intel doesn't have unsigned comparisons - return simd8(must_be_continuation(prev1, prev2, prev3) ^ is_continuation); + // is_2_3_continuation uses one more instruction than lookup2 + simd8 is_2_3_continuation = (simd8(input).max(simd8(prev1))) < int8_t(-64); + // must_be_2_3_continuation has two fewer instructions than lookup 2 + return simd8(must_be_2_3_continuation(prev2, prev3) ^ is_2_3_continuation); } + // // Return nonzero if there are incomplete multibyte characters at the end of the block: // e.g. if there is a 4-byte character, but it's 3 bytes from the end. @@ -8832,7 +8915,7 @@ namespace utf8_validation { } using utf8_validation::utf8_checker; -/* end file src/generic/stage1/utf8_lookup2_algorithm.h */ +/* end file src/generic/stage1/utf8_lookup3_algorithm.h */ /* begin file src/generic/stage1/json_structural_indexer.h */ // This file contains the common code every implementation uses in stage1 // It is intended to be included multiple times and compiled multiple times @@ -9762,7 +9845,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const src, } // we over-decrement by one when there is a '.' digit_count -= int(start - start_digits); - if (unlikely(digit_count >= 19)) { + if (digit_count >= 19) { // Ok, chances are good that we had an overflow! // this is almost never going to get called!!! // we start anew, going slowly!!! @@ -9772,7 +9855,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const src, // bool success = slow_float_parsing((const char *) src, writer); // The number was already written, but we made a copy of the writer - // when we passed it to the parse_large_integer() function, so + // when we passed it to the parse_large_integer() function, so writer.skip_double(); return success; } @@ -9811,7 +9894,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const src, // need to recover: we parse the whole thing again. bool success = parse_large_integer(src, writer, found_minus); // The number was already written, but we made a copy of the writer - // when we passed it to the parse_large_integer() function, so + // when we passed it to the parse_large_integer() function, so writer.skip_large_integer(); return success; } @@ -11327,6 +11410,14 @@ really_inline simd8 must_be_continuation(simd8 prev1, simd8(is_second_byte | is_third_byte | is_fourth_byte) > int8_t(0); } +really_inline simd8 must_be_2_3_continuation(simd8 prev2, simd8 prev3) { + simd8 is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0 + simd8 is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0 + // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine. + return simd8(is_third_byte | is_fourth_byte) > int8_t(0); +} + + /* begin file src/generic/stage1/buf_block_reader.h */ // Walks through a buffer in block-sized increments, loading the last part with spaces template @@ -11454,7 +11545,9 @@ public: really_inline error_code finish(bool streaming); private: + // Intended to be defined by the implementation really_inline uint64_t find_escaped(uint64_t escape); + really_inline uint64_t find_escaped_branchless(uint64_t escape); // Whether the last iteration was still inside a string (all 1's = true, all 0's = false). uint64_t prev_in_string = 0ULL; @@ -11489,7 +11582,7 @@ private: // desired | x | x x x x x x x x | // text | \\\ | \\\"\\\" \\\" \\"\\" | // -really_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { +really_inline uint64_t json_string_scanner::find_escaped_branchless(uint64_t backslash) { // If there was overflow, pretend the first character isn't a backslash backslash &= ~prev_escaped; uint64_t follows_escape = backslash << 1 | prev_escaped; @@ -11518,13 +11611,23 @@ really_inline json_string_block json_string_scanner::next(const simd::simd8x64(in_string) >> 63); + // Use ^ to turn the beginning quote off, and the end quote on. return { backslash, @@ -11650,6 +11753,13 @@ really_inline error_code json_scanner::finish(bool streaming) { } // namespace stage1 /* end file src/generic/stage1/json_scanner.h */ +namespace stage1 { +really_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { + if (!backslash) { uint64_t escaped = prev_escaped; prev_escaped = 0; return escaped; } + return find_escaped_branchless(backslash); +} +} + /* begin file src/generic/stage1/json_minifier.h */ // This file contains the common code every implementation uses in stage1 // It is intended to be included multiple times and compiled multiple times @@ -11821,7 +11931,7 @@ really_inline static size_t trim_partial_utf8(const uint8_t *buf, size_t len) { return len; } /* end file src/generic/stage1/find_next_document_index.h */ -/* begin file src/generic/stage1/utf8_lookup2_algorithm.h */ +/* begin file src/generic/stage1/utf8_lookup3_algorithm.h */ // // Detect Unicode errors. // @@ -11913,67 +12023,79 @@ namespace utf8_validation { static const int TOO_LARGE = 0x10; // 11110100 (1001|101_)____ static const int TOO_LARGE_2 = 0x20; // 1111(1___|011_|0101) 10______ + // New with lookup3. We want to catch the case where an non-continuation + // follows a leading byte + static const int TOO_SHORT_2_3_4 = 0x40; // (110_|1110|1111) ____ (0___|110_|1111) ____ + // We also want to catch a continuation that is preceded by an ASCII byte + static const int LONELY_CONTINUATION = 0x80; // 0___ ____ 01__ ____ + // After processing the rest of byte 1 (the low bits), we're still not done--we have to check // byte 2 to be sure which things are errors and which aren't. // Since high_bits is byte 5, byte 2 is high_bits.prev<3> static const int CARRY = OVERLONG_2 | TOO_LARGE_2; const simd8 byte_2_high = input.shr<4>().lookup_16( // ASCII: ________ [0___]____ - CARRY, CARRY, CARRY, CARRY, + CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4, + CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4, // ASCII: ________ [0___]____ - CARRY, CARRY, CARRY, CARRY, + CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4, + CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4, // Continuations: ________ [10__]____ - CARRY | OVERLONG_3 | OVERLONG_4, // ________ [1000]____ - CARRY | OVERLONG_3 | TOO_LARGE, // ________ [1001]____ - CARRY | TOO_LARGE | SURROGATE, // ________ [1010]____ - CARRY | TOO_LARGE | SURROGATE, // ________ [1011]____ + CARRY | OVERLONG_3 | OVERLONG_4 | LONELY_CONTINUATION, // ________ [1000]____ + CARRY | OVERLONG_3 | TOO_LARGE | LONELY_CONTINUATION, // ________ [1001]____ + CARRY | TOO_LARGE | SURROGATE | LONELY_CONTINUATION, // ________ [1010]____ + CARRY | TOO_LARGE | SURROGATE | LONELY_CONTINUATION, // ________ [1011]____ // Multibyte Leads: ________ [11__]____ - CARRY, CARRY, CARRY, CARRY + CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4, // 110_ + CARRY | TOO_SHORT_2_3_4, CARRY | TOO_SHORT_2_3_4 ); - const simd8 byte_1_high = prev1.shr<4>().lookup_16( // [0___]____ (ASCII) - 0, 0, 0, 0, - 0, 0, 0, 0, + LONELY_CONTINUATION, LONELY_CONTINUATION, LONELY_CONTINUATION, LONELY_CONTINUATION, + LONELY_CONTINUATION, LONELY_CONTINUATION, LONELY_CONTINUATION, LONELY_CONTINUATION, // [10__]____ (continuation) 0, 0, 0, 0, // [11__]____ (2+-byte leads) - OVERLONG_2, 0, // [110_]____ (2-byte lead) - OVERLONG_3 | SURROGATE, // [1110]____ (3-byte lead) - OVERLONG_4 | TOO_LARGE | TOO_LARGE_2 // [1111]____ (4+-byte lead) + OVERLONG_2 | TOO_SHORT_2_3_4, TOO_SHORT_2_3_4, // [110_]____ (2-byte lead) + OVERLONG_3 | SURROGATE | TOO_SHORT_2_3_4, // [1110]____ (3-byte lead) + OVERLONG_4 | TOO_LARGE | TOO_LARGE_2 | TOO_SHORT_2_3_4 // [1111]____ (4+-byte lead) ); - const simd8 byte_1_low = (prev1 & 0x0F).lookup_16( // ____[00__] ________ - OVERLONG_2 | OVERLONG_3 | OVERLONG_4, // ____[0000] ________ - OVERLONG_2, // ____[0001] ________ - 0, 0, + OVERLONG_2 | OVERLONG_3 | OVERLONG_4 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[0000] ________ + OVERLONG_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[0001] ________ + TOO_SHORT_2_3_4 | LONELY_CONTINUATION, + TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[01__] ________ - TOO_LARGE, // ____[0100] ________ - TOO_LARGE_2, - TOO_LARGE_2, - TOO_LARGE_2, + TOO_LARGE | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[0100] ________ + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[10__] ________ - TOO_LARGE_2, TOO_LARGE_2, TOO_LARGE_2, TOO_LARGE_2, + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[11__] ________ - TOO_LARGE_2, - TOO_LARGE_2 | SURROGATE, // ____[1101] ________ - TOO_LARGE_2, TOO_LARGE_2 + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, + TOO_LARGE_2 | SURROGATE | TOO_SHORT_2_3_4 | LONELY_CONTINUATION, // ____[1101] ________ + TOO_LARGE_2 | TOO_SHORT_2_3_4| LONELY_CONTINUATION, + TOO_LARGE_2 | TOO_SHORT_2_3_4 | LONELY_CONTINUATION ); - return byte_1_high & byte_1_low & byte_2_high; } - really_inline simd8 check_multibyte_lengths(simd8 input, simd8 prev_input, simd8 prev1) { + really_inline simd8 check_multibyte_lengths(simd8 input, simd8 prev_input, + simd8 prev1) { simd8 prev2 = input.prev<2>(prev_input); simd8 prev3 = input.prev<3>(prev_input); - - // Cont is 10000000-101111111 (-65...-128) - simd8 is_continuation = simd8(input) < int8_t(-64); - // must_be_continuation is architecture-specific because Intel doesn't have unsigned comparisons - return simd8(must_be_continuation(prev1, prev2, prev3) ^ is_continuation); + // is_2_3_continuation uses one more instruction than lookup2 + simd8 is_2_3_continuation = (simd8(input).max(simd8(prev1))) < int8_t(-64); + // must_be_2_3_continuation has two fewer instructions than lookup 2 + return simd8(must_be_2_3_continuation(prev2, prev3) ^ is_2_3_continuation); } + // // Return nonzero if there are incomplete multibyte characters at the end of the block: // e.g. if there is a 4-byte character, but it's 3 bytes from the end. @@ -12040,7 +12162,7 @@ namespace utf8_validation { } using utf8_validation::utf8_checker; -/* end file src/generic/stage1/utf8_lookup2_algorithm.h */ +/* end file src/generic/stage1/utf8_lookup3_algorithm.h */ /* begin file src/generic/stage1/json_structural_indexer.h */ // This file contains the common code every implementation uses in stage1 // It is intended to be included multiple times and compiled multiple times @@ -12973,7 +13095,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const src, } // we over-decrement by one when there is a '.' digit_count -= int(start - start_digits); - if (unlikely(digit_count >= 19)) { + if (digit_count >= 19) { // Ok, chances are good that we had an overflow! // this is almost never going to get called!!! // we start anew, going slowly!!! @@ -12983,7 +13105,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const src, // bool success = slow_float_parsing((const char *) src, writer); // The number was already written, but we made a copy of the writer - // when we passed it to the parse_large_integer() function, so + // when we passed it to the parse_large_integer() function, so writer.skip_double(); return success; } @@ -13022,7 +13144,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const src, // need to recover: we parse the whole thing again. bool success = parse_large_integer(src, writer, found_minus); // The number was already written, but we made a copy of the writer - // when we passed it to the parse_large_integer() function, so + // when we passed it to the parse_large_integer() function, so writer.skip_large_integer(); return success; } diff --git a/singleheader/simdjson.h b/singleheader/simdjson.h index 21efa8e49..0efec0167 100644 --- a/singleheader/simdjson.h +++ b/singleheader/simdjson.h @@ -1,4 +1,4 @@ -/* auto-generated on Fri 12 Jun 2020 13:09:36 EDT. Do not edit! */ +/* auto-generated on Sat Jun 20 21:50:03 PDT 2020. Do not edit! */ /* begin file include/simdjson.h */ #ifndef SIMDJSON_H #define SIMDJSON_H @@ -169,12 +169,12 @@ compiling for a known 64-bit platform." #define TARGET_WESTMERE TARGET_REGION("sse4.2,pclmul") #define TARGET_ARM64 -// Threading is disabled -#undef SIMDJSON_THREADS_ENABLED // Is threading enabled? #if defined(BOOST_HAS_THREADS) || defined(_REENTRANT) || defined(_MT) +#ifndef SIMDJSON_THREADS_ENABLED #define SIMDJSON_THREADS_ENABLED #endif +#endif // workaround for large stack sizes under -O0. @@ -183,7 +183,9 @@ compiling for a known 64-bit platform." #ifndef __OPTIMIZE__ // Apple systems have small stack sizes in secondary threads. // Lack of compiler optimization may generate high stack usage. -// So we are disabling multithreaded support for safety. +// Users may want to disable threads for safety, but only when +// in debug mode which we detect by the fact that the __OPTIMIZE__ +// macro is not defined. #undef SIMDJSON_THREADS_ENABLED #endif #endif @@ -251,6 +253,25 @@ static inline void aligned_free(void *mem_block) { static inline void aligned_free_char(char *mem_block) { aligned_free((void *)mem_block); } + +#ifdef NDEBUG + +#ifdef SIMDJSON_VISUAL_STUDIO +#define SIMDJSON_UNREACHABLE() __assume(0) +#define SIMDJSON_ASSUME(COND) __assume(COND) +#else +#define SIMDJSON_UNREACHABLE() __builtin_unreachable(); +#define SIMDJSON_ASSUME(COND) do { if (!(COND)) __builtin_unreachable(); } while (0) +#endif + +#else // NDEBUG + +#include +#define SIMDJSON_UNREACHABLE() assert(0); +#define SIMDJSON_ASSUME(COND) assert(COND) + +#endif + } // namespace simdjson #endif // SIMDJSON_PORTABILITY_H /* end file include/simdjson/portability.h */ @@ -2138,9 +2159,19 @@ struct simdjson_result_base : public std::pair { /** * Move the value and the error to the provided variables. + * + * @param value The variable to assign the value to. May not be set if there is an error. + * @param error The variable to assign the error to. Set to SUCCESS if there is no error. */ really_inline void tie(T &value, error_code &error) && noexcept; + /** + * Move the value to the provided variable. + * + * @param value The variable to assign the value to. May not be set if there is an error. + */ + really_inline error_code get(T &value) && noexcept; + /** * The error. */ @@ -2200,8 +2231,18 @@ struct simdjson_result : public internal::simdjson_result_base { /** * Move the value and the error to the provided variables. + * + * @param value The variable to assign the value to. May not be set if there is an error. + * @param error The variable to assign the error to. Set to SUCCESS if there is no error. */ - really_inline void tie(T& t, error_code & e) && noexcept; + really_inline void tie(T &value, error_code &error) && noexcept; + + /** + * Move the value to the provided variable. + * + * @param value The variable to assign the value to. May not be set if there is an error. + */ + WARN_UNUSED really_inline error_code get(T &value) && noexcept; /** * The error. @@ -2658,11 +2699,11 @@ public: /** * @private For internal implementation use * - * Run a full document parse (ensure_capacity, stage1 and stage2). + * Minify the input string assuming that it represents a JSON string, does not parse or validate. * * Overridden by each implementation. * - * @param buf the json document to parse. *MUST* be allocated up to len + SIMDJSON_PADDING bytes. + * @param buf the json document to minify. * @param len the length of the json document. * @param dst the buffer to write the minified document to. *MUST* be allocated up to len + SIMDJSON_PADDING bytes. * @param dst_len the number of bytes written. Output only. @@ -2884,6 +2925,23 @@ public: namespace simdjson { + + +/** + * + * Minify the input string assuming that it represents a JSON string, does not parse or validate. + * This function is much faster than parsing a JSON string and then writing a minified version of it. + * However, it does not validate the input. + * + * + * @param buf the json document to minify. + * @param len the length of the json document. + * @param dst the buffer to write the minified document to. *MUST* be allocated up to len + SIMDJSON_PADDING bytes. + * @param dst_len the number of bytes written. Output only. + * @return the error code, or SUCCESS if there was no error. + */ +WARN_UNUSED error_code minify(const char *buf, size_t len, char *dst, size_t &dst_len) noexcept; + /** * Minifies a JSON element or document, printing the smallest possible valid JSON. * @@ -2893,14 +2951,14 @@ namespace simdjson { * */ template -class minify { +class minifier { public: /** * Create a new minifier. * * @param _value The document or element to minify. */ - inline minify(const T &_value) noexcept : value{_value} {} + inline minifier(const T &_value) noexcept : value{_value} {} /** * Minify JSON to a string. @@ -2915,6 +2973,9 @@ private: const T &value; }; +template +inline minifier minify(const T &value) noexcept { return minifier(value); } + /** * Minify JSON to an output stream. * @@ -2923,7 +2984,7 @@ private: * @throw if there is an error with the underlying output stream. simdjson itself will not throw. */ template -inline std::ostream& operator<<(std::ostream& out, minify formatter) { return formatter.print(out); } +inline std::ostream& operator<<(std::ostream& out, minifier formatter) { return formatter.print(out); } } // namespace simdjson @@ -2940,12 +3001,12 @@ class element; /** * JSON array. */ -class array : protected internal::tape_ref { +class array { public: /** Create a new, invalid array */ really_inline array() noexcept; - class iterator : protected internal::tape_ref { + class iterator { public: /** * Get the actual value @@ -2965,7 +3026,8 @@ public: */ inline bool operator!=(const iterator& other) const noexcept; private: - really_inline iterator(const document *doc, size_t json_index) noexcept; + really_inline iterator(const internal::tape_ref &tape) noexcept; + internal::tape_ref tape; friend class array; }; @@ -3004,19 +3066,30 @@ public: inline simdjson_result at(const std::string_view &json_pointer) const noexcept; /** - * Get the value at the given index. + * Get the value at the given index. This function has linear-time complexity and + * is equivalent to the following: + * + * size_t i=0; + * for (auto element : *this) { + * if (i == index) { return element; } + * i++; + * } + * return INDEX_OUT_OF_BOUNDS; * + * Avoid calling the at() function repeatedly. + * * @return The value at the given index, or: * - INDEX_OUT_OF_BOUNDS if the array index is larger than an array length */ inline simdjson_result at(size_t index) const noexcept; private: - really_inline array(const document *doc, size_t json_index) noexcept; + really_inline array(const internal::tape_ref &tape) noexcept; + internal::tape_ref tape; friend class element; friend struct simdjson_result; template - friend class simdjson::minify; + friend class simdjson::minifier; }; /** @@ -3146,7 +3219,7 @@ public: private: inline error_code allocate(size_t len) noexcept; template - friend class simdjson::minify; + friend class simdjson::minifier; friend class parser; }; // class document @@ -3305,6 +3378,10 @@ public: * documents that consist of an object or array may omit the whitespace between them, concatenating * with no separator. documents that consist of a single primitive (i.e. documents that are not * arrays or objects) MUST be separated with whitespace. + * + * The documents must not exceed batch_size bytes (by default 1MB) or they will fail to parse. + * Setting batch_size to excessively large or excesively small values may impact negatively the + * performance. * * ### Error Handling * @@ -3335,20 +3412,19 @@ public: * spot is cache-related: small enough to fit in cache, yet big enough to * parse as many documents as possible in one tight loop. * Defaults to 10MB, which has been a reasonable sweet spot in our tests. - * @return The stream. If there is an error, it will be returned during iteration. An empty input - * will yield 0 documents rather than an EMPTY error. Errors: + * @return The stream, or an error. An empty input will yield 0 documents rather than an EMPTY error. Errors: * - IO_ERROR if there was an error opening or reading the file. * - MEMALLOC if the parser does not have enough capacity and memory allocation fails. * - CAPACITY if the parser does not have enough capacity and batch_size > max_capacity. * - other json errors if parsing fails. */ - inline document_stream load_many(const std::string &path, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept; + inline simdjson_result load_many(const std::string &path, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept; /** * Parse a buffer containing many JSON documents. * * dom::parser parser; - * for (const element doc : parser.parse_many(buf, len)) { + * for (element doc : parser.parse_many(buf, len)) { * cout << std::string(doc["title"]) << endl; * } * @@ -3362,6 +3438,10 @@ public: * documents that consist of an object or array may omit the whitespace between them, concatenating * with no separator. documents that consist of a single primitive (i.e. documents that are not * arrays or objects) MUST be separated with whitespace. + * + * The documents must not exceed batch_size bytes (by default 1MB) or they will fail to parse. + * Setting batch_size to excessively large or excesively small values may impact negatively the + * performance. * * ### Error Handling * @@ -3398,22 +3478,21 @@ public: * spot is cache-related: small enough to fit in cache, yet big enough to * parse as many documents as possible in one tight loop. * Defaults to 10MB, which has been a reasonable sweet spot in our tests. - * @return The stream. If there is an error, it will be returned during iteration. An empty input - * will yield 0 documents rather than an EMPTY error. Errors: + * @return The stream, or an error. An empty input will yield 0 documents rather than an EMPTY error. Errors: * - MEMALLOC if the parser does not have enough capacity and memory allocation fails * - CAPACITY if the parser does not have enough capacity and batch_size > max_capacity. * - other json errors if parsing fails. */ - inline document_stream parse_many(const uint8_t *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept; + inline simdjson_result parse_many(const uint8_t *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept; /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ - inline document_stream parse_many(const char *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept; + inline simdjson_result parse_many(const char *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept; /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ - inline document_stream parse_many(const std::string &s, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept; + inline simdjson_result parse_many(const std::string &s, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept; /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ - inline document_stream parse_many(const padded_string &s, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept; + inline simdjson_result parse_many(const padded_string &s, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept; /** @private We do not want to allow implicit conversion from C string to std::string. */ - really_inline simdjson_result parse_many(const char *buf, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept = delete; + simdjson_result parse_many(const char *buf, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept = delete; /** * Ensure this parser has enough memory to process JSON documents up to `capacity` bytes in length @@ -3562,11 +3641,64 @@ private: /* end file include/simdjson/dom/document.h */ #ifdef SIMDJSON_THREADS_ENABLED #include +#include +#include #endif namespace simdjson { namespace dom { + +#ifdef SIMDJSON_THREADS_ENABLED +/** @private Custom worker class **/ +struct stage1_worker { + stage1_worker() noexcept = default; + stage1_worker(const stage1_worker&) = delete; + stage1_worker(stage1_worker&&) = delete; + stage1_worker operator=(const stage1_worker&) = delete; + ~stage1_worker(); + /** + * We only start the thread when it is needed, not at object construction, this may throw. + * You should only call this once. + **/ + void start_thread(); + /** + * Start a stage 1 job. You should first call 'run', then 'finish'. + * You must call start_thread once before. + */ + void run(document_stream * ds, dom::parser * stage1, size_t next_batch_start); + /** Wait for the run to finish (blocking). You should first call 'run', then 'finish'. **/ + void finish(); + +private: + + /** + * Normally, we would never stop the thread. But we do in the destructor. + * This function is only safe assuming that you are not waiting for results. You + * should have called run, then finish, and be done. + **/ + void stop_thread(); + + std::thread thread{}; + /** These three variables define the work done by the thread. **/ + dom::parser * stage1_thread_parser{}; + size_t _next_batch_start{}; + document_stream * owner{}; + /** + * We have two state variables. This could be streamlined to one variable in the future but + * we use two for clarity. + */ + bool has_work{false}; + bool can_work{true}; + + /** + * We lock using a mutex. + */ + std::mutex locking_mutex{}; + std::condition_variable cond_var{}; +}; +#endif + /** * A forward-only stream of documents. * @@ -3575,8 +3707,20 @@ namespace dom { */ class document_stream { public: + /** + * Construct an uninitialized document_stream. + * + * ```c++ + * document_stream docs; + * error = parser.parse_many(json).get(docs); + * ``` + */ + really_inline document_stream() noexcept; /** Move one document_stream to another. */ - really_inline document_stream(document_stream && other) noexcept = default; + really_inline document_stream(document_stream &&other) noexcept = default; + /** Move one document_stream to another. */ + really_inline document_stream &operator=(document_stream &&other) noexcept = default; + really_inline ~document_stream() noexcept; /** @@ -3597,7 +3741,22 @@ public: * @param other the end iterator to compare to. */ really_inline bool operator!=(const iterator &other) const noexcept; - + /** + * @private + * + * Gives the current index in the input document in bytes. + * + * document_stream stream = parser.parse_many(json,window); + * for(auto i = stream.begin(); i != stream.end(); ++i) { + * auto doc = *i; + * size_t index = i.current_index(); + * } + * + * This function (current_index()) is experimental and the usage + * may change in future versions of simdjson: we find the API somewhat + * awkward and we would like to offer something friendlier. + */ + really_inline size_t current_index() noexcept; private: really_inline iterator(document_stream &s, bool finished) noexcept; /** The document_stream we're iterating through. */ @@ -3620,7 +3779,7 @@ private: document_stream &operator=(const document_stream &) = delete; // Disallow copying - document_stream(document_stream &other) = delete; // Disallow copying + document_stream(document_stream &other) = delete; // Disallow copying /** * Construct a document_stream. Does not allocate or parse anything until the iterator is @@ -3630,8 +3789,7 @@ private: dom::parser &parser, const uint8_t *buf, size_t len, - size_t batch_size, - error_code error = SUCCESS + size_t batch_size ) noexcept; /** @@ -3678,13 +3836,14 @@ private: /** Pass the next batch through stage 1 with the given parser. */ inline error_code run_stage1(dom::parser &p, size_t batch_start) noexcept; - dom::parser &parser; + dom::parser *parser; const uint8_t *buf; - const size_t len; - const size_t batch_size; - size_t batch_start{0}; + size_t len; + size_t batch_size; /** The error (or lack thereof) from the current document. */ error_code error; + size_t batch_start{0}; + size_t doc_index{}; #ifdef SIMDJSON_THREADS_ENABLED inline void load_from_stage1_thread() noexcept; @@ -3698,8 +3857,8 @@ private: /** The error returned from the stage 1 thread. */ error_code stage1_thread_error{UNINITIALIZED}; /** The thread used to run stage 1 against the next batch in the background. */ - std::thread stage1_thread{}; - + friend struct stage1_worker; + std::unique_ptr worker{new(std::nothrow) stage1_worker()}; /** * The parser used to run stage 1 in the background. Will be swapped * with the regular parser when finished. @@ -3708,9 +3867,31 @@ private: #endif // SIMDJSON_THREADS_ENABLED friend class dom::parser; + friend class simdjson_result; + friend class internal::simdjson_result_base; + }; // class document_stream } // namespace dom + +template<> +struct simdjson_result : public internal::simdjson_result_base { +public: + really_inline simdjson_result() noexcept; ///< @private + really_inline simdjson_result(error_code error) noexcept; ///< @private + really_inline simdjson_result(dom::document_stream &&value) noexcept; ///< @private + +#if SIMDJSON_EXCEPTIONS + really_inline dom::document_stream::iterator begin() noexcept(false); + really_inline dom::document_stream::iterator end() noexcept(false); +#else // SIMDJSON_EXCEPTIONS + [[deprecated("parse_many() and load_many() may return errors. Use document_stream stream; error = parser.parse_many().get(doc); instead.")]] + really_inline dom::document_stream::iterator begin() noexcept; + [[deprecated("parse_many() and load_many() may return errors. Use document_stream stream; error = parser.parse_many().get(doc); instead.")]] + really_inline dom::document_stream::iterator end() noexcept; +#endif // SIMDJSON_EXCEPTIONS +}; // struct simdjson_result + } // namespace simdjson #endif // SIMDJSON_DOCUMENT_STREAM_H @@ -3749,7 +3930,7 @@ enum class element_type { * References an element in a JSON document, representing a JSON null, boolean, string, number, * array or object. */ -class element : protected internal::tape_ref { +class element { public: /** Create a new, invalid element. */ really_inline element() noexcept; @@ -3757,8 +3938,135 @@ public: /** The type of this element. */ really_inline element_type type() const noexcept; - /** Whether this element is a json `null`. */ - really_inline bool is_null() const noexcept; + /** + * Cast this element to an array. + * + * Equivalent to get(). + * + * @returns An object that can be used to iterate the array, or: + * INCORRECT_TYPE if the JSON element is not an array. + */ + inline simdjson_result get_array() const noexcept; + /** + * Cast this element to an object. + * + * Equivalent to get(). + * + * @returns An object that can be used to look up or iterate the object's fields, or: + * INCORRECT_TYPE if the JSON element is not an object. + */ + inline simdjson_result get_object() const noexcept; + /** + * Cast this element to a string. + * + * Equivalent to get(). + * + * @returns An pointer to a null-terminated string. This string is stored in the parser and will + * be invalidated the next time it parses a document or when it is destroyed. + * Returns INCORRECT_TYPE if the JSON element is not a string. + */ + inline simdjson_result get_c_str() const noexcept; + /** + * Cast this element to a string. + * + * Equivalent to get(). + * + * @returns A string. The string is stored in the parser and will be invalidated the next time it + * parses a document or when it is destroyed. + * Returns INCORRECT_TYPE if the JSON element is not a string. + */ + inline simdjson_result get_string() const noexcept; + /** + * Cast this element to a signed integer. + * + * Equivalent to get(). + * + * @returns A signed 64-bit integer. + * Returns INCORRECT_TYPE if the JSON element is not an integer, or NUMBER_OUT_OF_RANGE + * if it is negative. + */ + inline simdjson_result get_int64_t() const noexcept; + /** + * Cast this element to an unsigned integer. + * + * Equivalent to get(). + * + * @returns An unsigned 64-bit integer. + * Returns INCORRECT_TYPE if the JSON element is not an integer, or NUMBER_OUT_OF_RANGE + * if it is too large. + */ + inline simdjson_result get_uint64_t() const noexcept; + /** + * Cast this element to an double floating-point. + * + * Equivalent to get(). + * + * @returns A double value. + * Returns INCORRECT_TYPE if the JSON element is not a number. + */ + inline simdjson_result get_double() const noexcept; + /** + * Cast this element to a bool. + * + * Equivalent to get(). + * + * @returns A bool value. + * Returns INCORRECT_TYPE if the JSON element is not a boolean. + */ + inline simdjson_result get_bool() const noexcept; + + /** + * Whether this element is a json array. + * + * Equivalent to is(). + */ + inline bool is_array() const noexcept; + /** + * Whether this element is a json object. + * + * Equivalent to is(). + */ + inline bool is_object() const noexcept; + /** + * Whether this element is a json string. + * + * Equivalent to is() or is(). + */ + inline bool is_string() const noexcept; + /** + * Whether this element is a json number that fits in a signed 64-bit integer. + * + * Equivalent to is(). + */ + inline bool is_int64_t() const noexcept; + /** + * Whether this element is a json number that fits in an unsigned 64-bit integer. + * + * Equivalent to is(). + */ + inline bool is_uint64_t() const noexcept; + /** + * Whether this element is a json number that fits in a double. + * + * Equivalent to is(). + */ + inline bool is_double() const noexcept; + /** + * Whether this element is a json number. + * + * Both integers and floating points will return true. + */ + inline bool is_number() const noexcept; + /** + * Whether this element is a json `true` or `false`. + * + * Equivalent to is(). + */ + inline bool is_bool() const noexcept; + /** + * Whether this element is a json `null`. + */ + inline bool is_null() const noexcept; /** * Tell whether the value can be cast to provided type (T). @@ -3791,7 +4099,44 @@ public: * INCORRECT_TYPE if the value cannot be cast to the given type. */ template - really_inline simdjson_result get() const noexcept; + inline simdjson_result get() const noexcept; + + /** + * Get the value as the provided type (T). + * + * Supported types: + * - Boolean: bool + * - Number: double, uint64_t, int64_t + * - String: std::string_view, const char * + * - Array: dom::array + * - Object: dom::object + * + * @tparam T bool, double, uint64_t, int64_t, std::string_view, const char *, dom::array, dom::object + * + * @param value The variable to set to the value. May not be set if there is an error. + * + * @returns The error that occurred, or SUCCESS if there was no error. + */ + template + WARN_UNUSED really_inline error_code get(T &value) const noexcept; + + /** + * Get the value as the provided type (T), setting error if it's not the given type. + * + * Supported types: + * - Boolean: bool + * - Number: double, uint64_t, int64_t + * - String: std::string_view, const char * + * - Array: dom::array + * - Object: dom::object + * + * @tparam T bool, double, uint64_t, int64_t, std::string_view, const char *, dom::array, dom::object + * + * @param value The variable to set to the given type. value is undefined if there is an error. + * @param error The variable to store the error. error is set to error_code::SUCCEED if there is an error. + */ + template + inline void tie(T &value, error_code &error) && noexcept; #if SIMDJSON_EXCEPTIONS /** @@ -3963,13 +4308,14 @@ public: inline bool dump_raw_tape(std::ostream &out) const noexcept; private: - really_inline element(const document *doc, size_t json_index) noexcept; + really_inline element(const internal::tape_ref &tape) noexcept; + internal::tape_ref tape; friend class document; friend class object; friend class array; friend struct simdjson_result; template - friend class simdjson::minify; + friend class simdjson::minifier; }; /** @@ -4002,32 +4348,51 @@ public: really_inline simdjson_result(dom::element &&value) noexcept; ///< @private really_inline simdjson_result(error_code error) noexcept; ///< @private - inline simdjson_result type() const noexcept; - inline simdjson_result is_null() const noexcept; + really_inline simdjson_result type() const noexcept; template - inline simdjson_result is() const noexcept; + really_inline simdjson_result is() const noexcept; template - inline simdjson_result get() const noexcept; + really_inline simdjson_result get() const noexcept; + template + WARN_UNUSED really_inline error_code get(T &value) const noexcept; - inline simdjson_result operator[](const std::string_view &key) const noexcept; - inline simdjson_result operator[](const char *key) const noexcept; - inline simdjson_result at(const std::string_view &json_pointer) const noexcept; - inline simdjson_result at(size_t index) const noexcept; - inline simdjson_result at_key(const std::string_view &key) const noexcept; - inline simdjson_result at_key_case_insensitive(const std::string_view &key) const noexcept; + really_inline simdjson_result get_array() const noexcept; + really_inline simdjson_result get_object() const noexcept; + really_inline simdjson_result get_c_str() const noexcept; + really_inline simdjson_result get_string() const noexcept; + really_inline simdjson_result get_int64_t() const noexcept; + really_inline simdjson_result get_uint64_t() const noexcept; + really_inline simdjson_result get_double() const noexcept; + really_inline simdjson_result get_bool() const noexcept; + + really_inline simdjson_result is_array() const noexcept; + really_inline simdjson_result is_object() const noexcept; + really_inline simdjson_result is_string() const noexcept; + really_inline simdjson_result is_int64_t() const noexcept; + really_inline simdjson_result is_uint64_t() const noexcept; + really_inline simdjson_result is_double() const noexcept; + really_inline simdjson_result is_bool() const noexcept; + really_inline simdjson_result is_null() const noexcept; + + really_inline simdjson_result operator[](const std::string_view &key) const noexcept; + really_inline simdjson_result operator[](const char *key) const noexcept; + really_inline simdjson_result at(const std::string_view &json_pointer) const noexcept; + really_inline simdjson_result at(size_t index) const noexcept; + really_inline simdjson_result at_key(const std::string_view &key) const noexcept; + really_inline simdjson_result at_key_case_insensitive(const std::string_view &key) const noexcept; #if SIMDJSON_EXCEPTIONS - inline operator bool() const noexcept(false); - inline explicit operator const char*() const noexcept(false); - inline operator std::string_view() const noexcept(false); - inline operator uint64_t() const noexcept(false); - inline operator int64_t() const noexcept(false); - inline operator double() const noexcept(false); - inline operator dom::array() const noexcept(false); - inline operator dom::object() const noexcept(false); + really_inline operator bool() const noexcept(false); + really_inline explicit operator const char*() const noexcept(false); + really_inline operator std::string_view() const noexcept(false); + really_inline operator uint64_t() const noexcept(false); + really_inline operator int64_t() const noexcept(false); + really_inline operator double() const noexcept(false); + really_inline operator dom::array() const noexcept(false); + really_inline operator dom::object() const noexcept(false); - inline dom::array::iterator begin() const noexcept(false); - inline dom::array::iterator end() const noexcept(false); + really_inline dom::array::iterator begin() const noexcept(false); + really_inline dom::array::iterator end() const noexcept(false); #endif // SIMDJSON_EXCEPTIONS }; @@ -4043,7 +4408,7 @@ public: * underlying output stream, that error will be propagated (simdjson_error will not be * thrown). */ -inline std::ostream& operator<<(std::ostream& out, const simdjson_result &value) noexcept(false); +really_inline std::ostream& operator<<(std::ostream& out, const simdjson_result &value) noexcept(false); #endif } // namespace simdjson @@ -4066,12 +4431,12 @@ class key_value_pair; /** * JSON object. */ -class object : protected internal::tape_ref { +class object { public: /** Create a new, invalid object */ really_inline object() noexcept; - class iterator : protected internal::tape_ref { + class iterator { public: /** * Get the actual key/value pair @@ -4119,7 +4484,10 @@ public: */ inline element value() const noexcept; private: - really_inline iterator(const document *doc, size_t json_index) noexcept; + really_inline iterator(const internal::tape_ref &tape) noexcept; + + internal::tape_ref tape; + friend class object; }; @@ -4150,6 +4518,8 @@ public: * parser.parse(R"({ "a\n": 1 })")["a\n"].get().value == 1 * parser.parse(R"({ "a\n": 1 })")["a\\n"].get().error == NO_SUCH_FIELD * + * This function has linear-time complexity: the keys are checked one by one. + * * @return The value associated with this field, or: * - NO_SUCH_FIELD if the field does not exist in the object * - INCORRECT_TYPE if this is not an object @@ -4165,6 +4535,8 @@ public: * parser.parse(R"({ "a\n": 1 })")["a\n"].get().value == 1 * parser.parse(R"({ "a\n": 1 })")["a\\n"].get().error == NO_SUCH_FIELD * + * This function has linear-time complexity: the keys are checked one by one. + * * @return The value associated with this field, or: * - NO_SUCH_FIELD if the field does not exist in the object * - INCORRECT_TYPE if this is not an object @@ -4196,6 +4568,8 @@ public: * parser.parse(R"({ "a\n": 1 })")["a\n"].get().value == 1 * parser.parse(R"({ "a\n": 1 })")["a\\n"].get().error == NO_SUCH_FIELD * + * This function has linear-time complexity: the keys are checked one by one. + * * @return The value associated with this field, or: * - NO_SUCH_FIELD if the field does not exist in the object */ @@ -4207,17 +4581,22 @@ public: * * Note: The key will be matched against **unescaped** JSON. * + * This function has linear-time complexity: the keys are checked one by one. + * * @return The value associated with this field, or: * - NO_SUCH_FIELD if the field does not exist in the object */ inline simdjson_result at_key_case_insensitive(const std::string_view &key) const noexcept; private: - really_inline object(const document *doc, size_t json_index) noexcept; + really_inline object(const internal::tape_ref &tape) noexcept; + + internal::tape_ref tape; + friend class element; friend struct simdjson_result; template - friend class simdjson::minify; + friend class simdjson::minifier; }; /** @@ -4840,16 +5219,16 @@ namespace dom { // // array inline implementation // -really_inline array::array() noexcept : internal::tape_ref() {} -really_inline array::array(const document *_doc, size_t _json_index) noexcept : internal::tape_ref(_doc, _json_index) {} +really_inline array::array() noexcept : tape{} {} +really_inline array::array(const internal::tape_ref &_tape) noexcept : tape{_tape} {} inline array::iterator array::begin() const noexcept { - return iterator(doc, json_index + 1); + return internal::tape_ref(tape.doc, tape.json_index + 1); } inline array::iterator array::end() const noexcept { - return iterator(doc, after_element() - 1); + return internal::tape_ref(tape.doc, tape.after_element() - 1); } inline size_t array::size() const noexcept { - return scope_count(); + return tape.scope_count(); } inline simdjson_result array::at(const std::string_view &json_pointer) const noexcept { // - means "the append position" or "the element after the end of the array" @@ -4873,7 +5252,7 @@ inline simdjson_result array::at(const std::string_view &json_pointer) if (i == 0) { return INVALID_JSON_POINTER; } // "Empty string in JSON pointer array index" // Get the child - auto child = array(doc, json_index).at(array_index); + auto child = array(tape).at(array_index); // If there is a /, we're not done yet, call recursively. if (i < json_pointer.length()) { child = child.at(json_pointer.substr(i+1)); @@ -4892,15 +5271,15 @@ inline simdjson_result array::at(size_t index) const noexcept { // // array::iterator inline implementation // -really_inline array::iterator::iterator(const document *_doc, size_t _json_index) noexcept : internal::tape_ref(_doc, _json_index) { } +really_inline array::iterator::iterator(const internal::tape_ref &_tape) noexcept : tape{_tape} { } inline element array::iterator::operator*() const noexcept { - return element(doc, json_index); + return element(tape); } inline bool array::iterator::operator!=(const array::iterator& other) const noexcept { - return json_index != other.json_index; + return tape.json_index != other.tape.json_index; } inline array::iterator& array::iterator::operator++() noexcept { - json_index = after_element(); + tape.json_index = tape.after_element(); return *this; } @@ -4911,7 +5290,7 @@ inline std::ostream& operator<<(std::ostream& out, const array &value) { } // namespace dom template<> -inline std::ostream& minify::print(std::ostream& out) { +inline std::ostream& minifier::print(std::ostream& out) { out << '['; auto iter = value.begin(); auto end = value.end(); @@ -4927,7 +5306,7 @@ inline std::ostream& minify::print(std::ostream& out) { #if SIMDJSON_EXCEPTIONS template<> -inline std::ostream& minify>::print(std::ostream& out) { +inline std::ostream& minifier>::print(std::ostream& out) { if (value.error()) { throw simdjson_error(value.error()); } return out << minify(value.first); } @@ -4949,34 +5328,95 @@ inline std::ostream& operator<<(std::ostream& out, const simdjson_result #include #include - namespace simdjson { namespace dom { +#ifdef SIMDJSON_THREADS_ENABLED +inline void stage1_worker::finish() { + std::unique_lock lock(locking_mutex); + cond_var.wait(lock, [this]{return has_work == false;}); +} + +inline stage1_worker::~stage1_worker() { + stop_thread(); +} + +inline void stage1_worker::start_thread() { + std::unique_lock lock(locking_mutex); + if(thread.joinable()) { + return; // This should never happen but we never want to create more than one thread. + } + thread = std::thread([this]{ + while(can_work) { + std::unique_lock thread_lock(locking_mutex); + cond_var.wait(thread_lock, [this]{return has_work || !can_work;}); + if(!can_work) { + break; + } + this->owner->stage1_thread_error = this->owner->run_stage1(*this->stage1_thread_parser, + this->_next_batch_start); + this->has_work = false; + thread_lock.unlock(); + cond_var.notify_one(); // will notify "finish" + } + } + ); +} + + +inline void stage1_worker::stop_thread() { + std::unique_lock lock(locking_mutex); + // We have to make sure that all locks can be released. + can_work = false; + has_work = false; + lock.unlock(); + cond_var.notify_all(); + if(thread.joinable()) { + thread.join(); + } +} + +inline void stage1_worker::run(document_stream * ds, dom::parser * stage1, size_t next_batch_start) { + std::unique_lock lock(locking_mutex); + owner = ds; + _next_batch_start = next_batch_start; + stage1_thread_parser = stage1; + has_work = true; + lock.unlock(); + cond_var.notify_one();// will notify the thread lock +} +#endif + really_inline document_stream::document_stream( dom::parser &_parser, const uint8_t *_buf, size_t _len, - size_t _batch_size, - error_code _error + size_t _batch_size ) noexcept - : parser{_parser}, + : parser{&_parser}, buf{_buf}, len{_len}, batch_size{_batch_size}, - error{_error} + error{SUCCESS} { -} - -inline document_stream::~document_stream() noexcept { #ifdef SIMDJSON_THREADS_ENABLED - // TODO kill the thread, why should people have to wait for a non-side-effecting operation to complete - if (stage1_thread.joinable()) { - stage1_thread.join(); + if(worker.get() == nullptr) { + error = MEMALLOC; } #endif } +really_inline document_stream::document_stream() noexcept + : parser{nullptr}, + buf{nullptr}, + len{0}, + batch_size{0}, + error{UNINITIALIZED} { +} + +really_inline document_stream::~document_stream() noexcept { +} + really_inline document_stream::iterator document_stream::begin() noexcept { start(); // If there are no documents, we're finished. @@ -4994,7 +5434,7 @@ really_inline document_stream::iterator::iterator(document_stream& _stream, bool really_inline simdjson_result document_stream::iterator::operator*() noexcept { // Once we have yielded any errors, we're finished. if (stream.error) { finished = true; return stream.error; } - return stream.parser.doc.root(); + return stream.parser->doc.root(); } really_inline document_stream::iterator& document_stream::iterator::operator++() noexcept { @@ -5011,12 +5451,12 @@ really_inline bool document_stream::iterator::operator!=(const document_stream:: inline void document_stream::start() noexcept { if (error) { return; } - error = parser.ensure_capacity(batch_size); + error = parser->ensure_capacity(batch_size); if (error) { return; } // Always run the first stage 1 parse immediately batch_start = 0; - error = run_stage1(parser, batch_start); + error = run_stage1(*parser, batch_start); if (error) { return; } #ifdef SIMDJSON_THREADS_ENABLED @@ -5024,6 +5464,7 @@ inline void document_stream::start() noexcept { // Kick off the first thread if needed error = stage1_thread_parser.ensure_capacity(batch_size); if (error) { return; } + worker->start_thread(); start_stage1_thread(); if (error) { return; } } @@ -5032,12 +5473,15 @@ inline void document_stream::start() noexcept { next(); } +really_inline size_t document_stream::iterator::current_index() noexcept { + return stream.doc_index; +} inline void document_stream::next() noexcept { if (error) { return; } // Load the next document from the batch - error = parser.implementation->stage2_next(parser.doc); - + doc_index = batch_start + parser->implementation->structural_indexes[parser->implementation->next_structural_index]; + error = parser->implementation->stage2_next(parser->doc); // If that was the last document in the batch, load another batch (if available) while (error == EMPTY) { batch_start = next_batch_start(); @@ -5046,17 +5490,17 @@ inline void document_stream::next() noexcept { #ifdef SIMDJSON_THREADS_ENABLED load_from_stage1_thread(); #else - error = run_stage1(parser, batch_start); + error = run_stage1(*parser, batch_start); #endif if (error) { continue; } // If the error was EMPTY, we may want to load another batch. - // Run stage 2 on the first document in the batch - error = parser.implementation->stage2_next(parser.doc); + doc_index = batch_start + parser->implementation->structural_indexes[parser->implementation->next_structural_index]; + error = parser->implementation->stage2_next(parser->doc); } } inline size_t document_stream::next_batch_start() const noexcept { - return batch_start + parser.implementation->structural_indexes[parser.implementation->n_structural_indexes]; + return batch_start + parser->implementation->structural_indexes[parser->implementation->n_structural_indexes]; } inline error_code document_stream::run_stage1(dom::parser &p, size_t _batch_start) noexcept { @@ -5072,11 +5516,10 @@ inline error_code document_stream::run_stage1(dom::parser &p, size_t _batch_star #ifdef SIMDJSON_THREADS_ENABLED inline void document_stream::load_from_stage1_thread() noexcept { - stage1_thread.join(); - + worker->finish(); // Swap to the parser that was loaded up in the thread. Make sure the parser has // enough memory to swap to, as well. - std::swap(parser, stage1_thread_parser); + std::swap(*parser, stage1_thread_parser); error = stage1_thread_error; if (error) { return; } @@ -5093,14 +5536,45 @@ inline void document_stream::start_stage1_thread() noexcept { // TODO this is NOT exception-safe. this->stage1_thread_error = UNINITIALIZED; // In case something goes wrong, make sure it's an error size_t _next_batch_start = this->next_batch_start(); - stage1_thread = std::thread([this, _next_batch_start] { - this->stage1_thread_error = run_stage1(this->stage1_thread_parser, _next_batch_start); - }); + + worker->run(this, & this->stage1_thread_parser, _next_batch_start); } #endif // SIMDJSON_THREADS_ENABLED } // namespace dom + +really_inline simdjson_result::simdjson_result() noexcept + : simdjson_result_base() { +} +really_inline simdjson_result::simdjson_result(error_code error) noexcept + : simdjson_result_base(error) { +} +really_inline simdjson_result::simdjson_result(dom::document_stream &&value) noexcept + : simdjson_result_base(std::forward(value)) { +} + +#if SIMDJSON_EXCEPTIONS +really_inline dom::document_stream::iterator simdjson_result::begin() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first.begin(); +} +really_inline dom::document_stream::iterator simdjson_result::end() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first.end(); +} +#else // SIMDJSON_EXCEPTIONS +really_inline dom::document_stream::iterator simdjson_result::begin() noexcept { + first.error = error(); + return first.begin(); +} +really_inline dom::document_stream::iterator simdjson_result::end() noexcept { + first.error = error(); + return first.end(); +} +#endif // SIMDJSON_EXCEPTIONS + + } // namespace simdjson #endif // SIMDJSON_INLINE_DOCUMENT_STREAM_H /* end file include/simdjson/inline/document_stream.h */ @@ -5120,7 +5594,7 @@ namespace dom { // document inline implementation // inline element document::root() const noexcept { - return element(this, 1); + return element(internal::tape_ref(this, 1)); } WARN_UNUSED @@ -5265,133 +5739,195 @@ inline simdjson_result simdjson_result::type() if (error()) { return error(); } return first.type(); } -inline simdjson_result simdjson_result::is_null() const noexcept { - if (error()) { return error(); } - return first.is_null(); -} + template -inline simdjson_result simdjson_result::is() const noexcept { +really_inline simdjson_result simdjson_result::is() const noexcept { if (error()) { return error(); } return first.is(); } template -inline simdjson_result simdjson_result::get() const noexcept { +really_inline simdjson_result simdjson_result::get() const noexcept { if (error()) { return error(); } return first.get(); } +template +WARN_UNUSED really_inline error_code simdjson_result::get(T &value) const noexcept { + if (error()) { return error(); } + return first.get(value); +} -inline simdjson_result simdjson_result::operator[](const std::string_view &key) const noexcept { +really_inline simdjson_result simdjson_result::get_array() const noexcept { + if (error()) { return error(); } + return first.get_array(); +} +really_inline simdjson_result simdjson_result::get_object() const noexcept { + if (error()) { return error(); } + return first.get_object(); +} +really_inline simdjson_result simdjson_result::get_c_str() const noexcept { + if (error()) { return error(); } + return first.get_c_str(); +} +really_inline simdjson_result simdjson_result::get_string() const noexcept { + if (error()) { return error(); } + return first.get_string(); +} +really_inline simdjson_result simdjson_result::get_int64_t() const noexcept { + if (error()) { return error(); } + return first.get_int64_t(); +} +really_inline simdjson_result simdjson_result::get_uint64_t() const noexcept { + if (error()) { return error(); } + return first.get_uint64_t(); +} +really_inline simdjson_result simdjson_result::get_double() const noexcept { + if (error()) { return error(); } + return first.get_double(); +} +really_inline simdjson_result simdjson_result::get_bool() const noexcept { + if (error()) { return error(); } + return first.get_bool(); +} + +really_inline simdjson_result simdjson_result::is_array() const noexcept { + if (error()) { return error(); } + return first.is_array(); +} +really_inline simdjson_result simdjson_result::is_object() const noexcept { + if (error()) { return error(); } + return first.is_object(); +} +really_inline simdjson_result simdjson_result::is_string() const noexcept { + if (error()) { return error(); } + return first.is_string(); +} +really_inline simdjson_result simdjson_result::is_int64_t() const noexcept { + if (error()) { return error(); } + return first.is_int64_t(); +} +really_inline simdjson_result simdjson_result::is_uint64_t() const noexcept { + if (error()) { return error(); } + return first.is_uint64_t(); +} +really_inline simdjson_result simdjson_result::is_double() const noexcept { + if (error()) { return error(); } + return first.is_double(); +} +really_inline simdjson_result simdjson_result::is_bool() const noexcept { + if (error()) { return error(); } + return first.is_bool(); +} + +really_inline simdjson_result simdjson_result::is_null() const noexcept { + if (error()) { return error(); } + return first.is_null(); +} + +really_inline simdjson_result simdjson_result::operator[](const std::string_view &key) const noexcept { if (error()) { return error(); } return first[key]; } -inline simdjson_result simdjson_result::operator[](const char *key) const noexcept { +really_inline simdjson_result simdjson_result::operator[](const char *key) const noexcept { if (error()) { return error(); } return first[key]; } -inline simdjson_result simdjson_result::at(const std::string_view &json_pointer) const noexcept { +really_inline simdjson_result simdjson_result::at(const std::string_view &json_pointer) const noexcept { if (error()) { return error(); } return first.at(json_pointer); } -inline simdjson_result simdjson_result::at(size_t index) const noexcept { +really_inline simdjson_result simdjson_result::at(size_t index) const noexcept { if (error()) { return error(); } return first.at(index); } -inline simdjson_result simdjson_result::at_key(const std::string_view &key) const noexcept { +really_inline simdjson_result simdjson_result::at_key(const std::string_view &key) const noexcept { if (error()) { return error(); } return first.at_key(key); } -inline simdjson_result simdjson_result::at_key_case_insensitive(const std::string_view &key) const noexcept { +really_inline simdjson_result simdjson_result::at_key_case_insensitive(const std::string_view &key) const noexcept { if (error()) { return error(); } return first.at_key_case_insensitive(key); } #if SIMDJSON_EXCEPTIONS -inline simdjson_result::operator bool() const noexcept(false) { +really_inline simdjson_result::operator bool() const noexcept(false) { return get(); } -inline simdjson_result::operator const char *() const noexcept(false) { +really_inline simdjson_result::operator const char *() const noexcept(false) { return get(); } -inline simdjson_result::operator std::string_view() const noexcept(false) { +really_inline simdjson_result::operator std::string_view() const noexcept(false) { return get(); } -inline simdjson_result::operator uint64_t() const noexcept(false) { +really_inline simdjson_result::operator uint64_t() const noexcept(false) { return get(); } -inline simdjson_result::operator int64_t() const noexcept(false) { +really_inline simdjson_result::operator int64_t() const noexcept(false) { return get(); } -inline simdjson_result::operator double() const noexcept(false) { +really_inline simdjson_result::operator double() const noexcept(false) { return get(); } -inline simdjson_result::operator dom::array() const noexcept(false) { +really_inline simdjson_result::operator dom::array() const noexcept(false) { return get(); } -inline simdjson_result::operator dom::object() const noexcept(false) { +really_inline simdjson_result::operator dom::object() const noexcept(false) { return get(); } -inline dom::array::iterator simdjson_result::begin() const noexcept(false) { +really_inline dom::array::iterator simdjson_result::begin() const noexcept(false) { if (error()) { throw simdjson_error(error()); } return first.begin(); } -inline dom::array::iterator simdjson_result::end() const noexcept(false) { +really_inline dom::array::iterator simdjson_result::end() const noexcept(false) { if (error()) { throw simdjson_error(error()); } return first.end(); } -#endif +#endif // SIMDJSON_EXCEPTIONS namespace dom { // // element inline implementation // -really_inline element::element() noexcept : internal::tape_ref() {} -really_inline element::element(const document *_doc, size_t _json_index) noexcept : internal::tape_ref(_doc, _json_index) { } +really_inline element::element() noexcept : tape{} {} +really_inline element::element(const internal::tape_ref &_tape) noexcept : tape{_tape} { } inline element_type element::type() const noexcept { - auto tape_type = tape_ref_type(); + auto tape_type = tape.tape_ref_type(); return tape_type == internal::tape_type::FALSE_VALUE ? element_type::BOOL : static_cast(tape_type); } -really_inline bool element::is_null() const noexcept { - return is_null_on_tape(); -} -template<> -inline simdjson_result element::get() const noexcept { - if(is_true()) { +inline simdjson_result element::get_bool() const noexcept { + if(tape.is_true()) { return true; - } else if(is_false()) { + } else if(tape.is_false()) { return false; } return INCORRECT_TYPE; } -template<> -inline simdjson_result element::get() const noexcept { - switch (tape_ref_type()) { +inline simdjson_result element::get_c_str() const noexcept { + switch (tape.tape_ref_type()) { case internal::tape_type::STRING: { - return get_c_str(); + return tape.get_c_str(); } default: return INCORRECT_TYPE; } } -template<> -inline simdjson_result element::get() const noexcept { - switch (tape_ref_type()) { +inline simdjson_result element::get_string() const noexcept { + switch (tape.tape_ref_type()) { case internal::tape_type::STRING: - return get_string_view(); + return tape.get_string_view(); default: return INCORRECT_TYPE; } } -template<> -inline simdjson_result element::get() const noexcept { - if(unlikely(!is_uint64())) { // branch rarely taken - if(is_int64()) { - int64_t result = next_tape_value(); +inline simdjson_result element::get_uint64_t() const noexcept { + if(unlikely(!tape.is_uint64())) { // branch rarely taken + if(tape.is_int64()) { + int64_t result = tape.next_tape_value(); if (result < 0) { return NUMBER_OUT_OF_RANGE; } @@ -5399,13 +5935,12 @@ inline simdjson_result element::get() const noexcept { } return INCORRECT_TYPE; } - return next_tape_value(); + return tape.next_tape_value(); } -template<> -inline simdjson_result element::get() const noexcept { - if(unlikely(!is_int64())) { // branch rarely taken - if(is_uint64()) { - uint64_t result = next_tape_value(); +inline simdjson_result element::get_int64_t() const noexcept { + if(unlikely(!tape.is_int64())) { // branch rarely taken + if(tape.is_uint64()) { + uint64_t result = tape.next_tape_value(); // Wrapping max in parens to handle Windows issue: https://stackoverflow.com/questions/11544073/how-do-i-deal-with-the-max-macro-in-windows-h-colliding-with-max-in-std if (result > uint64_t((std::numeric_limits::max)())) { return NUMBER_OUT_OF_RANGE; @@ -5414,10 +5949,9 @@ inline simdjson_result element::get() const noexcept { } return INCORRECT_TYPE; } - return next_tape_value(); + return tape.next_tape_value(); } -template<> -inline simdjson_result element::get() const noexcept { +inline simdjson_result element::get_double() const noexcept { // Performance considerations: // 1. Querying tape_ref_type() implies doing a shift, it is fast to just do a straight // comparison. @@ -5427,42 +5961,72 @@ inline simdjson_result element::get() const noexcept { // We can expect get to refer to a double type almost all the time. // It is important to craft the code accordingly so that the compiler can use this // information. (This could also be solved with profile-guided optimization.) - if(unlikely(!is_double())) { // branch rarely taken - if(is_uint64()) { - return double(next_tape_value()); - } else if(is_int64()) { - return double(next_tape_value()); + if(unlikely(!tape.is_double())) { // branch rarely taken + if(tape.is_uint64()) { + return double(tape.next_tape_value()); + } else if(tape.is_int64()) { + return double(tape.next_tape_value()); } return INCORRECT_TYPE; } // this is common: - return next_tape_value(); + return tape.next_tape_value(); } -template<> -inline simdjson_result element::get() const noexcept { - switch (tape_ref_type()) { +inline simdjson_result element::get_array() const noexcept { + switch (tape.tape_ref_type()) { case internal::tape_type::START_ARRAY: - return array(doc, json_index); + return array(tape); default: return INCORRECT_TYPE; } } -template<> -inline simdjson_result element::get() const noexcept { - switch (tape_ref_type()) { +inline simdjson_result element::get_object() const noexcept { + switch (tape.tape_ref_type()) { case internal::tape_type::START_OBJECT: - return object(doc, json_index); + return object(tape); default: return INCORRECT_TYPE; } } +template +WARN_UNUSED really_inline error_code element::get(T &value) const noexcept { + return get().get(value); +} +// An element-specific version prevents recursion with simdjson_result::get(value) +template<> +WARN_UNUSED really_inline error_code element::get(element &value) const noexcept { + value = element(tape); + return SUCCESS; +} + template really_inline bool element::is() const noexcept { auto result = get(); return !result.error(); } +template<> inline simdjson_result element::get() const noexcept { return get_array(); } +template<> inline simdjson_result element::get() const noexcept { return get_object(); } +template<> inline simdjson_result element::get() const noexcept { return get_c_str(); } +template<> inline simdjson_result element::get() const noexcept { return get_string(); } +template<> inline simdjson_result element::get() const noexcept { return get_int64_t(); } +template<> inline simdjson_result element::get() const noexcept { return get_uint64_t(); } +template<> inline simdjson_result element::get() const noexcept { return get_double(); } +template<> inline simdjson_result element::get() const noexcept { return get_bool(); } + +inline bool element::is_array() const noexcept { return is(); } +inline bool element::is_object() const noexcept { return is(); } +inline bool element::is_string() const noexcept { return is(); } +inline bool element::is_int64_t() const noexcept { return is(); } +inline bool element::is_uint64_t() const noexcept { return is(); } +inline bool element::is_double() const noexcept { return is(); } +inline bool element::is_bool() const noexcept { return is(); } + +inline bool element::is_null() const noexcept { + return tape.is_null_on_tape(); +} + #if SIMDJSON_EXCEPTIONS inline element::operator bool() const noexcept(false) { return get(); } @@ -5490,11 +6054,11 @@ inline simdjson_result element::operator[](const char *key) const noexc return at_key(key); } inline simdjson_result element::at(const std::string_view &json_pointer) const noexcept { - switch (tape_ref_type()) { + switch (tape.tape_ref_type()) { case internal::tape_type::START_OBJECT: - return object(doc, json_index).at(json_pointer); + return object(tape).at(json_pointer); case internal::tape_type::START_ARRAY: - return array(doc, json_index).at(json_pointer); + return array(tape).at(json_pointer); default: return INCORRECT_TYPE; } @@ -5510,7 +6074,7 @@ inline simdjson_result element::at_key_case_insensitive(const std::stri } inline bool element::dump_raw_tape(std::ostream &out) const noexcept { - return doc->dump_raw_tape(out); + return tape.doc->dump_raw_tape(out); } inline std::ostream& operator<<(std::ostream& out, const element &value) { @@ -5543,7 +6107,7 @@ inline std::ostream& operator<<(std::ostream& out, element_type type) { } // namespace dom template<> -inline std::ostream& minify::print(std::ostream& out) { +inline std::ostream& minifier::print(std::ostream& out) { using tape_type=internal::tape_type; size_t depth = 0; constexpr size_t MAX_DEPTH = 16; @@ -5551,7 +6115,7 @@ inline std::ostream& minify::print(std::ostream& out) { is_object[0] = false; bool after_value = false; - internal::tape_ref iter(value); + internal::tape_ref iter(value.tape); do { // print commas after each value if (after_value) { @@ -5569,7 +6133,7 @@ inline std::ostream& minify::print(std::ostream& out) { // If we're too deep, we need to recurse to go deeper. depth++; if (unlikely(depth >= MAX_DEPTH)) { - out << minify(dom::array(iter.doc, iter.json_index)); + out << minify(dom::array(iter)); iter.json_index = iter.matching_brace_index() - 1; // Jump to the ] depth--; break; @@ -5596,7 +6160,7 @@ inline std::ostream& minify::print(std::ostream& out) { // If we're too deep, we need to recurse to go deeper. depth++; if (unlikely(depth >= MAX_DEPTH)) { - out << minify(dom::object(iter.doc, iter.json_index)); + out << minify(dom::object(iter)); iter.json_index = iter.matching_brace_index() - 1; // Jump to the } depth--; break; @@ -5669,12 +6233,12 @@ inline std::ostream& minify::print(std::ostream& out) { #if SIMDJSON_EXCEPTIONS template<> -inline std::ostream& minify>::print(std::ostream& out) { +really_inline std::ostream& minifier>::print(std::ostream& out) { if (value.error()) { throw simdjson_error(value.error()); } return out << minify(value.first); } -inline std::ostream& operator<<(std::ostream& out, const simdjson_result &value) noexcept(false) { +really_inline std::ostream& operator<<(std::ostream& out, const simdjson_result &value) noexcept(false) { return out << minify>(value); } #endif @@ -5730,8 +6294,15 @@ really_inline void simdjson_result_base::tie(T &value, error_code &error) && // on the clang compiler that comes with current macOS (Apple clang version 11.0.0), // tie(width, error) = size["w"].get(); // fails with "error: no viable overloaded '='"" - value = std::forward>(*this).first; error = this->second; + if (!error) { + value = std::forward>(*this).first; + } +} + +template +WARN_UNUSED really_inline error_code simdjson_result_base::get(T &value) && noexcept { + return std::forward>(*this).get(value); } template @@ -5784,6 +6355,11 @@ really_inline void simdjson_result::tie(T &value, error_code &error) && noexc std::forward>(*this).tie(value, error); } +template +WARN_UNUSED really_inline error_code simdjson_result::get(T &value) && noexcept { + return std::forward>(*this).get(value); +} + template really_inline error_code simdjson_result::error() const noexcept { return internal::simdjson_result_base::error(); @@ -5887,16 +6463,16 @@ namespace dom { // // object inline implementation // -really_inline object::object() noexcept : internal::tape_ref() {} -really_inline object::object(const document *_doc, size_t _json_index) noexcept : internal::tape_ref(_doc, _json_index) { } +really_inline object::object() noexcept : tape{} {} +really_inline object::object(const internal::tape_ref &_tape) noexcept : tape{_tape} { } inline object::iterator object::begin() const noexcept { - return iterator(doc, json_index + 1); + return internal::tape_ref(tape.doc, tape.json_index + 1); } inline object::iterator object::end() const noexcept { - return iterator(doc, after_element() - 1); + return internal::tape_ref(tape.doc, tape.after_element() - 1); } inline size_t object::size() const noexcept { - return scope_count(); + return tape.scope_count(); } inline simdjson_result object::operator[](const std::string_view &key) const noexcept { @@ -5967,29 +6543,29 @@ inline simdjson_result object::at_key_case_insensitive(const std::strin // // object::iterator inline implementation // -really_inline object::iterator::iterator(const document *_doc, size_t _json_index) noexcept : internal::tape_ref(_doc, _json_index) { } +really_inline object::iterator::iterator(const internal::tape_ref &_tape) noexcept : tape{_tape} { } inline const key_value_pair object::iterator::operator*() const noexcept { return key_value_pair(key(), value()); } inline bool object::iterator::operator!=(const object::iterator& other) const noexcept { - return json_index != other.json_index; + return tape.json_index != other.tape.json_index; } inline object::iterator& object::iterator::operator++() noexcept { - json_index++; - json_index = after_element(); + tape.json_index++; + tape.json_index = tape.after_element(); return *this; } inline std::string_view object::iterator::key() const noexcept { - return get_string_view(); + return tape.get_string_view(); } inline uint32_t object::iterator::key_length() const noexcept { - return get_string_length(); + return tape.get_string_length(); } inline const char* object::iterator::key_c_str() const noexcept { - return reinterpret_cast(&doc->string_buf[size_t(tape_value()) + sizeof(uint32_t)]); + return reinterpret_cast(&tape.doc->string_buf[size_t(tape.tape_value()) + sizeof(uint32_t)]); } inline element object::iterator::value() const noexcept { - return element(doc, json_index + 1); + return element(internal::tape_ref(tape.doc, tape.json_index + 1)); } /** @@ -6044,7 +6620,7 @@ inline std::ostream& operator<<(std::ostream& out, const key_value_pair &value) } // namespace dom template<> -inline std::ostream& minify::print(std::ostream& out) { +inline std::ostream& minifier::print(std::ostream& out) { out << '{'; auto pair = value.begin(); auto end = value.end(); @@ -6058,14 +6634,14 @@ inline std::ostream& minify::print(std::ostream& out) { } template<> -inline std::ostream& minify::print(std::ostream& out) { +inline std::ostream& minifier::print(std::ostream& out) { return out << '"' << internal::escape_json_string(value.key) << "\":" << value.value; } #if SIMDJSON_EXCEPTIONS template<> -inline std::ostream& minify>::print(std::ostream& out) { +inline std::ostream& minifier>::print(std::ostream& out) { if (value.error()) { throw simdjson_error(value.error()); } return out << minify(value.first); } @@ -6786,23 +7362,21 @@ inline simdjson_result parser::read_file(const std::string &path) noexce inline simdjson_result parser::load(const std::string &path) & noexcept { size_t len; - error_code code; - read_file(path).tie(len, code); - if (code) { return code; } - + auto _error = read_file(path).get(len); + if (_error) { return _error; } return parse(loaded_bytes.get(), len, false); } -inline document_stream parser::load_many(const std::string &path, size_t batch_size) noexcept { +inline simdjson_result parser::load_many(const std::string &path, size_t batch_size) noexcept { size_t len; - error_code code; - read_file(path).tie(len, code); - return document_stream(*this, (const uint8_t*)loaded_bytes.get(), len, batch_size, code); + auto _error = read_file(path).get(len); + if (_error) { return _error; } + return document_stream(*this, (const uint8_t*)loaded_bytes.get(), len, batch_size); } inline simdjson_result parser::parse(const uint8_t *buf, size_t len, bool realloc_if_needed) & noexcept { - error_code code = ensure_capacity(len); - if (code) { return code; } + error_code _error = ensure_capacity(len); + if (_error) { return _error; } if (realloc_if_needed) { const uint8_t *tmp_buf = buf; @@ -6812,11 +7386,11 @@ inline simdjson_result parser::parse(const uint8_t *buf, size_t len, bo memcpy((void *)buf, tmp_buf, len); } - code = implementation->parse(buf, len, doc); + _error = implementation->parse(buf, len, doc); if (realloc_if_needed) { aligned_free((void *)buf); // must free before we exit } - if (code) { return code; } + if (_error) { return _error; } return doc.root(); } @@ -6830,16 +7404,16 @@ really_inline simdjson_result parser::parse(const padded_string &s) & n return parse(s.data(), s.length(), false); } -inline document_stream parser::parse_many(const uint8_t *buf, size_t len, size_t batch_size) noexcept { +inline simdjson_result parser::parse_many(const uint8_t *buf, size_t len, size_t batch_size) noexcept { return document_stream(*this, buf, len, batch_size); } -inline document_stream parser::parse_many(const char *buf, size_t len, size_t batch_size) noexcept { +inline simdjson_result parser::parse_many(const char *buf, size_t len, size_t batch_size) noexcept { return parse_many((const uint8_t *)buf, len, batch_size); } -inline document_stream parser::parse_many(const std::string &s, size_t batch_size) noexcept { +inline simdjson_result parser::parse_many(const std::string &s, size_t batch_size) noexcept { return parse_many(s.data(), s.length(), batch_size); } -inline document_stream parser::parse_many(const padded_string &s, size_t batch_size) noexcept { +inline simdjson_result parser::parse_many(const padded_string &s, size_t batch_size) noexcept { return parse_many(s.data(), s.length(), batch_size); } @@ -6859,15 +7433,22 @@ inline error_code parser::allocate(size_t capacity, size_t max_depth) noexcept { // Reallocate implementation and document if needed // error_code err; + // + // It is possible that we change max_depth without touching capacity, in + // which case, we do not want to reallocate the document buffers. + // + bool need_doc_allocation{false}; if (implementation) { + need_doc_allocation = implementation->capacity() != capacity || !doc.tape; err = implementation->allocate(capacity, max_depth); } else { + need_doc_allocation = true; err = simdjson::active_implementation->create_dom_parser_implementation(capacity, max_depth, implementation); } if (err) { return err; } - - if (implementation->capacity() != capacity || !doc.tape) { - return doc.allocate(capacity); + if (need_doc_allocation) { + err = doc.allocate(capacity); + if (err) { return err; } } return SUCCESS; } diff --git a/tests/basictests.cpp b/tests/basictests.cpp index 42c4ca417..80dc3d745 100644 --- a/tests/basictests.cpp +++ b/tests/basictests.cpp @@ -404,15 +404,13 @@ namespace document_stream_tests { } simdjson::dom::parser parser; const size_t window = 32; // deliberately small - auto stream = parser.parse_many(json,window); + simdjson::dom::document_stream stream; + ASSERT_SUCCESS( parser.parse_many(json,window).get(stream) ); auto i = stream.begin(); size_t count = 0; for(; i != stream.end(); ++i) { auto doc = *i; - if (doc.error()) { - std::cerr << doc.error() << std::endl; - return false; - } + ASSERT_SUCCESS(doc.error()); if( i.current_index() != count) { std::cout << "index:" << i.current_index() << std::endl; std::cout << "expected index:" << count << std::endl; @@ -428,7 +426,9 @@ namespace document_stream_tests { simdjson::dom::parser parser; size_t count = 0; size_t window_size = 10; // deliberately too small - for (auto doc : parser.parse_many(json, window_size)) { + simdjson::dom::document_stream stream; + ASSERT_SUCCESS( parser.parse_many(json, window_size).get(stream) ); + for (auto doc : stream) { if (!doc.error()) { std::cerr << "Expected a capacity error " << doc.error() << std::endl; return false; @@ -449,7 +449,9 @@ namespace document_stream_tests { simdjson::dom::parser parser; size_t count = 0; uint64_t window_size{17179869184}; // deliberately too big - for (auto doc : parser.parse_many(json, size_t(window_size))) { + simdjson::dom::document_stream stream; + ASSERT_SUCCESS( parser.parse_many(json, size_t(window_size)).get(stream) ); + for (auto doc : stream) { if (!doc.error()) { std::cerr << "I expected a failure (too big) but got " << doc.error() << std::endl; return false; @@ -462,7 +464,9 @@ namespace document_stream_tests { static bool parse_json_message_issue467(simdjson::padded_string &json, size_t expectedcount) { simdjson::dom::parser parser; size_t count = 0; - for (auto doc : parser.parse_many(json)) { + simdjson::dom::document_stream stream; + ASSERT_SUCCESS( parser.parse_many(json).get(stream) ); + for (auto doc : stream) { if (doc.error()) { std::cerr << "Failed with simdjson error= " << doc.error() << std::endl; return false; @@ -512,7 +516,9 @@ namespace document_stream_tests { simdjson::padded_string str(data); simdjson::dom::parser parser; size_t count = 0; - for (auto [doc, error] : parser.parse_many(str, batch_size)) { + simdjson::dom::document_stream stream; + ASSERT_SUCCESS( parser.parse_many(str, batch_size).get(stream) ); + for (auto [doc, error] : stream) { if (error) { printf("Error at on document %zd at batch size %zu: %s\n", count, batch_size, simdjson::error_message(error)); return false; @@ -562,7 +568,9 @@ namespace document_stream_tests { simdjson::padded_string str(data); simdjson::dom::parser parser; size_t count = 0; - for (auto [doc, error] : parser.parse_many(str, batch_size)) { + simdjson::dom::document_stream stream; + ASSERT_SUCCESS( parser.parse_many(str, batch_size).get(stream) ); + for (auto [doc, error] : stream) { if (error) { printf("Error at on document %zd at batch size %zu: %s\n", count, batch_size, simdjson::error_message(error)); return false; @@ -618,6 +626,23 @@ namespace parse_api_tests { return true; } bool parser_parse_many() { + std::cout << "Running " << __func__ << std::endl; + dom::parser parser; + int count = 0; + simdjson::dom::document_stream stream; + ASSERT_SUCCESS( parser.parse_many(BASIC_NDJSON).get(stream) ); + for (auto [doc, error] : stream) { + if (error) { cerr << "Error in parse_many: " << endl; return false; } + if (!doc.is()) { cerr << "Document did not parse as an array" << endl; return false; } + count++; + } + if (count != 2) { cerr << "parse_many returned " << count << " documents, expected 2" << endl; return false; } + return true; + } + + SIMDJSON_PUSH_DISABLE_WARNINGS + SIMDJSON_DISABLE_DEPRECATED_WARNING + bool parser_parse_many_deprecated() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; int count = 0; @@ -629,11 +654,14 @@ namespace parse_api_tests { if (count != 2) { cerr << "parse_many returned " << count << " documents, expected 2" << endl; return false; } return true; } + SIMDJSON_POP_DISABLE_WARNINGS bool parser_parse_many_empty() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; int count = 0; - for (auto doc : parser.parse_many(EMPTY_NDJSON)) { + simdjson::dom::document_stream stream; + ASSERT_SUCCESS( parser.parse_many(EMPTY_NDJSON).get(stream) ); + for (auto doc : stream) { if (doc.error()) { cerr << "Error in parse_many: " << doc.error() << endl; return false; } count++; } @@ -651,7 +679,9 @@ namespace parse_api_tests { memcpy(&empty_batches_ndjson[BATCH_SIZE*3+2], "1", 1); memcpy(&empty_batches_ndjson[BATCH_SIZE*10+4], "2", 1); memcpy(&empty_batches_ndjson[BATCH_SIZE*11+6], "3", 1); - for (auto [doc, error] : parser.parse_many(empty_batches_ndjson, BATCH_SIZE*16)) { + simdjson::dom::document_stream stream; + ASSERT_SUCCESS( parser.parse_many(empty_batches_ndjson, BATCH_SIZE*16).get(stream) ); + for (auto [doc, error] : stream) { if (error) { cerr << "Error in parse_many: " << error << endl; return false; } count++; auto [val, val_error] = doc.get(); @@ -671,6 +701,33 @@ namespace parse_api_tests { return true; } bool parser_load_many() { + std::cout << "Running " << __func__ << " on " << AMAZON_CELLPHONES_NDJSON << std::endl; + dom::parser parser; + int count = 0; + simdjson::dom::document_stream stream; + ASSERT_SUCCESS( parser.load_many(AMAZON_CELLPHONES_NDJSON).get(stream) ); + for (auto [doc, error] : stream) { + if (error) { cerr << error << endl; return false; } + + dom::array arr; + error = doc.get(arr); // let us get the array + if (error) { cerr << error << endl; return false; } + + if(arr.size() != 9) { cerr << "bad array size"<< endl; return false; } + + size_t c = 0; + for(auto v : arr) { c++; (void)v; } + if(c != 9) { cerr << "mismatched array size"<< endl; return false; } + + count++; + } + if (count != 793) { cerr << "Expected 793 documents, but load_many loaded " << count << " documents." << endl; return false; } + return true; + } + + SIMDJSON_PUSH_DISABLE_WARNINGS + SIMDJSON_DISABLE_DEPRECATED_WARNING + bool parser_load_many_deprecated() { std::cout << "Running " << __func__ << " on " << AMAZON_CELLPHONES_NDJSON << std::endl; dom::parser parser; int count = 0; @@ -692,6 +749,7 @@ namespace parse_api_tests { if (count != 793) { cerr << "Expected 793 documents, but load_many loaded " << count << " documents." << endl; return false; } return true; } + SIMDJSON_POP_DISABLE_WARNINGS #if SIMDJSON_EXCEPTIONS @@ -744,10 +802,12 @@ namespace parse_api_tests { bool run() { return parser_parse() && parser_parse_many() && + parser_parse_many_deprecated() && parser_parse_many_empty() && parser_parse_many_empty_batches() && parser_load() && parser_load_many() && + parser_load_many_deprecated() && #if SIMDJSON_EXCEPTIONS parser_parse_exception() && parser_parse_many_exception() && diff --git a/tests/errortests.cpp b/tests/errortests.cpp index aaaa9bcc3..6af38277b 100644 --- a/tests/errortests.cpp +++ b/tests/errortests.cpp @@ -14,16 +14,8 @@ using namespace simdjson; using namespace std; -#ifndef SIMDJSON_BENCHMARK_DATA_DIR -#define SIMDJSON_BENCHMARK_DATA_DIR "jsonexamples/" -#endif -const char *TWITTER_JSON = SIMDJSON_BENCHMARK_DATA_DIR "twitter.json"; +#include "test_macros.h" -#define TEST_START() { cout << "Running " << __func__ << " ..." << endl; } -#define ASSERT_ERROR(ACTUAL, EXPECTED) if ((ACTUAL) != (EXPECTED)) { cerr << "FAIL: Unexpected error \"" << (ACTUAL) << "\" (expected \"" << (EXPECTED) << "\")" << endl; return false; } -#define ASSERT_SUCCESS(CODE) do { simdjson::error_code error = CODE; if (error) { cerr << "FAIL: Unexpected error " << error << endl; return false; } } while (0); -#define TEST_FAIL(MESSAGE) { cerr << "FAIL: " << (MESSAGE) << endl; return false; } -#define TEST_SUCCEED() { return true; } namespace parser_load { const char * NONEXISTENT_FILE = "this_file_does_not_exist.json"; bool parser_load_capacity() { @@ -112,7 +104,9 @@ namespace parser_load { bool parser_load_many_nonexistent() { TEST_START(); dom::parser parser; - for (auto doc : parser.load_many(NONEXISTENT_FILE)) { + dom::document_stream stream; + ASSERT_SUCCESS(parser.load_many(NONEXISTENT_FILE).get(stream)); + for (auto doc : stream) { ASSERT_ERROR(doc.error(), IO_ERROR); TEST_SUCCEED(); } @@ -135,7 +129,9 @@ namespace parser_load { bool parser_load_many_chain() { TEST_START(); dom::parser parser; - for (auto doc : parser.load_many(NONEXISTENT_FILE)) { + dom::document_stream stream; + ASSERT_SUCCESS( parser.load_many(NONEXISTENT_FILE).get(stream) ); + for (auto doc : stream) { auto error = doc["foo"].get().error(); ASSERT_ERROR(error, IO_ERROR); TEST_SUCCEED(); diff --git a/tests/test_macros.h b/tests/test_macros.h index b671f0b9b..35228ca24 100644 --- a/tests/test_macros.h +++ b/tests/test_macros.h @@ -26,9 +26,14 @@ template<> bool equals_expected(const char *actual, const char *expected) { return !strcmp(actual, expected); } + +#define TEST_START() { cout << "Running " << __func__ << " ..." << endl; } #define ASSERT_EQUAL(ACTUAL, EXPECTED) do { auto _actual = (ACTUAL); auto _expected = (EXPECTED); if (!equals_expected(_actual, _expected)) { std::cerr << "Expected " << #ACTUAL << " to be " << _expected << ", got " << _actual << " instead!" << std::endl; return false; } } while(0); +#define ASSERT_ERROR(ACTUAL, EXPECTED) do { auto _actual = (ACTUAL); auto _expected = (EXPECTED); if (_actual != _expected) { std::cerr << "FAIL: Unexpected error \"" << _actual << "\" (expected \"" << _expected << "\")" << std::endl; return false; } } while (0); #define ASSERT(RESULT, MESSAGE) if (!(RESULT)) { std::cerr << MESSAGE << std::endl; return false; } #define RUN_TEST(RESULT) if (!RESULT) { return false; } #define ASSERT_SUCCESS(ERROR) do { auto _error = (ERROR); if (_error) { std::cerr << _error << std::endl; return false; } } while(0); +#define TEST_FAIL(MESSAGE) { std::cerr << "FAIL: " << (MESSAGE) << std::endl; return false; } +#define TEST_SUCCEED() { return true; } #endif // TEST_MACROS_H \ No newline at end of file From 1b1a122b1f41956236b0c4b6e412d7566fde2b2d Mon Sep 17 00:00:00 2001 From: John Keiser Date: Sun, 21 Jun 2020 11:49:52 -0700 Subject: [PATCH 04/10] Fix copy constructor issue on older gcc --- include/simdjson/dom/document_stream.h | 3 +-- singleheader/amalgamate_demo.cpp | 2 +- singleheader/simdjson.cpp | 2 +- singleheader/simdjson.h | 14 +++++++------- tests/errortests.cpp | 21 ++++++--------------- 5 files changed, 16 insertions(+), 26 deletions(-) diff --git a/include/simdjson/dom/document_stream.h b/include/simdjson/dom/document_stream.h index a650bf4ee..eb4f53c54 100644 --- a/include/simdjson/dom/document_stream.h +++ b/include/simdjson/dom/document_stream.h @@ -143,8 +143,7 @@ public: private: document_stream &operator=(const document_stream &) = delete; // Disallow copying - - document_stream(document_stream &other) = delete; // Disallow copying + document_stream(const document_stream &other) = delete; // Disallow copying /** * Construct a document_stream. Does not allocate or parse anything until the iterator is diff --git a/singleheader/amalgamate_demo.cpp b/singleheader/amalgamate_demo.cpp index 94f984a95..3dfbddcf5 100644 --- a/singleheader/amalgamate_demo.cpp +++ b/singleheader/amalgamate_demo.cpp @@ -1,4 +1,4 @@ -/* auto-generated on Sat Jun 20 21:35:29 PDT 2020. Do not edit! */ +/* auto-generated on Sun Jun 21 11:49:12 PDT 2020. Do not edit! */ #include #include "simdjson.h" diff --git a/singleheader/simdjson.cpp b/singleheader/simdjson.cpp index ab26d2ad9..d78ad5df6 100644 --- a/singleheader/simdjson.cpp +++ b/singleheader/simdjson.cpp @@ -1,4 +1,4 @@ -/* auto-generated on Sat Jun 20 21:35:29 PDT 2020. Do not edit! */ +/* auto-generated on Sun Jun 21 11:49:12 PDT 2020. Do not edit! */ /* begin file src/simdjson.cpp */ #include "simdjson.h" diff --git a/singleheader/simdjson.h b/singleheader/simdjson.h index 0efec0167..63a8f14ec 100644 --- a/singleheader/simdjson.h +++ b/singleheader/simdjson.h @@ -1,4 +1,4 @@ -/* auto-generated on Sat Jun 20 21:50:03 PDT 2020. Do not edit! */ +/* auto-generated on Sun Jun 21 11:49:12 PDT 2020. Do not edit! */ /* begin file include/simdjson.h */ #ifndef SIMDJSON_H #define SIMDJSON_H @@ -3778,8 +3778,7 @@ public: private: document_stream &operator=(const document_stream &) = delete; // Disallow copying - - document_stream(document_stream &other) = delete; // Disallow copying + document_stream(const document_stream &other) = delete; // Disallow copying /** * Construct a document_stream. Does not allocate or parse anything until the iterator is @@ -3867,8 +3866,8 @@ private: #endif // SIMDJSON_THREADS_ENABLED friend class dom::parser; - friend class simdjson_result; - friend class internal::simdjson_result_base; + friend struct simdjson_result; + friend struct internal::simdjson_result_base; }; // class document_stream @@ -5574,7 +5573,6 @@ really_inline dom::document_stream::iterator simdjson_result::tie(T &value, error_code &error) && template WARN_UNUSED really_inline error_code simdjson_result_base::get(T &value) && noexcept { - return std::forward>(*this).get(value); + error_code error; + std::forward>(*this).tie(value, error); + return error; } template diff --git a/tests/errortests.cpp b/tests/errortests.cpp index 6af38277b..3be94fcd0 100644 --- a/tests/errortests.cpp +++ b/tests/errortests.cpp @@ -105,12 +105,8 @@ namespace parser_load { TEST_START(); dom::parser parser; dom::document_stream stream; - ASSERT_SUCCESS(parser.load_many(NONEXISTENT_FILE).get(stream)); - for (auto doc : stream) { - ASSERT_ERROR(doc.error(), IO_ERROR); - TEST_SUCCEED(); - } - TEST_FAIL("No documents returned"); + ASSERT_ERROR(parser.load_many(NONEXISTENT_FILE).get(stream), IO_ERROR); + TEST_SUCCEED(); } bool padded_string_load_nonexistent() { TEST_START(); @@ -122,21 +118,16 @@ namespace parser_load { bool parser_load_chain() { TEST_START(); dom::parser parser; - auto error = parser.load(NONEXISTENT_FILE)["foo"].get().error(); - ASSERT_ERROR(error, IO_ERROR); + UNUSED uint64_t foo; + ASSERT_ERROR( parser.load(NONEXISTENT_FILE)["foo"].get(foo) , IO_ERROR); TEST_SUCCEED(); } bool parser_load_many_chain() { TEST_START(); dom::parser parser; dom::document_stream stream; - ASSERT_SUCCESS( parser.load_many(NONEXISTENT_FILE).get(stream) ); - for (auto doc : stream) { - auto error = doc["foo"].get().error(); - ASSERT_ERROR(error, IO_ERROR); - TEST_SUCCEED(); - } - TEST_FAIL("No documents returned"); + ASSERT_ERROR( parser.load_many(NONEXISTENT_FILE).get(stream) , IO_ERROR ); + TEST_SUCCEED(); } bool run() { return true From 6fa5abcd7e481738c5588a286a02ad59d3a506c0 Mon Sep 17 00:00:00 2001 From: John Keiser Date: Sun, 21 Jun 2020 14:36:38 -0700 Subject: [PATCH 05/10] Replace x.get() with x.get(v) or T(x) --- benchmark/bench_dom_api.cpp | 6 +- benchmark/distinctuseridcompetition.cpp | 11 +- benchmark/parseandstatcompetition.cpp | 4 +- doc/basics.md | 19 +- include/simdjson/dom/element.h | 12 +- include/simdjson/dom/object.h | 12 +- tests/basictests.cpp | 531 +++++++++++------------- tests/errortests.cpp | 36 +- tests/integer_tests.cpp | 66 +-- tests/pointercheck.cpp | 52 ++- tests/readme_examples.cpp | 12 +- tests/test_macros.h | 18 +- 12 files changed, 366 insertions(+), 413 deletions(-) diff --git a/benchmark/bench_dom_api.cpp b/benchmark/bench_dom_api.cpp index 1440942c7..294977e25 100644 --- a/benchmark/bench_dom_api.cpp +++ b/benchmark/bench_dom_api.cpp @@ -407,7 +407,7 @@ static void iterator_twitter_default_profile(State& state) { set default_users; ParsedJson::Iterator iter(pj); - // for (dom::object tweet : doc["statuses"].get()) { + // for (dom::object tweet : doc["statuses"]) { if (!(iter.move_to_key("statuses") && iter.is_array())) { return; } if (iter.down()) { // first status do { @@ -480,7 +480,7 @@ static void iterator_twitter_image_sizes(State& state) { set> image_sizes; ParsedJson::Iterator iter(pj); - // for (dom::object tweet : doc["statuses"].get()) { + // for (dom::object tweet : doc["statuses"]) { if (!(iter.move_to_key("statuses") && iter.is_array())) { return; } if (iter.down()) { // first status do { @@ -492,7 +492,7 @@ static void iterator_twitter_image_sizes(State& state) { if (iter.move_to_key("media")) { if (!iter.is_array()) { return; } - // for (dom::object image : media.get()) { + // for (dom::object image : media) { if (iter.down()) { // first media do { diff --git a/benchmark/distinctuseridcompetition.cpp b/benchmark/distinctuseridcompetition.cpp index 100c48f66..a9bb86dc6 100644 --- a/benchmark/distinctuseridcompetition.cpp +++ b/benchmark/distinctuseridcompetition.cpp @@ -40,17 +40,18 @@ void print_vec(const std::vector &v) { // simdjson_recurse below come be implemented like so but it is slow: /*void simdjson_recurse(std::vector & v, simdjson::dom::element element) { - if (element.is()) { - auto [array, array_error] = element.get(); + error_code error; + if (element.is_array()) { + dom::array array; + error = element.get(array); for (auto child : array) { if (child.is() || child.is()) { simdjson_recurse(v, child); } } - } else if (element.is()) { - auto [object, error] = element.get(); + } else if (element.is_object()) { int64_t id; - error = object["user"]["id"].get(id); + error = element["user"]["id"].get(id); if(!error) { v.push_back(id); } diff --git a/benchmark/parseandstatcompetition.cpp b/benchmark/parseandstatcompetition.cpp index eb017ebd9..d18ffc992 100644 --- a/benchmark/parseandstatcompetition.cpp +++ b/benchmark/parseandstatcompetition.cpp @@ -154,11 +154,11 @@ static void GenStatPlus(Stat &stat, const dom::element &v) { break; case dom::element_type::STRING: { stat.stringCount++; - std::string_view sv = v.get(); + auto sv = std::string_view(v); stat.stringLength += sv.size(); } break; case dom::element_type::BOOL: - if (v.get()) { + if (bool(v)) { stat.trueCount++; } else { stat.falseCount++; diff --git a/doc/basics.md b/doc/basics.md index e1bb0708c..0f5c5d541 100644 --- a/doc/basics.md +++ b/doc/basics.md @@ -164,7 +164,7 @@ And another one: auto abstract_json = R"( { "str" : { "123" : {"abc" : 3.14 } } } )"_padded; dom::parser parser; - double v = parser.parse(abstract_json)["str"]["123"]["abc"].get(); + double v = parser.parse(abstract_json)["str"]["123"]["abc"]; cout << "number: " << v << endl; ``` @@ -191,14 +191,13 @@ Though it does not validate the JSON input, it will detect when the document end C++17 Support ------------- -While the simdjson library can be used in any project using C++ 11 and above, it has special support -for C++ 17. The APIs for field iteration and error handling in particular are designed to work -nicely with C++17's destructuring syntax. For example: +While the simdjson library can be used in any project using C++ 11 and above, field iteration has special support C++ 17's destructuring syntax. For example: ```c++ -dom::parser parser; padded_string json = R"( { "foo": 1, "bar": 2 } )"_padded; -auto [object, error] = parser.parse(json).get(); +dom::parser parser; +dom::object object; +auto error = parser.parse(json).get(object); if (error) { cerr << error << endl; return; } for (auto [key, value] : object) { cout << key << " = " << value << endl; @@ -209,11 +208,10 @@ For comparison, here is the C++ 11 version of the same code: ```c++ // C++ 11 version for comparison -dom::parser parser; padded_string json = R"( { "foo": 1, "bar": 2 } )"_padded; -simdjson::error_code error; +dom::parser parser; dom::object object; -error = parser.parse(json).get(object); +auto error = parser.parse(json).get(object); if (!error) { cerr << error << endl; return; } for (dom::key_value_pair field : object) { cout << field.key << " = " << field.value << endl; @@ -378,8 +376,7 @@ And another one: cout << "number: " << v << endl; ``` -Notice how we can string several operation (`parser.parse(abstract_json)["str"]["123"]["abc"].get()`) and only check for the error once, a strategy we call *error chaining*. - +Notice how we can string several operations (`parser.parse(abstract_json)["str"]["123"]["abc"].get(v)`) and only check for the error once, a strategy we call *error chaining*. The next two functions will take as input a JSON document containing an array with a single element, either a string or a number. They return true upon success. diff --git a/include/simdjson/dom/element.h b/include/simdjson/dom/element.h index 6d9af51dd..9a9ce6f13 100644 --- a/include/simdjson/dom/element.h +++ b/include/simdjson/dom/element.h @@ -336,8 +336,8 @@ public: * The key will be matched against **unescaped** JSON: * * dom::parser parser; - * parser.parse(R"({ "a\n": 1 })")["a\n"].get().value == 1 - * parser.parse(R"({ "a\n": 1 })")["a\\n"].get().error == NO_SUCH_FIELD + * parser.parse(R"({ "a\n": 1 })")["a\n"].get().first == 1 + * parser.parse(R"({ "a\n": 1 })")["a\\n"].get().error() == NO_SUCH_FIELD * * @return The value associated with this field, or: * - NO_SUCH_FIELD if the field does not exist in the object @@ -351,8 +351,8 @@ public: * The key will be matched against **unescaped** JSON: * * dom::parser parser; - * parser.parse(R"({ "a\n": 1 })")["a\n"].get().value == 1 - * parser.parse(R"({ "a\n": 1 })")["a\\n"].get().error == NO_SUCH_FIELD + * parser.parse(R"({ "a\n": 1 })")["a\n"].get().first == 1 + * parser.parse(R"({ "a\n": 1 })")["a\\n"].get().error() == NO_SUCH_FIELD * * @return The value associated with this field, or: * - NO_SUCH_FIELD if the field does not exist in the object @@ -391,8 +391,8 @@ public: * The key will be matched against **unescaped** JSON: * * dom::parser parser; - * parser.parse(R"({ "a\n": 1 })")["a\n"].get().value == 1 - * parser.parse(R"({ "a\n": 1 })")["a\\n"].get().error == NO_SUCH_FIELD + * parser.parse(R"({ "a\n": 1 })")["a\n"].get().first == 1 + * parser.parse(R"({ "a\n": 1 })")["a\\n"].get().error() == NO_SUCH_FIELD * * @return The value associated with this field, or: * - NO_SUCH_FIELD if the field does not exist in the object diff --git a/include/simdjson/dom/object.h b/include/simdjson/dom/object.h index 9316914d9..301dad945 100644 --- a/include/simdjson/dom/object.h +++ b/include/simdjson/dom/object.h @@ -101,8 +101,8 @@ public: * The key will be matched against **unescaped** JSON: * * dom::parser parser; - * parser.parse(R"({ "a\n": 1 })")["a\n"].get().value == 1 - * parser.parse(R"({ "a\n": 1 })")["a\\n"].get().error == NO_SUCH_FIELD + * parser.parse(R"({ "a\n": 1 })")["a\n"].get().first == 1 + * parser.parse(R"({ "a\n": 1 })")["a\\n"].get().error() == NO_SUCH_FIELD * * This function has linear-time complexity: the keys are checked one by one. * @@ -118,8 +118,8 @@ public: * The key will be matched against **unescaped** JSON: * * dom::parser parser; - * parser.parse(R"({ "a\n": 1 })")["a\n"].get().value == 1 - * parser.parse(R"({ "a\n": 1 })")["a\\n"].get().error == NO_SUCH_FIELD + * parser.parse(R"({ "a\n": 1 })")["a\n"].get().first == 1 + * parser.parse(R"({ "a\n": 1 })")["a\\n"].get().error() == NO_SUCH_FIELD * * This function has linear-time complexity: the keys are checked one by one. * @@ -151,8 +151,8 @@ public: * The key will be matched against **unescaped** JSON: * * dom::parser parser; - * parser.parse(R"({ "a\n": 1 })")["a\n"].get().value == 1 - * parser.parse(R"({ "a\n": 1 })")["a\\n"].get().error == NO_SUCH_FIELD + * parser.parse(R"({ "a\n": 1 })")["a\n"].get().first == 1 + * parser.parse(R"({ "a\n": 1 })")["a\\n"].get().error() == NO_SUCH_FIELD * * This function has linear-time complexity: the keys are checked one by one. * diff --git a/tests/basictests.cpp b/tests/basictests.cpp index 80dc3d745..e5f3d8e85 100644 --- a/tests/basictests.cpp +++ b/tests/basictests.cpp @@ -17,6 +17,8 @@ #include "cast_tester.h" #include "test_macros.h" +const size_t AMAZON_CELLPHONES_NDJSON_DOC_COUNT = 793; + namespace number_tests { // ulp distance @@ -39,8 +41,8 @@ namespace number_tests { for (int m = 10; m < 20; m++) { for (int i = -1024; i < 1024; i++) { auto str = std::to_string(i); - auto [actual, error] = parser.parse(str).get(); - if (error) { std::cerr << error << std::endl; return false; } + int64_t actual; + ASSERT_SUCCESS(parser.parse(str).get(actual)); if (actual != i) { std::cerr << "JSON '" << str << "' parsed to " << actual << " instead of " << i << std::endl; return false; @@ -60,7 +62,8 @@ namespace number_tests { size_t n = snprintf(buf, sizeof(buf), "%.*e", std::numeric_limits::max_digits10 - 1, expected); if (n >= sizeof(buf)) { abort(); } fflush(NULL); - auto [actual, error] = parser.parse(buf, n).get(); + double actual; + auto error = parser.parse(buf, n).get(actual); if (error) { std::cerr << error << std::endl; return false; } uint64_t ulp = f64_ulp_dist(actual,expected); if(ulp > maxulp) maxulp = ulp; @@ -154,7 +157,8 @@ namespace number_tests { if (n >= sizeof(buf)) { abort(); } fflush(NULL); - auto [actual, error] = parser.parse(buf, n).get(); + double actual; + auto error = parser.parse(buf, n).get(actual); if (error) { std::cerr << error << std::endl; return false; } double expected = ((i >= -307) ? testing_power_of_ten[i + 307]: std::pow(10, i)); int ulp = (int) f64_ulp_dist(actual, expected); @@ -229,41 +233,27 @@ namespace document_tests { std::cout << __func__ << std::endl; simdjson::padded_string smalljson = "[1,2,3]"_padded; simdjson::dom::parser parser; - auto [doc, error] = parser.parse(smalljson).get(); - if (error) { - printf("This json should be valid %s.\n", smalljson.data()); - return false; - } - if(doc.size() != 3) { - printf("This json should have size three but found %zu : %s.\n", doc.size(), smalljson.data()); - return false; - } + simdjson::dom::array array; + ASSERT_SUCCESS( parser.parse(smalljson).get(array) ); + ASSERT_EQUAL( array.size(), 3 ); return true; } bool count_object_example() { std::cout << __func__ << std::endl; simdjson::padded_string smalljson = "{\"1\":1,\"2\":1,\"3\":1}"_padded; simdjson::dom::parser parser; - auto [doc, error] = parser.parse(smalljson).get(); - if (error) { - printf("This json should be valid %s.\n", smalljson.data()); - return false; - } - if(doc.size() != 3) { - printf("This json should have size three but found %zu : %s.\n", doc.size(), smalljson.data()); - return false; - } + simdjson::dom::object object; + ASSERT_SUCCESS( parser.parse(smalljson).get(object) ); + ASSERT_EQUAL( object.size(), 3 ); return true; } bool padded_with_open_bracket() { std::cout << __func__ << std::endl; simdjson::dom::parser parser; // This is an invalid document padded with open braces. - auto error1 = parser.parse("[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[", 2, false).error(); - if (!error1) { std::cerr << "We expected an error but got: " << error1 << std::endl; return false; } + ASSERT_ERROR( parser.parse("[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[", 2, false).error(), simdjson::TAPE_ERROR); // This is a valid document padded with open braces. - auto error2 = parser.parse("[][[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[", 2, false).error(); - if (error2) { std::cerr << "Error: " << error2 << std::endl; return false; } + ASSERT_SUCCESS( parser.parse("[][[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[", 2, false).error() ); return true; } // returns true if successful @@ -524,16 +514,9 @@ namespace document_stream_tests { return false; } - auto [keyid, error2] = doc["id"].get(); - if (error2) { - printf("Error getting id as int64 on document %zd at batch size %zu: %s\n", count, batch_size, simdjson::error_message(error2)); - return false; - } - - if (keyid != int64_t(count)) { - printf("key does not match %" PRId64 ", expected %zd on document %zd at batch size %zu\n", keyid, count, count, batch_size); - return false; - } + int64_t keyid; + ASSERT_SUCCESS( doc["id"].get(keyid) ); + ASSERT_EQUAL( keyid, int64_t(count) ); count++; } @@ -576,23 +559,13 @@ namespace document_stream_tests { return false; } - auto [keyid, error2] = doc["id"].get(); - if (error2) { - printf("Error getting id as int64 on document %zd at batch size %zu: %s\n", count, batch_size, simdjson::error_message(error2)); - return false; - } - - if (keyid != int64_t(count)) { - printf("key does not match %" PRId64 ", expected %zd on document %zd at batch size %zu\n", keyid, count, count, batch_size); - return false; - } + int64_t keyid; + ASSERT_SUCCESS( doc["id"].get(keyid) ); + ASSERT_EQUAL( keyid, int64_t(count) ); count++; } - if(count != n_records) { - printf("Found wrong number of documents %zd, expected %zd at batch size %zu\n", count, n_records, batch_size); - return false; - } + ASSERT_EQUAL( count, n_records ) } printf("ok\n"); return true; @@ -662,10 +635,10 @@ namespace parse_api_tests { simdjson::dom::document_stream stream; ASSERT_SUCCESS( parser.parse_many(EMPTY_NDJSON).get(stream) ); for (auto doc : stream) { - if (doc.error()) { cerr << "Error in parse_many: " << doc.error() << endl; return false; } + ASSERT_SUCCESS(doc.error()); count++; } - if (count != 0) { cerr << "parse_many returned " << count << " documents, expected 0" << endl; return false; } + ASSERT_EQUAL(count, 0); return true; } @@ -682,22 +655,21 @@ namespace parse_api_tests { simdjson::dom::document_stream stream; ASSERT_SUCCESS( parser.parse_many(empty_batches_ndjson, BATCH_SIZE*16).get(stream) ); for (auto [doc, error] : stream) { - if (error) { cerr << "Error in parse_many: " << error << endl; return false; } + ASSERT_SUCCESS(error); count++; - auto [val, val_error] = doc.get(); - if (val_error) { cerr << "Document is not an unsigned int: " << val_error << endl; return false; } - if (val != count) { cerr << "Expected document #" << count << " to equal " << count << ", but got " << val << " instead!" << endl; return false; } + uint64_t val; + ASSERT_SUCCESS( doc.get(val) ); + ASSERT_EQUAL( val, count ); } - if (count != 3) { cerr << "parse_many returned " << count << " documents, expected 0" << endl; return false; } + ASSERT_EQUAL(count, 3); return true; } bool parser_load() { std::cout << "Running " << __func__ << " on " << TWITTER_JSON << std::endl; dom::parser parser; - auto [doc, error] = parser.load(TWITTER_JSON); - if (error) { cerr << error << endl; return false; } - if (!doc.is()) { cerr << "Document did not parse as an object" << endl; return false; } + dom::object object; + ASSERT_SUCCESS( parser.load(TWITTER_JSON).get(object) ); return true; } bool parser_load_many() { @@ -707,21 +679,19 @@ namespace parse_api_tests { simdjson::dom::document_stream stream; ASSERT_SUCCESS( parser.load_many(AMAZON_CELLPHONES_NDJSON).get(stream) ); for (auto [doc, error] : stream) { - if (error) { cerr << error << endl; return false; } + ASSERT_SUCCESS( error ); dom::array arr; - error = doc.get(arr); // let us get the array - if (error) { cerr << error << endl; return false; } + ASSERT_SUCCESS( doc.get(arr) ); // let us get the array + ASSERT_EQUAL(arr.size(), 9); - if(arr.size() != 9) { cerr << "bad array size"<< endl; return false; } - - size_t c = 0; - for(auto v : arr) { c++; (void)v; } - if(c != 9) { cerr << "mismatched array size"<< endl; return false; } + size_t arr_count = 0; + for (auto v : arr) { arr_count++; (void)v; } + ASSERT_EQUAL(arr_count, 9); count++; } - if (count != 793) { cerr << "Expected 793 documents, but load_many loaded " << count << " documents." << endl; return false; } + ASSERT_EQUAL(count, AMAZON_CELLPHONES_NDJSON_DOC_COUNT); return true; } @@ -735,18 +705,16 @@ namespace parse_api_tests { if (error) { cerr << error << endl; return false; } dom::array arr; - error = doc.get(arr); // let us get the array - if (error) { cerr << error << endl; return false; } + ASSERT_SUCCESS( doc.get(arr) ); + ASSERT_EQUAL( arr.size(), 9 ); - if(arr.size() != 9) { cerr << "bad array size"<< endl; return false; } - - size_t c = 0; - for(auto v : arr) { c++; (void)v; } - if(c != 9) { cerr << "mismatched array size"<< endl; return false; } + size_t arr_count = 0; + for (auto v : arr) { arr_count++; (void)v; } + ASSERT_EQUAL( arr_count, 9 ); count++; } - if (count != 793) { cerr << "Expected 793 documents, but load_many loaded " << count << " documents." << endl; return false; } + ASSERT_EQUAL( count, AMAZON_CELLPHONES_NDJSON_DOC_COUNT ); return true; } SIMDJSON_POP_DISABLE_WARNINGS @@ -756,45 +724,39 @@ namespace parse_api_tests { bool parser_parse_exception() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; - element doc = parser.parse(BASIC_JSON); - if (!doc.is()) { cerr << "Document did not parse as an array" << endl; return false; } + UNUSED dom::array array = parser.parse(BASIC_JSON); return true; } bool parser_parse_many_exception() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; int count = 0; - for (const element doc : parser.parse_many(BASIC_NDJSON)) { - if (!doc.is()) { cerr << "Document did not parse as an array" << endl; return false; } + for (UNUSED dom::array doc : parser.parse_many(BASIC_NDJSON)) { count++; } - if (count != 2) { cerr << "parse_many returned " << count << " documents, expected 2" << endl; return false; } + ASSERT_EQUAL(count, 2); return true; } bool parser_load_exception() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; - const element doc = parser.load(TWITTER_JSON); - if (!doc.is()) { cerr << "Document did not parse as an object" << endl; return false; } - size_t c = 0; - dom::object obj = doc.get().value(); // let us get the object - for (auto x : obj) { - c++; - (void) x; + size_t count = 0; + dom::object object = parser.load(TWITTER_JSON); + for (UNUSED auto field : object) { + count++; } - if(c != obj.size()) { cerr << "Mismatched size" << endl; return false; } + ASSERT_EQUAL( count, object.size() ); return true; } bool parser_load_many_exception() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; int count = 0; - for (const element doc : parser.load_many(AMAZON_CELLPHONES_NDJSON)) { - if (!doc.is()) { cerr << "Document did not parse as an array" << endl; return false; } + for (UNUSED dom::array doc : parser.load_many(AMAZON_CELLPHONES_NDJSON)) { count++; } - if (count != 793) { cerr << "Expected 1 document, but load_many loaded " << count << " documents." << endl; return false; } + ASSERT_EQUAL( count, AMAZON_CELLPHONES_NDJSON_DOC_COUNT ); return true; } #endif @@ -933,16 +895,17 @@ namespace dom_api_tests { string json(R"({ "a": 1, "b": 2, "c": 3 })"); const char* expected_key[] = { "a", "b", "c" }; uint64_t expected_value[] = { 1, 2, 3 }; - int i = 0; dom::parser parser; - auto [object, error] = parser.parse(json).get(); - if (error) { cerr << "Error: " << error << endl; return false; } + dom::object object; + ASSERT_SUCCESS( parser.parse(json).get(object) ); + int i = 0; for (auto [key, value] : object) { - if (key != expected_key[i] || value.get().first != expected_value[i]) { cerr << "Expected " << expected_key[i] << " = " << expected_value[i] << ", got " << key << "=" << value << endl; return false; } + ASSERT_EQUAL( key, expected_key[i] ); + ASSERT_EQUAL( value.get().value(), expected_value[i] ); i++; } - if (i*sizeof(uint64_t) != sizeof(expected_value)) { cout << "Expected " << sizeof(expected_value) << " values, got " << i << endl; return false; } + ASSERT_EQUAL( i*sizeof(uint64_t), sizeof(expected_value) ); return true; } @@ -950,16 +913,18 @@ namespace dom_api_tests { std::cout << "Running " << __func__ << std::endl; string json(R"([ 1, 10, 100 ])"); uint64_t expected_value[] = { 1, 10, 100 }; - int i=0; dom::parser parser; - auto [array, error] = parser.parse(json).get(); - if (error) { cerr << "Error: " << error << endl; return false; } + dom::array array; + ASSERT_SUCCESS( parser.parse(json).get(array) ); + int i=0; for (auto value : array) { - if (value.get().first != expected_value[i]) { cerr << "Expected " << expected_value[i] << ", got " << value << endl; return false; } + uint64_t v; + ASSERT_SUCCESS( value.get(v) ); + ASSERT_EQUAL( v, expected_value[i] ); i++; } - if (i*sizeof(uint64_t) != sizeof(expected_value)) { cout << "Expected " << sizeof(expected_value) << " values, got " << i << endl; return false; } + ASSERT_EQUAL( i*sizeof(uint64_t), sizeof(expected_value) ); return true; } @@ -969,13 +934,13 @@ namespace dom_api_tests { int i = 0; dom::parser parser; - auto [object, error] = parser.parse(json).get(); - if (error) { cerr << "Error: " << error << endl; return false; } - for (auto [key, value] : object) { - cout << "Unexpected " << key << " = " << value << endl; + dom::object object; + ASSERT_SUCCESS( parser.parse(json).get(object) ); + for (UNUSED auto field : object) { + TEST_FAIL("Unexpected field"); i++; } - if (i > 0) { cout << "Expected 0 values, got " << i << endl; return false; } + ASSERT_EQUAL(i, 0); return true; } @@ -985,13 +950,13 @@ namespace dom_api_tests { int i=0; dom::parser parser; - auto [array, error] = parser.parse(json).get(); - if (error) { cerr << "Error: " << error << endl; return false; } - for (auto value : array) { - cout << "Unexpected value " << value << endl; + dom::array array; + ASSERT_SUCCESS( parser.parse(json).get(array) ); + for (UNUSED auto value : array) { + TEST_FAIL("Unexpected value"); i++; } - if (i > 0) { cout << "Expected 0 values, got " << i << endl; return false; } + ASSERT_EQUAL(i, 0); return true; } @@ -999,13 +964,18 @@ namespace dom_api_tests { std::cout << "Running " << __func__ << std::endl; string json(R"([ "hi", "has backslash\\" ])"); dom::parser parser; - auto [array, error] = parser.parse(json).get(); - if (error) { cerr << "Error: " << error << endl; return false; } - auto val = array.begin(); + dom::array array; + ASSERT_SUCCESS( parser.parse(json).get(array) ); + + auto iter = array.begin(); + std::string_view val; + ASSERT_SUCCESS( (*iter).get(val) ); + ASSERT_EQUAL( val, "hi" ); + + ++iter; + ASSERT_SUCCESS( (*iter).get(val) ); + ASSERT_EQUAL( val, "has backslash\\" ); - if ((*val).get().first != "hi") { cerr << "Expected value to be \"hi\", was " << (*val).get().first << endl; return false; } - ++val; - if ((*val).get().first != "has backslash\\") { cerr << "Expected string_view(\"has backslash\\\\\") to be \"has backslash\\\", was " << (*val).get().first << endl; return false; } return true; } @@ -1013,22 +983,22 @@ namespace dom_api_tests { std::cout << "Running " << __func__ << std::endl; string json(R"([ 0, 1, -1, 1.1 ])"); dom::parser parser; - auto [array, error] = parser.parse(json).get(); - if (error) { cerr << "Error: " << error << endl; return false; } - auto val = array.begin(); + dom::array array; + ASSERT_SUCCESS( parser.parse(json).get(array) ); - if ((*val).get().first != 0) { cerr << "Expected uint64_t(0) to be 0, was " << (*val) << endl; return false; } - if ((*val).get().first != 0) { cerr << "Expected int64_t(0) to be 0, was " << (*val).get().first << endl; return false; } - if ((*val).get().first != 0) { cerr << "Expected double(0) to be 0, was " << (*val).get().first << endl; return false; } - ++val; - if ((*val).get().first != 1) { cerr << "Expected uint64_t(1) to be 1, was " << (*val) << endl; return false; } - if ((*val).get().first != 1) { cerr << "Expected int64_t(1) to be 1, was " << (*val).get().first << endl; return false; } - if ((*val).get().first != 1) { cerr << "Expected double(1) to be 1, was " << (*val).get().first << endl; return false; } - ++val; - if ((*val).get().first != -1) { cerr << "Expected int64_t(-1) to be -1, was " << (*val).get().first << endl; return false; } - if ((*val).get().first != -1) { cerr << "Expected double(-1) to be -1, was " << (*val).get().first << endl; return false; } - ++val; - if ((*val).get().first != 1.1) { cerr << "Expected double(1.1) to be 1.1, was " << (*val).get().first << endl; return false; } + auto iter = array.begin(); + ASSERT_EQUAL( (*iter).get().value(), 0 ); + ASSERT_EQUAL( (*iter).get().value(), 0 ); + ASSERT_EQUAL( (*iter).get().value(), 0 ); + ++iter; + ASSERT_EQUAL( (*iter).get().value(), 1 ); + ASSERT_EQUAL( (*iter).get().value(), 1 ); + ASSERT_EQUAL( (*iter).get().value(), 1 ); + ++iter; + ASSERT_EQUAL( (*iter).get().value(), -1 ); + ASSERT_EQUAL( (*iter).get().value(), -1 ); + ++iter; + ASSERT_EQUAL( (*iter).get().value(), 1.1 ); return true; } @@ -1036,13 +1006,13 @@ namespace dom_api_tests { std::cout << "Running " << __func__ << std::endl; string json(R"([ true, false ])"); dom::parser parser; - auto [array, error] = parser.parse(json).get(); - if (error) { cerr << "Error: " << error << endl; return false; } - auto val = array.begin(); + dom::array array; + ASSERT_SUCCESS( parser.parse(json).get(array) ); - if ((*val).get().first != true) { cerr << "Expected bool(true) to be true, was " << (*val) << endl; return false; } + auto val = array.begin(); + ASSERT_EQUAL( (*val).get().first, true ); ++val; - if ((*val).get().first != false) { cerr << "Expected bool(false) to be false, was " << (*val) << endl; return false; } + ASSERT_EQUAL( (*val).get().first, false ); return true; } @@ -1050,10 +1020,11 @@ namespace dom_api_tests { std::cout << "Running " << __func__ << std::endl; string json(R"([ null ])"); dom::parser parser; - auto [array, error] = parser.parse(json).get(); - if (error) { cerr << "Error: " << error << endl; return false; } + dom::array array; + ASSERT_SUCCESS( parser.parse(json).get(array) ); + auto val = array.begin(); - if (!(*val).is_null()) { cerr << "Expected null to be null!" << endl; return false; } + ASSERT_EQUAL( !(*val).is_null(), 0 ); return true; } @@ -1061,27 +1032,29 @@ namespace dom_api_tests { std::cout << "Running " << __func__ << std::endl; string json(R"({ "a": 1, "b": 2, "c/d": 3})"); dom::parser parser; - auto [doc, error] = parser.parse(json); - if (doc["a"].get().first != 1) { cerr << "Expected uint64_t(doc[\"a\"]) to be 1, was " << doc["a"].first << endl; return false; } - if (doc["b"].get().first != 2) { cerr << "Expected uint64_t(doc[\"b\"]) to be 2, was " << doc["b"].first << endl; return false; } - if (doc["c/d"].get().first != 3) { cerr << "Expected uint64_t(doc[\"c/d\"]) to be 3, was " << doc["c"].first << endl; return false; } + dom::object object; + ASSERT_SUCCESS( parser.parse(json).get(object) ); + ASSERT_EQUAL( object["a"].get().first, 1 ); + ASSERT_EQUAL( object["b"].get().first, 2 ); + ASSERT_EQUAL( object["c/d"].get().first, 3 ); // Check all three again in backwards order, to ensure we can go backwards - if (doc["c/d"].get().first != 3) { cerr << "Expected uint64_t(doc[\"c/d\"]) to be 3, was " << doc["c"].first << endl; return false; } - if (doc["b"].get().first != 2) { cerr << "Expected uint64_t(doc[\"b\"]) to be 2, was " << doc["b"].first << endl; return false; } - if (doc["a"].get().first != 1) { cerr << "Expected uint64_t(doc[\"a\"]) to be 1, was " << doc["a"].first << endl; return false; } + ASSERT_EQUAL( object["c/d"].get().first, 3 ); + ASSERT_EQUAL( object["b"].get().first, 2 ); + ASSERT_EQUAL( object["a"].get().first, 1 ); + simdjson::error_code error; UNUSED element val; #ifndef _LIBCPP_VERSION // should work everywhere but with libc++, must include the header. - std::tie(val,error) = doc["d"]; - if (error != simdjson::NO_SUCH_FIELD) { cerr << "Expected NO_SUCH_FIELD error for uint64_t(doc[\"d\"]), got " << error << endl; return false; } - std::tie(std::ignore,error) = doc["d"]; - if (error != simdjson::NO_SUCH_FIELD) { cerr << "Expected NO_SUCH_FIELD error for uint64_t(doc[\"d\"]), got " << error << endl; return false; } + std::tie(val,error) = object["d"]; + ASSERT_ERROR( error, NO_SUCH_FIELD ); + std::tie(std::ignore,error) = object["d"]; + ASSERT_ERROR( error, NO_SUCH_FIELD ); #endif - // tie(val, error) = doc["d"]; fails with "no viable overloaded '='" on Apple clang version 11.0.0 tie(val, error) = doc["d"]; - doc["d"].tie(val, error); - if (error != simdjson::NO_SUCH_FIELD) { cerr << "Expected NO_SUCH_FIELD error for uint64_t(doc[\"d\"]), got " << error << endl; return false; } - if (doc["d"].get(val) != simdjson::NO_SUCH_FIELD) { cerr << "Expected NO_SUCH_FIELD error for uint64_t(doc[\"d\"]), got " << error << endl; return false; } - if (doc["d"].error() != simdjson::NO_SUCH_FIELD) { cerr << "Expected NO_SUCH_FIELD error for uint64_t(doc[\"d\"]), got " << error << endl; return false; } + // tie(val, error) = object["d"]; fails with "no viable overloaded '='" on Apple clang version 11.0.0 tie(val, error) = doc["d"]; + object["d"].tie(val, error); + ASSERT_ERROR( error, NO_SUCH_FIELD ); + ASSERT_ERROR( object["d"].get(val), NO_SUCH_FIELD ); + ASSERT_ERROR( object["d"].error(), NO_SUCH_FIELD ); return true; } @@ -1089,26 +1062,25 @@ namespace dom_api_tests { std::cout << "Running " << __func__ << std::endl; string json(R"({ "obj": { "a": 1, "b": 2, "c/d": 3 } })"); dom::parser parser; - auto [doc, error] = parser.parse(json); - if (error) { cerr << "Error: " << error << endl; return false; } - if (doc["obj"]["a"].get().first != 1) { cerr << "Expected uint64_t(doc[\"obj\"][\"a\"]) to be 1, was " << doc["obj"]["a"].first << endl; return false; } + dom::element doc; + ASSERT_SUCCESS( parser.parse(json).get(doc) ); + ASSERT_EQUAL( doc["obj"]["a"].get().first, 1); object obj; - error = doc.get(obj); - if (error) { cerr << "Error: " << error << endl; return false; } - if (obj["obj"]["a"].get().first != 1) { cerr << "Expected uint64_t(doc[\"obj\"][\"a\"]) to be 1, was " << doc["obj"]["a"].first << endl; return false; } + ASSERT_SUCCESS( doc.get(obj) ); + ASSERT_EQUAL( obj["obj"]["a"].get().first, 1); - error = obj["obj"].get(obj); - if (obj["a"].get().first != 1) { cerr << "Expected uint64_t(obj[\"a\"]) to be 1, was " << obj["a"].first << endl; return false; } - if (obj["b"].get().first != 2) { cerr << "Expected uint64_t(obj[\"b\"]) to be 2, was " << obj["b"].first << endl; return false; } - if (obj["c/d"].get().first != 3) { cerr << "Expected uint64_t(obj[\"c\"]) to be 3, was " << obj["c"].first << endl; return false; } + ASSERT_SUCCESS( obj["obj"].get(obj) ); + ASSERT_EQUAL( obj["a"].get().first, 1 ); + ASSERT_EQUAL( obj["b"].get().first, 2 ); + ASSERT_EQUAL( obj["c/d"].get().first, 3 ); // Check all three again in backwards order, to ensure we can go backwards - if (obj["c/d"].get().first != 3) { cerr << "Expected uint64_t(obj[\"c\"]) to be 3, was " << obj["c"].first << endl; return false; } - if (obj["b"].get().first != 2) { cerr << "Expected uint64_t(obj[\"b\"]) to be 2, was " << obj["b"].first << endl; return false; } - if (obj["a"].get().first != 1) { cerr << "Expected uint64_t(obj[\"a\"]) to be 1, was " << obj["a"].first << endl; return false; } + ASSERT_EQUAL( obj["c/d"].get().first, 3 ); + ASSERT_EQUAL( obj["b"].get().first, 2 ); + ASSERT_EQUAL( obj["a"].get().first, 1 ); UNUSED element val; - if (doc["d"].get(val) != simdjson::NO_SUCH_FIELD) { cerr << "Expected NO_SUCH_FIELD error for uint64_t(obj[\"d\"]), got " << error << endl; return false; } + ASSERT_ERROR( doc["d"].get(val), NO_SUCH_FIELD); return true; } @@ -1116,9 +1088,9 @@ namespace dom_api_tests { std::cout << "Running " << __func__ << std::endl; // Prints the number of results in twitter.json dom::parser parser; - auto [result_count, error] = parser.load(TWITTER_JSON)["search_metadata"]["count"].get(); - if (error) { cerr << "Error: " << error << endl; return false; } - if (result_count != 100) { cerr << "Expected twitter.json[metadata_count][count] = 100, got " << result_count << endl; return false; } + uint64_t result_count; + ASSERT_SUCCESS( parser.load(TWITTER_JSON)["search_metadata"]["count"].get(result_count) ); + ASSERT_EQUAL( result_count, 100 ); return true; } @@ -1128,20 +1100,19 @@ namespace dom_api_tests { set default_users; dom::parser parser; dom::array tweets; - auto error = parser.load(TWITTER_JSON)["statuses"].get(tweets); - if (error) { cerr << "Error: " << error << endl; return false; } + ASSERT_SUCCESS( parser.load(TWITTER_JSON)["statuses"].get(tweets) ); for (auto tweet : tweets) { object user; - if ((error = tweet["user"].get(user))) { cerr << "Error: " << error << endl; return false; } + ASSERT_SUCCESS( tweet["user"].get(user) ); bool default_profile; - if ((error = user["default_profile"].get(default_profile))) { cerr << "Error: " << error << endl; return false; } + ASSERT_SUCCESS( user["default_profile"].get(default_profile) ); if (default_profile) { std::string_view screen_name; - if ((error = user["screen_name"].get(screen_name))) { cerr << "Error: " << error << endl; return false; } + ASSERT_SUCCESS( user["screen_name"].get(screen_name) ); default_users.insert(screen_name); } } - if (default_users.size() != 86) { cerr << "Expected twitter.json[statuses][user] to contain 86 default_profile users, got " << default_users.size() << endl; return false; } + ASSERT_EQUAL( default_users.size(), 86 ); return true; } @@ -1149,26 +1120,26 @@ namespace dom_api_tests { std::cout << "Running " << __func__ << std::endl; // Print image names and sizes set> image_sizes; + simdjson::error_code error; dom::parser parser; dom::array tweets; - auto error = parser.load(TWITTER_JSON)["statuses"].get(tweets); - if (error) { cerr << "Error: " << error << endl; return false; } + ASSERT_SUCCESS( parser.load(TWITTER_JSON)["statuses"].get(tweets) ); for (auto tweet : tweets) { dom::array media; if (not (error = tweet["entities"]["media"].get(media))) { for (auto image : media) { object sizes; - if ((error = image["sizes"].get(sizes))) { cerr << "Error: " << error << endl; return false; } + ASSERT_SUCCESS( image["sizes"].get(sizes) ); for (auto size : sizes) { uint64_t width, height; - if ((error = size.value["w"].get(width))) { cerr << "Error: " << error << endl; return false; } - if ((error = size.value["h"].get(height))) { cerr << "Error: " << error << endl; return false; } + ASSERT_SUCCESS( size.value["w"].get(width) ); + ASSERT_SUCCESS( size.value["h"].get(height) ); image_sizes.insert(make_pair(width, height)); } } } } - if (image_sizes.size() != 15) { cerr << "Expected twitter.json[statuses][entities][media][sizes] to contain 15 different sizes, got " << image_sizes.size() << endl; return false; } + ASSERT_EQUAL( image_sizes.size(), 15 ); return true; } @@ -1182,12 +1153,12 @@ namespace dom_api_tests { int i = 0; dom::parser parser; - element doc = parser.parse(json); - for (auto [key, value] : doc.get()) { - if (key != expected_key[i] || uint64_t(value) != expected_value[i]) { cerr << "Expected " << expected_key[i] << " = " << expected_value[i] << ", got " << key << "=" << uint64_t(value) << endl; return false; } + for (auto [key, value] : dom::object(parser.parse(json))) { + ASSERT_EQUAL( key, expected_key[i]); + ASSERT_EQUAL( uint64_t(value), expected_value[i] ); i++; } - if (i*sizeof(uint64_t) != sizeof(expected_value)) { cout << "Expected " << sizeof(expected_value) << " values, got " << i << endl; return false; } + ASSERT_EQUAL( i*sizeof(uint64_t), sizeof(expected_value) ); return true; } @@ -1198,69 +1169,61 @@ namespace dom_api_tests { int i=0; dom::parser parser; - element doc = parser.parse(json); - for (uint64_t value : doc.get()) { - if (value != expected_value[i]) { cerr << "Expected " << expected_value[i] << ", got " << value << endl; return false; } + for (uint64_t value : parser.parse(json)) { + ASSERT_EQUAL( value, expected_value[i] ); i++; } - if (i*sizeof(uint64_t) != sizeof(expected_value)) { cout << "Expected " << sizeof(expected_value) << " values, got " << i << endl; return false; } + ASSERT_EQUAL( i*sizeof(uint64_t), sizeof(expected_value) ); return true; } bool string_value_exception() { std::cout << "Running " << __func__ << std::endl; - string json(R"([ "hi", "has backslash\\" ])"); dom::parser parser; - auto val = parser.parse(json).get().begin(); - - if (strcmp((const char*)*val, "hi")) { cerr << "Expected const char*(\"hi\") to be \"hi\", was " << (const char*)*val << endl; return false; } - if (string_view(*val) != "hi") { cerr << "Expected string_view(\"hi\") to be \"hi\", was " << string_view(*val) << endl; return false; } - ++val; - if (strcmp((const char*)*val, "has backslash\\")) { cerr << "Expected const char*(\"has backslash\\\\\") to be \"has backslash\\\", was " << (const char*)*val << endl; return false; } - if (string_view(*val) != "has backslash\\") { cerr << "Expected string_view(\"has backslash\\\\\") to be \"has backslash\\\", was " << string_view(*val) << endl; return false; } + ASSERT_EQUAL( (const char *)parser.parse(R"("hi")"_padded), "hi" ); + ASSERT_EQUAL( string_view(parser.parse(R"("hi")"_padded)), "hi" ); + ASSERT_EQUAL( (const char *)parser.parse(R"("has backslash\\")"_padded), "has backslash\\"); + ASSERT_EQUAL( string_view(parser.parse(R"("has backslash\\")"_padded)), "has backslash\\" ); return true; } bool numeric_values_exception() { std::cout << "Running " << __func__ << std::endl; - string json(R"([ 0, 1, -1, 1.1 ])"); dom::parser parser; - auto val = parser.parse(json).get().begin(); - if (uint64_t(*val) != 0) { cerr << "Expected uint64_t(0) to be 0, was " << uint64_t(*val) << endl; return false; } - if (int64_t(*val) != 0) { cerr << "Expected int64_t(0) to be 0, was " << int64_t(*val) << endl; return false; } - if (double(*val) != 0) { cerr << "Expected double(0) to be 0, was " << double(*val) << endl; return false; } - ++val; - if (uint64_t(*val) != 1) { cerr << "Expected uint64_t(1) to be 1, was " << uint64_t(*val) << endl; return false; } - if (int64_t(*val) != 1) { cerr << "Expected int64_t(1) to be 1, was " << int64_t(*val) << endl; return false; } - if (double(*val) != 1) { cerr << "Expected double(1) to be 1, was " << double(*val) << endl; return false; } - ++val; - if (int64_t(*val) != -1) { cerr << "Expected int64_t(-1) to be -1, was " << int64_t(*val) << endl; return false; } - if (double(*val) != -1) { cerr << "Expected double(-1) to be -1, was " << double(*val) << endl; return false; } - ++val; - if (double(*val) != 1.1) { cerr << "Expected double(1.1) to be 1.1, was " << double(*val) << endl; return false; } + ASSERT_EQUAL( uint64_t(parser.parse("0"_padded)), 0); + ASSERT_EQUAL( int64_t(parser.parse("0"_padded)), 0); + ASSERT_EQUAL( double(parser.parse("0"_padded)), 0); + + ASSERT_EQUAL( uint64_t(parser.parse("1"_padded)), 1); + ASSERT_EQUAL( int64_t(parser.parse("1"_padded)), 1); + ASSERT_EQUAL( double(parser.parse("1"_padded)), 1); + + ASSERT_EQUAL( int64_t(parser.parse("-1"_padded)), -1); + ASSERT_EQUAL( double(parser.parse("-1"_padded)), -1); + + ASSERT_EQUAL( double(parser.parse("1.1"_padded)), 1.1); + return true; } bool boolean_values_exception() { std::cout << "Running " << __func__ << std::endl; - string json(R"([ true, false ])"); dom::parser parser; - auto val = parser.parse(json).get().begin(); - if (bool(*val) != true) { cerr << "Expected bool(true) to be true, was " << bool(*val) << endl; return false; } - ++val; - if (bool(*val) != false) { cerr << "Expected bool(false) to be false, was " << bool(*val) << endl; return false; } + ASSERT_EQUAL( bool(parser.parse("true"_padded)), true); + + ASSERT_EQUAL( bool(parser.parse("false"_padded)), false); + return true; } bool null_value_exception() { std::cout << "Running " << __func__ << std::endl; - string json(R"([ null ])"); dom::parser parser; - auto val = parser.parse(json).get().begin(); - if (!(*val).is_null()) { cerr << "Expected null to be null!" << endl; return false; } + ASSERT_EQUAL( bool(parser.parse("null"_padded).is_null()), true ); + return true; } @@ -1268,8 +1231,10 @@ namespace dom_api_tests { std::cout << "Running " << __func__ << std::endl; string json(R"({ "a": 1, "b": 2, "c": 3})"); dom::parser parser; - element doc = parser.parse(json); - if (uint64_t(doc["a"]) != 1) { cerr << "Expected uint64_t(doc[\"a\"]) to be 1, was " << uint64_t(doc["a"]) << endl; return false; } + auto obj = parser.parse(json); + + ASSERT_EQUAL(uint64_t(obj["a"]), 1); + return true; } @@ -1278,7 +1243,9 @@ namespace dom_api_tests { string json(R"({ "obj": { "a": 1, "b": 2, "c": 3 } })"); dom::parser parser; object obj = parser.parse(json)["obj"]; - if (uint64_t(obj["a"]) != 1) { cerr << "Expected uint64_t(doc[\"a\"]) to be 1, was " << uint64_t(obj["a"]) << endl; return false; } + + ASSERT_EQUAL( uint64_t(obj["a"]), 1); + return true; } @@ -1313,18 +1280,17 @@ namespace dom_api_tests { // Print image names and sizes set> image_sizes; dom::parser parser; - element doc = parser.load(TWITTER_JSON); - for (object tweet : doc["statuses"].get()) { - auto [media, not_found] = tweet["entities"]["media"]; - if (!not_found) { - for (object image : media.get()) { - for (auto size : image["sizes"].get()) { + for (object tweet : parser.load(TWITTER_JSON)["statuses"]) { + auto media = tweet["entities"]["media"]; + if (!media.error()) { + for (object image : media) { + for (auto size : object(image["sizes"])) { image_sizes.insert(make_pair(size.value["w"], size.value["h"])); } } } } - if (image_sizes.size() != 15) { cerr << "Expected twitter.json[statuses][entities][media][sizes] to contain 15 different sizes, got " << image_sizes.size() << endl; return false; } + ASSERT_EQUAL( image_sizes.size(), 15 ); return true; } @@ -1721,21 +1687,10 @@ namespace minify_tests { return false; } size_t newlength{}; - auto error = simdjson::minify(input, length, buffer.get(), newlength); - if(error != simdjson::SUCCESS) { - std::cerr << "error " << error << std::endl; - return false; - } - // memcmp - if(newlength != expected_length) { - std::cerr << "lengths do not match " << std::endl; - return false; - } + ASSERT_SUCCESS( simdjson::minify(input, length, buffer.get(), newlength) ); + ASSERT_EQUAL( newlength, expected_length); for(size_t i = 0; i < newlength; i++) { - if(buffer.get()[i] != expected[i]) { - std::cerr << "Inputs do not match (but same length) " << std::endl; - return false; - } + ASSERT_EQUAL( buffer.get()[i], expected[i]); } return true; } @@ -1785,8 +1740,8 @@ namespace format_tests { bool print_parser_parse() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; - auto [doc, error] = parser.parse(DOCUMENT); - if (error) { cerr << error << endl; return false; } + dom::element doc; + ASSERT_SUCCESS( parser.parse(DOCUMENT).get(doc) ); ostringstream s; s << doc; return assert_minified(s); @@ -1794,8 +1749,8 @@ namespace format_tests { bool print_minify_parser_parse() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; - auto [doc, error] = parser.parse(DOCUMENT); - if (error) { cerr << error << endl; return false; } + dom::element doc; + ASSERT_SUCCESS( parser.parse(DOCUMENT).get(doc) ); ostringstream s; s << minify(doc); return assert_minified(s); @@ -1804,8 +1759,8 @@ namespace format_tests { bool print_element() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; - auto [value, error] = parser.parse(DOCUMENT)["foo"]; - if (error) { cerr << error << endl; return false; } + dom::element value; + ASSERT_SUCCESS( parser.parse(DOCUMENT)["foo"].get(value) ); ostringstream s; s << value; return assert_minified(s, "1"); @@ -1813,8 +1768,8 @@ namespace format_tests { bool print_minify_element() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; - auto [value, error] = parser.parse(DOCUMENT)["foo"]; - if (error) { cerr << error << endl; return false; } + dom::element value; + ASSERT_SUCCESS( parser.parse(DOCUMENT)["foo"].get(value) ); ostringstream s; s << minify(value); return assert_minified(s, "1"); @@ -1823,38 +1778,38 @@ namespace format_tests { bool print_array() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; - auto [value, error] = parser.parse(DOCUMENT)["bar"].get(); - if (error) { cerr << error << endl; return false; } + dom::array array; + ASSERT_SUCCESS( parser.parse(DOCUMENT)["bar"].get(array) ); ostringstream s; - s << value; + s << array; return assert_minified(s, "[1,2,3]"); } bool print_minify_array() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; - auto [value, error] = parser.parse(DOCUMENT)["bar"].get(); - if (error) { cerr << error << endl; return false; } + dom::array array; + ASSERT_SUCCESS( parser.parse(DOCUMENT)["bar"].get(array) ); ostringstream s; - s << minify(value); + s << minify(array); return assert_minified(s, "[1,2,3]"); } bool print_object() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; - auto [value, error] = parser.parse(DOCUMENT)["baz"].get(); - if (error) { cerr << error << endl; return false; } + dom::object object; + ASSERT_SUCCESS( parser.parse(DOCUMENT)["baz"].get(object) ); ostringstream s; - s << value; + s << object; return assert_minified(s, R"({"a":1,"b":2,"c":3})"); } bool print_minify_object() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; - auto [value, error] = parser.parse(DOCUMENT)["baz"].get(); - if (error) { cerr << error << endl; return false; } + dom::object object; + ASSERT_SUCCESS( parser.parse(DOCUMENT)["baz"].get(object) ); ostringstream s; - s << minify(value); + s << minify(object); return assert_minified(s, R"({"a":1,"b":2,"c":3})"); } @@ -1878,25 +1833,22 @@ namespace format_tests { bool print_element_result_exception() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; - element doc = parser.parse(DOCUMENT); ostringstream s; - s << doc["foo"]; + s << parser.parse(DOCUMENT)["foo"]; return assert_minified(s, "1"); } bool print_minify_element_result_exception() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; - element doc = parser.parse(DOCUMENT); ostringstream s; - s << minify(doc["foo"]); + s << minify(parser.parse(DOCUMENT)["foo"]); return assert_minified(s, "1"); } bool print_element_exception() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; - element doc = parser.parse(DOCUMENT); - element value = doc["foo"]; + element value = parser.parse(DOCUMENT)["foo"]; ostringstream s; s << value; return assert_minified(s, "1"); @@ -1904,8 +1856,7 @@ namespace format_tests { bool print_minify_element_exception() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; - element doc = parser.parse(DOCUMENT); - element value = doc["foo"]; + element value = parser.parse(DOCUMENT)["foo"]; ostringstream s; s << minify(value); return assert_minified(s, "1"); @@ -1914,66 +1865,64 @@ namespace format_tests { bool print_array_result_exception() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; - element doc = parser.parse(DOCUMENT); ostringstream s; - s << doc["bar"].get(); + s << parser.parse(DOCUMENT)["bar"].get(); return assert_minified(s, "[1,2,3]"); } bool print_minify_array_result_exception() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; - element doc = parser.parse(DOCUMENT); ostringstream s; - s << minify(doc["bar"].get()); + s << minify(parser.parse(DOCUMENT)["bar"].get()); return assert_minified(s, "[1,2,3]"); } bool print_object_result_exception() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; - element doc = parser.parse(DOCUMENT); ostringstream s; - s << doc["baz"].get(); + s << parser.parse(DOCUMENT)["baz"].get(); return assert_minified(s, R"({"a":1,"b":2,"c":3})"); } bool print_minify_object_result_exception() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; - element doc = parser.parse(DOCUMENT); ostringstream s; - s << minify(doc["baz"].get()); + s << minify(parser.parse(DOCUMENT)["baz"].get()); return assert_minified(s, R"({"a":1,"b":2,"c":3})"); } bool print_array_exception() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; + dom::array array = parser.parse(DOCUMENT)["bar"]; ostringstream s; - s << parser.parse(DOCUMENT)["bar"]; + s << array; return assert_minified(s, "[1,2,3]"); } bool print_minify_array_exception() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; + dom::array array = parser.parse(DOCUMENT)["bar"]; ostringstream s; - s << minify(parser.parse(DOCUMENT)["bar"]); + s << minify(array); return assert_minified(s, "[1,2,3]"); } bool print_object_exception() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; + dom::object object = parser.parse(DOCUMENT)["baz"]; ostringstream s; - s << parser.parse(DOCUMENT)["baz"]; + s << object; return assert_minified(s, R"({"a":1,"b":2,"c":3})"); } bool print_minify_object_exception() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; - element doc = parser.parse(DOCUMENT); - object value = doc["baz"]; + dom::object object = parser.parse(DOCUMENT)["baz"]; ostringstream s; - s << minify(value); + s << minify(object); return assert_minified(s, R"({"a":1,"b":2,"c":3})"); } #endif // SIMDJSON_EXCEPTIONS diff --git a/tests/errortests.cpp b/tests/errortests.cpp index 3be94fcd0..f2050d5af 100644 --- a/tests/errortests.cpp +++ b/tests/errortests.cpp @@ -46,15 +46,16 @@ namespace parser_load { ASSERT_SUCCESS(parser.parse_many(DOC).get(docs)); for (auto doc : docs) { count++; - auto [val, error] = doc.get(); + uint64_t val; + auto error = doc.get(val); if (count == 3) { ASSERT_ERROR(error, TAPE_ERROR); } else { - if (error) { TEST_FAIL(error); } - if (val != count) { cerr << "FAIL: expected " << count << ", got " << val << endl; return false; } + ASSERT_SUCCESS(error); + ASSERT_EQUAL(val, count); } } - if (count != 3) { cerr << "FAIL: expected 2 documents and 1 error, got " << count << " total things" << endl; return false; } + ASSERT_EQUAL(count, 3); TEST_SUCCEED(); } @@ -69,7 +70,7 @@ namespace parser_load { count++; ASSERT_ERROR(doc.error(), TAPE_ERROR); } - if (count != 1) { cerr << "FAIL: expected no documents and 1 error, got " << count << " total things" << endl; return false; } + ASSERT_EQUAL(count, 1); TEST_SUCCEED(); } @@ -82,36 +83,34 @@ namespace parser_load { ASSERT_SUCCESS(parser.parse_many(DOC).get(docs)); for (auto doc : docs) { count++; - auto [val, error] = doc.get(); + uint64_t val; + auto error = doc.get(val); if (count == 3) { ASSERT_ERROR(error, TAPE_ERROR); } else { - if (error) { TEST_FAIL(error); } - if (val != count) { cerr << "FAIL: expected " << count << ", got " << val << endl; return false; } + ASSERT_SUCCESS(error); + ASSERT_EQUAL(val, count); } } - if (count != 3) { cerr << "FAIL: expected 2 documents and 1 error, got " << count << " total things" << endl; return false; } + ASSERT_EQUAL(count, 3); TEST_SUCCEED(); } bool parser_load_nonexistent() { TEST_START(); dom::parser parser; - auto error = parser.load(NONEXISTENT_FILE).error(); - ASSERT_ERROR(error, IO_ERROR); + ASSERT_ERROR( parser.load(NONEXISTENT_FILE).error(), IO_ERROR ); TEST_SUCCEED(); } bool parser_load_many_nonexistent() { TEST_START(); dom::parser parser; - dom::document_stream stream; - ASSERT_ERROR(parser.load_many(NONEXISTENT_FILE).get(stream), IO_ERROR); + ASSERT_ERROR( parser.load_many(NONEXISTENT_FILE).error(), IO_ERROR ); TEST_SUCCEED(); } bool padded_string_load_nonexistent() { TEST_START(); - auto error = padded_string::load(NONEXISTENT_FILE).error(); - ASSERT_ERROR(error, IO_ERROR); + ASSERT_ERROR(padded_string::load(NONEXISTENT_FILE).error(), IO_ERROR); TEST_SUCCEED(); } @@ -119,16 +118,17 @@ namespace parser_load { TEST_START(); dom::parser parser; UNUSED uint64_t foo; - ASSERT_ERROR( parser.load(NONEXISTENT_FILE)["foo"].get(foo) , IO_ERROR); + ASSERT_ERROR( parser.load(NONEXISTENT_FILE)["foo"].get(foo), IO_ERROR); TEST_SUCCEED(); } bool parser_load_many_chain() { TEST_START(); dom::parser parser; - dom::document_stream stream; - ASSERT_ERROR( parser.load_many(NONEXISTENT_FILE).get(stream) , IO_ERROR ); + UNUSED dom::document_stream stream; + ASSERT_ERROR( parser.load_many(NONEXISTENT_FILE).get(stream), IO_ERROR ); TEST_SUCCEED(); } + bool run() { return true && parser_load_capacity() diff --git a/tests/integer_tests.cpp b/tests/integer_tests.cpp index c072ac05f..c35d717c8 100644 --- a/tests/integer_tests.cpp +++ b/tests/integer_tests.cpp @@ -3,6 +3,7 @@ #include #include "simdjson.h" +#include "test_macros.h" // we define our own asserts to get around NDEBUG #ifndef ASSERT @@ -29,44 +30,43 @@ template static const std::string make_json(T value) { } template -static void parse_and_validate(const std::string src, T expected) { +static bool parse_and_validate(const std::string src, T expected) { std::cout << "src: " << src << ", "; const padded_string pstr{src}; simdjson::dom::parser parser; - bool result; if constexpr (std::is_same::value) { - auto [actual, error] = parser.parse(pstr).get()["key"].get(); - if (error) { std::cerr << error << std::endl; abort(); } - result = (expected == actual); + int64_t actual; + ASSERT_SUCCESS( parser.parse(pstr)["key"].get(actual) ); + std::cout << std::boolalpha << "test: " << (expected == actual) << std::endl; + ASSERT_EQUAL( expected, actual ); } else { - auto [actual, error] = parser.parse(pstr).get()["key"].get(); - if (error) { std::cerr << error << std::endl; abort(); } - result = (expected == actual); - } - std::cout << std::boolalpha << "test: " << result << std::endl; - if(!result) { - std::cerr << "bug detected" << std::endl; - exit(EXIT_FAILURE); + uint64_t actual; + ASSERT_SUCCESS( parser.parse(pstr)["key"].get(actual) ); + std::cout << std::boolalpha << "test: " << (expected == actual) << std::endl; + ASSERT_EQUAL( expected, actual ); } + return true; } static bool parse_and_check_signed(const std::string src) { std::cout << "src: " << src << ", expecting signed" << std::endl; const padded_string pstr{src}; simdjson::dom::parser parser; - auto [value, error] = parser.parse(pstr).get()["key"]; - if (error) { std::cerr << error << std::endl; abort(); } - return value.is(); + simdjson::dom::element value; + ASSERT_SUCCESS( parser.parse(pstr).get()["key"].get(value) ); + ASSERT_EQUAL( value.is(), true ); + return true; } static bool parse_and_check_unsigned(const std::string src) { std::cout << "src: " << src << ", expecting signed" << std::endl; const padded_string pstr{src}; simdjson::dom::parser parser; - auto [value, error] = parser.parse(pstr).get()["key"]; - if (error) { std::cerr << error << std::endl; abort(); } - return value.is(); + simdjson::dom::element value; + ASSERT_SUCCESS( parser.parse(pstr).get()["key"].get(value) ); + ASSERT_EQUAL( value.is(), true ); + return true; } int main() { @@ -75,21 +75,21 @@ int main() { constexpr auto int64_min = numeric_limits::lowest(); constexpr auto uint64_max = numeric_limits::max(); constexpr auto uint64_min = numeric_limits::lowest(); - parse_and_validate(make_json(int64_max), int64_max); - parse_and_validate(make_json(int64_min), int64_min); - parse_and_validate(make_json(uint64_max), uint64_max); - parse_and_validate(make_json(uint64_min), uint64_min); constexpr auto int64_max_plus1 = static_cast(int64_max) + 1; - parse_and_validate(make_json(int64_max_plus1), int64_max_plus1); - if(!parse_and_check_signed(make_json(int64_max))) { - std::cerr << "bug: large signed integers should be represented as signed integers" << std::endl; - return EXIT_FAILURE; + if (true + && parse_and_validate(make_json(int64_max), int64_max) + && parse_and_validate(make_json(uint64_max), uint64_max) + && parse_and_validate(make_json(uint64_min), uint64_min) + && parse_and_validate(make_json(int64_min), int64_min) + && parse_and_validate(make_json(uint64_max), uint64_max) + && parse_and_validate(make_json(uint64_min), uint64_min) + && parse_and_validate(make_json(int64_max_plus1), int64_max_plus1) + && parse_and_check_signed(make_json(int64_max)) + && parse_and_check_unsigned(make_json(uint64_max)) + ) { + std::cout << "All ok." << std::endl; + return EXIT_SUCCESS; } - if(!parse_and_check_unsigned(make_json(uint64_max))) { - std::cerr << "bug: a large unsigned integers is not represented as an unsigned integer" << std::endl; - return EXIT_FAILURE; - } - std::cout << "All ok." << std::endl; - return EXIT_SUCCESS; + return EXIT_FAILURE; } diff --git a/tests/pointercheck.cpp b/tests/pointercheck.cpp index 58e67ce3e..e0517edb5 100644 --- a/tests/pointercheck.cpp +++ b/tests/pointercheck.cpp @@ -1,6 +1,7 @@ #include #include "simdjson.h" +#include "test_macros.h" // we define our own asserts to get around NDEBUG #ifndef ASSERT @@ -35,49 +36,46 @@ const padded_string TEST_JSON = R"( bool json_pointer_success_test(const char *json_pointer, std::string_view expected_value) { std::cout << "Running successful JSON pointer test '" << json_pointer << "' ..." << std::endl; dom::parser parser; - auto [value, error] = parser.parse(TEST_JSON).at(json_pointer).get(); - if (error) { std::cerr << "Unexpected Error: " << error << std::endl; return false; } - ASSERT(value == expected_value); + std::string_view value; + ASSERT_SUCCESS( parser.parse(TEST_JSON).at(json_pointer).get(value) ); + ASSERT_EQUAL(value, expected_value); return true; } bool json_pointer_success_test(const char *json_pointer) { std::cout << "Running successful JSON pointer test '" << json_pointer << "' ..." << std::endl; dom::parser parser; - auto error = parser.parse(TEST_JSON).at(json_pointer).error(); - if (error) { std::cerr << "Unexpected Error: " << error << std::endl; return false; } + ASSERT_SUCCESS( parser.parse(TEST_JSON).at(json_pointer).error() ); return true; } -bool json_pointer_failure_test(const char *json_pointer, error_code expected_failure_test) { +bool json_pointer_failure_test(const char *json_pointer, error_code expected_error) { std::cout << "Running invalid JSON pointer test '" << json_pointer << "' ..." << std::endl; dom::parser parser; - auto error = parser.parse(TEST_JSON).at(json_pointer).error(); - ASSERT(error == expected_failure_test); + ASSERT_ERROR(parser.parse(TEST_JSON).at(json_pointer).error(), expected_error); return true; } int main() { - if ( - json_pointer_success_test("") && - json_pointer_success_test("~1~001abc") && - json_pointer_success_test("~1~001abc/1") && - json_pointer_success_test("~1~001abc/1/\\\" 0") && - json_pointer_success_test("~1~001abc/1/\\\" 0/0", "value0") && - json_pointer_success_test("~1~001abc/1/\\\" 0/1", "value1") && - json_pointer_failure_test("~1~001abc/1/\\\" 0/2", INDEX_OUT_OF_BOUNDS) && // index actually out of bounds - json_pointer_success_test("arr") && // get array - json_pointer_failure_test("arr/0", INDEX_OUT_OF_BOUNDS) && // array index 0 out of bounds on empty array - json_pointer_success_test("~1~001abc") && // get object - json_pointer_success_test("0", "0 ok") && // object index with integer-ish key - json_pointer_success_test("01", "01 ok") && // object index with key that would be an invalid integer - json_pointer_success_test("", "empty ok") && // object index with empty key - json_pointer_failure_test("~01abc", NO_SUCH_FIELD) && // Test that we don't try to compare the literal key - json_pointer_failure_test("~1~001abc/01", INVALID_JSON_POINTER) && // Leading 0 in integer index - json_pointer_failure_test("~1~001abc/", INVALID_JSON_POINTER) && // Empty index to array - json_pointer_failure_test("~1~001abc/-", INDEX_OUT_OF_BOUNDS) && // End index is always out of bounds - true + if (true + && json_pointer_success_test("") + && json_pointer_success_test("~1~001abc") + && json_pointer_success_test("~1~001abc/1") + && json_pointer_success_test("~1~001abc/1/\\\" 0") + && json_pointer_success_test("~1~001abc/1/\\\" 0/0", "value0") + && json_pointer_success_test("~1~001abc/1/\\\" 0/1", "value1") + && json_pointer_failure_test("~1~001abc/1/\\\" 0/2", INDEX_OUT_OF_BOUNDS) // index actually out of bounds + && json_pointer_success_test("arr") // get array + && json_pointer_failure_test("arr/0", INDEX_OUT_OF_BOUNDS) // array index 0 out of bounds on empty array + && json_pointer_success_test("~1~001abc") // get object + && json_pointer_success_test("0", "0 ok") // object index with integer-ish key + && json_pointer_success_test("01", "01 ok") // object index with key that would be an invalid integer + && json_pointer_success_test("", "empty ok") // object index with empty key + && json_pointer_failure_test("~01abc", NO_SUCH_FIELD) // Test that we don't try to compare the literal key + && json_pointer_failure_test("~1~001abc/01", INVALID_JSON_POINTER) // Leading 0 in integer index + && json_pointer_failure_test("~1~001abc/", INVALID_JSON_POINTER) // Empty index to array + && json_pointer_failure_test("~1~001abc/-", INDEX_OUT_OF_BOUNDS) // End index is always out of bounds ) { std::cout << "Success!" << std::endl; return 0; diff --git a/tests/readme_examples.cpp b/tests/readme_examples.cpp index c8712197f..1242f3ce5 100644 --- a/tests/readme_examples.cpp +++ b/tests/readme_examples.cpp @@ -88,7 +88,7 @@ void basics_dom_4() { auto abstract_json = R"( { "str" : { "123" : {"abc" : 3.14 } } } )"_padded; dom::parser parser; - double v = parser.parse(abstract_json)["str"]["123"]["abc"].get(); + double v = parser.parse(abstract_json)["str"]["123"]["abc"]; cout << "number: " << v << endl; } @@ -141,9 +141,10 @@ namespace treewalk_1 { #ifdef SIMDJSON_CPLUSPLUS17 void basics_cpp17_1() { - dom::parser parser; padded_string json = R"( { "foo": 1, "bar": 2 } )"_padded; - auto [object, error] = parser.parse(json).get(); + dom::parser parser; + dom::object object; + auto error = parser.parse(json).get(object); if (error) { cerr << error << endl; return; } for (auto [key, value] : object) { cout << key << " = " << value << endl; @@ -153,11 +154,10 @@ void basics_cpp17_1() { void basics_cpp17_2() { // C++ 11 version for comparison - dom::parser parser; padded_string json = R"( { "foo": 1, "bar": 2 } )"_padded; - simdjson::error_code error; + dom::parser parser; dom::object object; - error = parser.parse(json).get(object); + auto error = parser.parse(json).get(object); if (!error) { cerr << error << endl; return; } for (dom::key_value_pair field : object) { cout << field.key << " = " << field.value << endl; diff --git a/tests/test_macros.h b/tests/test_macros.h index 35228ca24..a44855534 100644 --- a/tests/test_macros.h +++ b/tests/test_macros.h @@ -18,17 +18,25 @@ const char *SMALLDEMO_JSON = SIMDJSON_BENCHMARK_SMALLDATA_DIR "smalldemo.json"; const char *TRUENULL_JSON = SIMDJSON_BENCHMARK_SMALLDATA_DIR "truenull.json"; // For the ASSERT_EQUAL macro -template -bool equals_expected(T actual, T expected) { - return actual == expected; +template +bool equals_expected(T actual, S expected) { + return actual == T(expected); } template<> -bool equals_expected(const char *actual, const char *expected) { +bool equals_expected(const char *actual, const char *expected) { return !strcmp(actual, expected); } #define TEST_START() { cout << "Running " << __func__ << " ..." << endl; } -#define ASSERT_EQUAL(ACTUAL, EXPECTED) do { auto _actual = (ACTUAL); auto _expected = (EXPECTED); if (!equals_expected(_actual, _expected)) { std::cerr << "Expected " << #ACTUAL << " to be " << _expected << ", got " << _actual << " instead!" << std::endl; return false; } } while(0); +#define ASSERT_EQUAL(ACTUAL, EXPECTED) \ +do { \ + auto _actual = (ACTUAL); \ + auto _expected = (EXPECTED); \ + if (!equals_expected(_actual, _expected)) { \ + std::cerr << "Expected " << (#ACTUAL) << " to be " << _expected << ", got " << _actual << " instead!" << std::endl; \ + return false; \ + } \ +} while(0); #define ASSERT_ERROR(ACTUAL, EXPECTED) do { auto _actual = (ACTUAL); auto _expected = (EXPECTED); if (_actual != _expected) { std::cerr << "FAIL: Unexpected error \"" << _actual << "\" (expected \"" << _expected << "\")" << std::endl; return false; } } while (0); #define ASSERT(RESULT, MESSAGE) if (!(RESULT)) { std::cerr << MESSAGE << std::endl; return false; } #define RUN_TEST(RESULT) if (!RESULT) { return false; } From 1ff55c27296ca2a2205d81f88769e37d0aed9778 Mon Sep 17 00:00:00 2001 From: John Keiser Date: Sun, 21 Jun 2020 15:26:44 -0700 Subject: [PATCH 06/10] Replace auto [x,error] with .get() everywhere --- benchmark/bench_dom_api.cpp | 5 +- benchmark/distinctuseridcompetition.cpp | 3 +- benchmark/minifiercompetition.cpp | 3 +- benchmark/parseandstatcompetition.cpp | 13 ++- benchmark/parsingcompetition.cpp | 7 +- benchmark/statisticalmodel.cpp | 6 +- doc/basics.md | 20 +--- doc/performance.md | 10 +- include/simdjson/dom/parser.h | 20 ++-- include/simdjson/error.h | 3 +- tests/allparserscheckfile.cpp | 17 ++-- tests/basictests.cpp | 116 +++++++++--------------- tests/jsoncheck.cpp | 3 +- tests/numberparsingcheck.cpp | 3 +- tests/readme_examples.cpp | 22 +++-- tests/readme_examples_noexceptions.cpp | 19 +--- tests/singleheadertest.cpp | 7 +- tests/stringparsingcheck.cpp | 3 +- tests/test_macros.h | 12 ++- tools/json2json.cpp | 8 +- tools/jsonpointer.cpp | 9 +- tools/jsonstats.cpp | 6 +- tools/minify.cpp | 3 +- 23 files changed, 155 insertions(+), 163 deletions(-) diff --git a/benchmark/bench_dom_api.cpp b/benchmark/bench_dom_api.cpp index 294977e25..ed6467bd9 100644 --- a/benchmark/bench_dom_api.cpp +++ b/benchmark/bench_dom_api.cpp @@ -485,7 +485,8 @@ static void iterator_twitter_image_sizes(State& state) { if (iter.down()) { // first status do { - // auto [media, not_found] = tweet["entities"]["media"]; + // dom::object media; + // not_found = tweet["entities"]["media"].get(media); // if (!not_found) { if (iter.move_to_key("entities")) { if (!iter.is_object()) { return; } @@ -496,7 +497,7 @@ static void iterator_twitter_image_sizes(State& state) { if (iter.down()) { // first media do { - // for (auto [key, size] : image["sizes"].get()) { + // for (auto [key, size] : dom::object(image["sizes"])) { if (!(iter.move_to_key("sizes") && iter.is_object())) { return; } if (iter.down()) { // first size do { diff --git a/benchmark/distinctuseridcompetition.cpp b/benchmark/distinctuseridcompetition.cpp index a9bb86dc6..f2c4a6434 100644 --- a/benchmark/distinctuseridcompetition.cpp +++ b/benchmark/distinctuseridcompetition.cpp @@ -331,7 +331,8 @@ int main(int argc, char *argv[]) { std::cerr << "warning: ignoring everything after " << argv[optind + 1] << std::endl; } - auto [p, error] = simdjson::padded_string::load(filename); + simdjson::padded_string p; + auto error = simdjson::padded_string::load(filename).get(p); if (error) { std::cerr << "Could not load the file " << filename << std::endl; return EXIT_FAILURE; diff --git a/benchmark/minifiercompetition.cpp b/benchmark/minifiercompetition.cpp index 2a8c1611a..08c80f7aa 100644 --- a/benchmark/minifiercompetition.cpp +++ b/benchmark/minifiercompetition.cpp @@ -75,7 +75,8 @@ int main(int argc, char *argv[]) { exit(1); } const char *filename = argv[optind]; - auto [p, error] = simdjson::padded_string::load(filename); + simdjson::padded_string p; + auto error = simdjson::padded_string::load(filename).get(p); if (error) { std::cerr << "Could not load the file " << filename << std::endl; return EXIT_FAILURE; diff --git a/benchmark/parseandstatcompetition.cpp b/benchmark/parseandstatcompetition.cpp index d18ffc992..72972ca92 100644 --- a/benchmark/parseandstatcompetition.cpp +++ b/benchmark/parseandstatcompetition.cpp @@ -105,7 +105,8 @@ void simdjson_recurse(stat_t &s, simdjson::dom::element element) { never_inline stat_t simdjson_compute_stats(const simdjson::padded_string &p) { stat_t s{}; simdjson::dom::parser parser; - auto [doc, error] = parser.parse(p); + simdjson::dom::element doc; + auto error = parser.parse(p).get(doc); if (error) { s.valid = false; return s; @@ -409,7 +410,8 @@ int main(int argc, char *argv[]) { std::cerr << "warning: ignoring everything after " << argv[optind + 1] << std::endl; } - auto [p, error] = simdjson::padded_string::load(filename); + simdjson::padded_string p; + auto error = simdjson::padded_string::load(filename).get(p); if (error) { std::cerr << "Could not load the file " << filename << std::endl; return EXIT_FAILURE; @@ -464,9 +466,10 @@ int main(int argc, char *argv[]) { printf("API traversal tests\n"); printf("Based on https://github.com/miloyip/nativejson-benchmark\n"); simdjson::dom::parser parser; - auto [doc, err] = parser.parse(p); - if (err) { - std::cerr << err << std::endl; + simdjson::dom::element doc; + auto error = parser.parse(p).get(doc); + if (error) { + std::cerr << error << std::endl; } size_t refval = simdjson_compute_stats_refplus(doc).objectCount; diff --git a/benchmark/parsingcompetition.cpp b/benchmark/parsingcompetition.cpp index 80a2a76d9..509e44087 100644 --- a/benchmark/parsingcompetition.cpp +++ b/benchmark/parsingcompetition.cpp @@ -82,9 +82,10 @@ inline void reset_stream(std::stringstream & is) { bool bench(const char *filename, bool verbose, bool just_data, double repeat_multiplier) { - auto [p, err] = simdjson::padded_string::load(filename); - if (err) { - std::cerr << "Could not load the file " << filename << std::endl; + simdjson::padded_string p; + auto error = simdjson::padded_string::load(filename).get(p); + if (error) { + std::cerr << "Could not load the file " << filename << ": " << error << std::endl; return false; } diff --git a/benchmark/statisticalmodel.cpp b/benchmark/statisticalmodel.cpp index 3838f5ef9..89efe82d6 100644 --- a/benchmark/statisticalmodel.cpp +++ b/benchmark/statisticalmodel.cpp @@ -96,7 +96,8 @@ void simdjson_recurse(stat_t &s, simdjson::dom::element element) { stat_t simdjson_compute_stats(const simdjson::padded_string &p) { stat_t answer{}; simdjson::dom::parser parser; - auto [doc, error] = parser.parse(p); + simdjson::dom::element doc; + auto error = parser.parse(p).get(doc); if (error) { answer.valid = false; return answer; @@ -136,7 +137,8 @@ int main(int argc, char *argv[]) { std::cerr << "warning: ignoring everything after " << argv[optind + 1] << std::endl; } - auto [p, error] = simdjson::padded_string::load(filename); + simdjson::padded_string p; + auto error = simdjson::padded_string::load(filename).get(p); if (error) { std::cerr << "Could not load the file " << filename << std::endl; return EXIT_FAILURE; diff --git a/doc/basics.md b/doc/basics.md index 0f5c5d541..3751b2582 100644 --- a/doc/basics.md +++ b/doc/basics.md @@ -239,29 +239,18 @@ Error Handling -------------- All simdjson APIs that can fail return `simdjson_result`, which is a <value, error_code> -pair. The error codes and values can be accessed directly, reading the error like so: +pair. You can retrieve the value with .get(), like so: ```c++ -auto [doc, error] = parser.parse(json); // doc is a dom::element +dom::element doc; +auto error = parser.parse(json).get(doc); if (error) { cerr << error << endl; exit(1); } -// Use document here now that we've checked for the error ``` When you use the code this way, it is your responsibility to check for error before using the result: if there is an error, the result value will not be valid and using it will caused undefined behavior. -> Note: because of the way `auto [x, y]` works in C++, you have to define new variables each time you -> use it. If your project treats aliased, this means you can't use the same names in `auto [x, error]` -> without triggering warnings or error (and particularly can't use the word "error" every time). To -> circumvent this, you can use this instead: -> -> ```c++ -> dom::element doc; -> auto error = parser.parse(json).get(doc); // <-- Assigns to doc and error just like "auto [doc, error]" -> ``` - - We can write a "quick start" example where we attempt to parse a file and access some data, without triggering exceptions: ```C++ @@ -269,11 +258,12 @@ We can write a "quick start" example where we attempt to parse a file and access int main(void) { simdjson::dom::parser parser; + simdjson::dom::element tweets; auto error = parser.load("twitter.json").get(tweets); if (error) { std::cerr << error << std::endl; return EXIT_FAILURE; } - simdjson::dom::element res; + simdjson::dom::element res; if ((error = tweets["search_metadata"]["count"].get(res))) { std::cerr << "could not access keys" << std::endl; return EXIT_FAILURE; diff --git a/doc/performance.md b/doc/performance.md index bf022a4cd..1b65e995b 100644 --- a/doc/performance.md +++ b/doc/performance.md @@ -68,7 +68,8 @@ without bound: ```c++ dom::parser parser(1000*1000); // Never grow past documents > 1MB for (web_request request : listen()) { - auto [doc, error] = parser.parse(request.body); + dom::element doc; + auto error = parser.parse(request.body).get(doc); // If the document was above our limit, emit 413 = payload too large if (error == CAPACITY) { request.respond(413); continue; } // ... @@ -82,11 +83,12 @@ without bound: ```c++ dom::parser parser(0); // This parser will refuse to automatically grow capacity - simdjson::error_code allocate_error = parser.allocate(1000*1000); // This allocates enough capacity to handle documents <= 1MB - if (allocate_error) { cerr << allocate_error << endl; exit(1); } + auto error = parser.allocate(1000*1000); // This allocates enough capacity to handle documents <= 1MB + if (error) { cerr << error << endl; exit(1); } for (web_request request : listen()) { - auto [doc, error] = parser.parse(request.body); + dom::element doc; + error = parser.parse(request.body).get(doc); // If the document was above our limit, emit 413 = payload too large if (error == CAPACITY) { request.respond(413); continue; } // ... diff --git a/include/simdjson/dom/parser.h b/include/simdjson/dom/parser.h index ba3d9b668..25eccf430 100644 --- a/include/simdjson/dom/parser.h +++ b/include/simdjson/dom/parser.h @@ -173,9 +173,13 @@ public: * the same interface, requiring you to check the error before using the document: * * dom::parser parser; - * for (auto [doc, error] : parser.load_many(path)) { - * if (error) { cerr << error << endl; exit(1); } - * cout << std::string(doc["title"]) << endl; + * dom::document_stream docs; + * auto error = parser.load_many(path).get(docs); + * if (error) { cerr << error << endl; exit(1); } + * for (auto doc : docs) { + * std::string_view title; + * if ((error = doc["title"].get(title)) { cerr << error << endl; exit(1); } + * cout << title << endl; * } * * ### Threads @@ -233,9 +237,13 @@ public: * the same interface, requiring you to check the error before using the document: * * dom::parser parser; - * for (auto [doc, error] : parser.parse_many(buf, len)) { - * if (error) { cerr << error << endl; exit(1); } - * cout << std::string(doc["title"]) << endl; + * dom::document_stream docs; + * auto error = parser.load_many(path).get(docs); + * if (error) { cerr << error << endl; exit(1); } + * for (auto doc : docs) { + * std::string_view title; + * if ((error = doc["title"].get(title)) { cerr << error << endl; exit(1); } + * cout << title << endl; * } * * ### REQUIRED: Buffer Padding diff --git a/include/simdjson/error.h b/include/simdjson/error.h index f272abd83..308bc1f0b 100644 --- a/include/simdjson/error.h +++ b/include/simdjson/error.h @@ -42,7 +42,8 @@ enum error_code { * Get the error message for the given error code. * * dom::parser parser; - * auto [doc, error] = parser.parse("foo"); + * dom::element doc; + * auto error = parser.parse("foo").get(doc); * if (error) { printf("Error: %s\n", error_message(error)); } * * @return The error message. diff --git a/tests/allparserscheckfile.cpp b/tests/allparserscheckfile.cpp index 6784a0603..959268e63 100644 --- a/tests/allparserscheckfile.cpp +++ b/tests/allparserscheckfile.cpp @@ -63,9 +63,10 @@ int main(int argc, char *argv[]) { exit(1); } const char *filename = argv[optind]; - auto [p, loaderr] = simdjson::padded_string::load(filename); - if (loaderr) { - std::cerr << "Could not load the file " << filename << ": " << loaderr << std::endl; + simdjson::padded_string p; + auto error = simdjson::padded_string::load(filename).get(p); + if (error) { + std::cerr << "Could not load the file " << filename << ": " << error << std::endl; return EXIT_FAILURE; } if (verbose) { @@ -79,7 +80,7 @@ int main(int argc, char *argv[]) { std::cout << std::endl; } simdjson::dom::parser parser; - auto err = parser.parse(p).error(); + error = parser.parse(p).error(); rapidjson::Document d; @@ -95,19 +96,19 @@ int main(int argc, char *argv[]) { .is_valid(); if (just_favorites) { printf("our parser : %s \n", - (err == simdjson::error_code::SUCCESS) ? "correct" : "invalid"); + (error == simdjson::error_code::SUCCESS) ? "correct" : "invalid"); printf("rapid (check encoding) : %s \n", rapid_correct_checkencoding ? "correct" : "invalid"); printf("sajson : %s \n", sajson_correct ? "correct" : "invalid"); - if (err == simdjson::DEPTH_ERROR) { + if (error == simdjson::DEPTH_ERROR) { printf("simdjson encountered a DEPTH_ERROR, it was parametrized to " "reject documents with depth exceeding %zu.\n", parser.max_depth()); } - if (((err == simdjson::error_code::SUCCESS) != rapid_correct_checkencoding) || + if (((error == simdjson::error_code::SUCCESS) != rapid_correct_checkencoding) || (rapid_correct_checkencoding != sajson_correct) || - ((err == simdjson::SUCCESS) != sajson_correct)) { + ((error == simdjson::SUCCESS) != sajson_correct)) { printf("WARNING: THEY DISAGREE\n\n"); return EXIT_FAILURE; } diff --git a/tests/basictests.cpp b/tests/basictests.cpp index e5f3d8e85..994652890 100644 --- a/tests/basictests.cpp +++ b/tests/basictests.cpp @@ -178,17 +178,14 @@ namespace number_tests { } namespace document_tests { - int issue938() { + bool issue938() { std::vector json_strings{"[true,false]", "[1,2,3,null]", R"({"yay":"json!"})"}; simdjson::dom::parser parser1; for (simdjson::padded_string str : json_strings) { - auto [element, error] = parser1.parse(str); - if(error) { - std::cerr << error << std::endl; - } else { - std::cout << element << std::endl; - } + simdjson::dom::element element; + ASSERT_SUCCESS( parser1.parse(str).get(element) ); + std::cout << element << std::endl; } std::vector file_paths{ ADVERSARIAL_JSON, FLATADVERSARIAL_JSON, DEMO_JSON, @@ -196,23 +193,17 @@ namespace document_tests { TRUENULL_JSON}; for (auto path : file_paths) { simdjson::dom::parser parser2; + simdjson::dom::element element; std::cout << "file: " << path << std::endl; - auto [element, error] = parser2.load(path); - if(error) { - std::cerr << error << std::endl; - } else { - std::cout << element.type() << std::endl; - } + ASSERT_SUCCESS( parser2.load(path).get(element) ); + std::cout << element.type() << std::endl; } simdjson::dom::parser parser3; for (auto path : file_paths) { + simdjson::dom::element element; std::cout << "file: " << path << std::endl; - auto [element, error] = parser3.load(path); - if(error) { - std::cerr << error << std::endl; - } else { - std::cout << element.type() << std::endl; - } + ASSERT_SUCCESS( parser3.load(path).get(element) ); + std::cout << element.type() << std::endl; } return true; } @@ -222,11 +213,7 @@ namespace document_tests { std::cout << __func__ << std::endl; simdjson::padded_string badjson = "[7,7,7,7,6,7,7,7,6,7,7,6,[7,7,7,7,6,7,7,7,6,7,7,6,7,7,7,7,7,7,6"_padded; simdjson::dom::parser parser; - auto error = parser.parse(badjson).error(); - if (!error) { - printf("This json should not be valid %s.\n", badjson.data()); - return false; - } + ASSERT_ERROR( parser.parse(badjson), simdjson::TAPE_ERROR ); return true; } bool count_array_example() { @@ -251,9 +238,9 @@ namespace document_tests { std::cout << __func__ << std::endl; simdjson::dom::parser parser; // This is an invalid document padded with open braces. - ASSERT_ERROR( parser.parse("[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[", 2, false).error(), simdjson::TAPE_ERROR); + ASSERT_ERROR( parser.parse("[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[", 2, false), simdjson::TAPE_ERROR); // This is a valid document padded with open braces. - ASSERT_SUCCESS( parser.parse("[][[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[", 2, false).error() ); + ASSERT_SUCCESS( parser.parse("[][[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[", 2, false) ); return true; } // returns true if successful @@ -400,7 +387,7 @@ namespace document_stream_tests { size_t count = 0; for(; i != stream.end(); ++i) { auto doc = *i; - ASSERT_SUCCESS(doc.error()); + ASSERT_SUCCESS(doc); if( i.current_index() != count) { std::cout << "index:" << i.current_index() << std::endl; std::cout << "expected index:" << count << std::endl; @@ -508,12 +495,7 @@ namespace document_stream_tests { size_t count = 0; simdjson::dom::document_stream stream; ASSERT_SUCCESS( parser.parse_many(str, batch_size).get(stream) ); - for (auto [doc, error] : stream) { - if (error) { - printf("Error at on document %zd at batch size %zu: %s\n", count, batch_size, simdjson::error_message(error)); - return false; - } - + for (auto doc : stream) { int64_t keyid; ASSERT_SUCCESS( doc["id"].get(keyid) ); ASSERT_EQUAL( keyid, int64_t(count) ); @@ -553,12 +535,7 @@ namespace document_stream_tests { size_t count = 0; simdjson::dom::document_stream stream; ASSERT_SUCCESS( parser.parse_many(str, batch_size).get(stream) ); - for (auto [doc, error] : stream) { - if (error) { - printf("Error at on document %zd at batch size %zu: %s\n", count, batch_size, simdjson::error_message(error)); - return false; - } - + for (auto doc : stream) { int64_t keyid; ASSERT_SUCCESS( doc["id"].get(keyid) ); ASSERT_EQUAL( keyid, int64_t(count) ); @@ -593,9 +570,9 @@ namespace parse_api_tests { bool parser_parse() { std::cout << "Running " << __func__ << std::endl; dom::parser parser; - auto [doc, error] = parser.parse(BASIC_JSON); - if (error) { cerr << error << endl; return false; } - if (!doc.is()) { cerr << "Document did not parse as an array" << endl; return false; } + dom::element doc; + ASSERT_SUCCESS( parser.parse(BASIC_JSON).get(doc) ); + ASSERT_EQUAL( doc.is(), true ); return true; } bool parser_parse_many() { @@ -604,12 +581,12 @@ namespace parse_api_tests { int count = 0; simdjson::dom::document_stream stream; ASSERT_SUCCESS( parser.parse_many(BASIC_NDJSON).get(stream) ); - for (auto [doc, error] : stream) { - if (error) { cerr << "Error in parse_many: " << endl; return false; } - if (!doc.is()) { cerr << "Document did not parse as an array" << endl; return false; } + for (auto doc : stream) { + UNUSED dom::array array; + ASSERT_SUCCESS( doc.get(array) ); count++; } - if (count != 2) { cerr << "parse_many returned " << count << " documents, expected 2" << endl; return false; } + ASSERT_EQUAL(count, 2); return true; } @@ -619,12 +596,12 @@ namespace parse_api_tests { std::cout << "Running " << __func__ << std::endl; dom::parser parser; int count = 0; - for (auto [doc, error] : parser.parse_many(BASIC_NDJSON)) { - if (error) { cerr << "Error in parse_many: " << endl; return false; } - if (!doc.is()) { cerr << "Document did not parse as an array" << endl; return false; } + for (auto doc : parser.parse_many(BASIC_NDJSON)) { + UNUSED dom::array array; + ASSERT_SUCCESS( doc.get(array) ); count++; } - if (count != 2) { cerr << "parse_many returned " << count << " documents, expected 2" << endl; return false; } + ASSERT_EQUAL(count, 2); return true; } SIMDJSON_POP_DISABLE_WARNINGS @@ -635,7 +612,7 @@ namespace parse_api_tests { simdjson::dom::document_stream stream; ASSERT_SUCCESS( parser.parse_many(EMPTY_NDJSON).get(stream) ); for (auto doc : stream) { - ASSERT_SUCCESS(doc.error()); + ASSERT_SUCCESS( doc ); count++; } ASSERT_EQUAL(count, 0); @@ -654,8 +631,7 @@ namespace parse_api_tests { memcpy(&empty_batches_ndjson[BATCH_SIZE*11+6], "3", 1); simdjson::dom::document_stream stream; ASSERT_SUCCESS( parser.parse_many(empty_batches_ndjson, BATCH_SIZE*16).get(stream) ); - for (auto [doc, error] : stream) { - ASSERT_SUCCESS(error); + for (auto doc : stream) { count++; uint64_t val; ASSERT_SUCCESS( doc.get(val) ); @@ -678,16 +654,14 @@ namespace parse_api_tests { int count = 0; simdjson::dom::document_stream stream; ASSERT_SUCCESS( parser.load_many(AMAZON_CELLPHONES_NDJSON).get(stream) ); - for (auto [doc, error] : stream) { - ASSERT_SUCCESS( error ); - + for (auto doc : stream) { dom::array arr; ASSERT_SUCCESS( doc.get(arr) ); // let us get the array - ASSERT_EQUAL(arr.size(), 9); + ASSERT_EQUAL( arr.size(), 9 ); size_t arr_count = 0; for (auto v : arr) { arr_count++; (void)v; } - ASSERT_EQUAL(arr_count, 9); + ASSERT_EQUAL( arr_count, 9 ); count++; } @@ -701,9 +675,7 @@ namespace parse_api_tests { std::cout << "Running " << __func__ << " on " << AMAZON_CELLPHONES_NDJSON << std::endl; dom::parser parser; int count = 0; - for (auto [doc, error] : parser.load_many(AMAZON_CELLPHONES_NDJSON)) { - if (error) { cerr << error << endl; return false; } - + for (auto doc : parser.load_many(AMAZON_CELLPHONES_NDJSON)) { dom::array arr; ASSERT_SUCCESS( doc.get(arr) ); ASSERT_EQUAL( arr.size(), 9 ); @@ -902,7 +874,7 @@ namespace dom_api_tests { int i = 0; for (auto [key, value] : object) { ASSERT_EQUAL( key, expected_key[i] ); - ASSERT_EQUAL( value.get().value(), expected_value[i] ); + ASSERT_EQUAL( value.get().first, expected_value[i] ); i++; } ASSERT_EQUAL( i*sizeof(uint64_t), sizeof(expected_value) ); @@ -987,18 +959,18 @@ namespace dom_api_tests { ASSERT_SUCCESS( parser.parse(json).get(array) ); auto iter = array.begin(); - ASSERT_EQUAL( (*iter).get().value(), 0 ); - ASSERT_EQUAL( (*iter).get().value(), 0 ); - ASSERT_EQUAL( (*iter).get().value(), 0 ); + ASSERT_EQUAL( (*iter).get().first, 0 ); + ASSERT_EQUAL( (*iter).get().first, 0 ); + ASSERT_EQUAL( (*iter).get().first, 0 ); ++iter; - ASSERT_EQUAL( (*iter).get().value(), 1 ); - ASSERT_EQUAL( (*iter).get().value(), 1 ); - ASSERT_EQUAL( (*iter).get().value(), 1 ); + ASSERT_EQUAL( (*iter).get().first, 1 ); + ASSERT_EQUAL( (*iter).get().first, 1 ); + ASSERT_EQUAL( (*iter).get().first, 1 ); ++iter; - ASSERT_EQUAL( (*iter).get().value(), -1 ); - ASSERT_EQUAL( (*iter).get().value(), -1 ); + ASSERT_EQUAL( (*iter).get().first, -1 ); + ASSERT_EQUAL( (*iter).get().first, -1 ); ++iter; - ASSERT_EQUAL( (*iter).get().value(), 1.1 ); + ASSERT_EQUAL( (*iter).get().first, 1.1 ); return true; } @@ -1054,7 +1026,7 @@ namespace dom_api_tests { object["d"].tie(val, error); ASSERT_ERROR( error, NO_SUCH_FIELD ); ASSERT_ERROR( object["d"].get(val), NO_SUCH_FIELD ); - ASSERT_ERROR( object["d"].error(), NO_SUCH_FIELD ); + ASSERT_ERROR( object["d"], NO_SUCH_FIELD ); return true; } diff --git a/tests/jsoncheck.cpp b/tests/jsoncheck.cpp index 55c734e69..35001c7ce 100644 --- a/tests/jsoncheck.cpp +++ b/tests/jsoncheck.cpp @@ -60,7 +60,8 @@ bool validate(const char *dirname) { char *fullpath = static_cast(malloc(fullpathlen)); snprintf(fullpath, fullpathlen, "%s%s%s", dirname, needsep ? "/" : "", name); - auto [p, error] = simdjson::padded_string::load(fullpath); + simdjson::padded_string p; + auto error = simdjson::padded_string::load(fullpath).get(p); if (error) { std::cerr << "Could not load the file " << fullpath << std::endl; return EXIT_FAILURE; diff --git a/tests/numberparsingcheck.cpp b/tests/numberparsingcheck.cpp index 92ab25f37..dbcc30477 100644 --- a/tests/numberparsingcheck.cpp +++ b/tests/numberparsingcheck.cpp @@ -172,7 +172,8 @@ bool validate(const char *dirname) { } else { strcpy(fullpath + dirlen, name); } - auto [p, error] = simdjson::padded_string::load(fullpath); + simdjson::padded_string p; + auto error = simdjson::padded_string::load(fullpath).get(p); if (error) { std::cerr << "Could not load the file " << fullpath << std::endl; return EXIT_FAILURE; diff --git a/tests/readme_examples.cpp b/tests/readme_examples.cpp index 1242f3ce5..c870f03ce 100644 --- a/tests/readme_examples.cpp +++ b/tests/readme_examples.cpp @@ -216,26 +216,28 @@ SIMDJSON_PUSH_DISABLE_ALL_WARNINGS // The web_request part of this is aspirational, so we compile as much as we can here void performance_2() { dom::parser parser(1000*1000); // Never grow past documents > 1MB -// for (web_request request : listen()) { - auto [doc, error] = parser.parse("1"_padded/*request.body*/); -// // If the document was above our limit, emit 413 = payload too large + /* for (web_request request : listen()) */ { + dom::element doc; + auto error = parser.parse("1"_padded/*request.body*/).get(doc); + // If the document was above our limit, emit 413 = payload too large if (error == CAPACITY) { /* request.respond(413); continue; */ } -// // ... -// } + // ... + } } // The web_request part of this is aspirational, so we compile as much as we can here void performance_3() { dom::parser parser(0); // This parser will refuse to automatically grow capacity - simdjson::error_code allocate_error = parser.allocate(1000*1000); // This allocates enough capacity to handle documents <= 1MB - if (allocate_error) { cerr << allocate_error << endl; exit(1); } + auto error = parser.allocate(1000*1000); // This allocates enough capacity to handle documents <= 1MB + if (error) { cerr << error << endl; exit(1); } - // for (web_request request : listen()) { - auto [doc, error] = parser.parse("1"_padded/*request.body*/); + /* for (web_request request : listen()) */ { + dom::element doc; + auto error = parser.parse("1"_padded/*request.body*/).get(doc); // If the document was above our limit, emit 413 = payload too large if (error == CAPACITY) { /* request.respond(413); continue; */ } // ... - // } + } } SIMDJSON_POP_DISABLE_WARNINGS #endif diff --git a/tests/readme_examples_noexceptions.cpp b/tests/readme_examples_noexceptions.cpp index 36bf89ced..e668a0f40 100644 --- a/tests/readme_examples_noexceptions.cpp +++ b/tests/readme_examples_noexceptions.cpp @@ -10,7 +10,8 @@ void basics_error_1() { dom::parser parser; auto json = "1"_padded; - auto [doc, error] = parser.parse(json); // doc is a dom::element + dom::element doc; + auto error = parser.parse(json).get(doc); if (error) { cerr << error << endl; exit(1); } // Use document here now that we've checked for the error } @@ -18,14 +19,6 @@ SIMDJSON_POP_DISABLE_WARNINGS #endif void basics_error_2() { - dom::parser parser; - auto json = "1"_padded; - - dom::element doc; - UNUSED auto error = parser.parse(json).get(doc); // <-- Assigns to doc and error just like "auto [doc, error]"} -} - -void basics_error_3() { auto cars_json = R"( [ { "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] }, { "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] }, @@ -70,8 +63,7 @@ void basics_error_3() { } } - -void basics_error_4() { +void basics_error_3() { auto abstract_json = R"( [ { "12345" : {"a":12.34, "b":56.78, "c": 9998877} }, { "12545" : {"a":11.44, "b":12.78, "c": 11111111} } @@ -102,7 +94,7 @@ void basics_error_4() { } } -void basics_error_5() { +void basics_error_4() { auto abstract_json = R"( { "str" : { "123" : {"abc" : 3.14 } } } )"_padded; dom::parser parser; @@ -116,7 +108,7 @@ void basics_error_5() { #ifdef SIMDJSON_CPLUSPLUS17 -void basics_error_3_cpp17() { +void basics_error_2_cpp17() { auto cars_json = R"( [ { "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] }, { "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] }, @@ -201,6 +193,5 @@ int main() { basics_error_2(); basics_error_3(); basics_error_4(); - basics_error_5(); return EXIT_SUCCESS; } diff --git a/tests/singleheadertest.cpp b/tests/singleheadertest.cpp index b2d0051d8..46a0e3cb0 100644 --- a/tests/singleheadertest.cpp +++ b/tests/singleheadertest.cpp @@ -6,12 +6,13 @@ using namespace simdjson; int main() { const char *filename = SIMDJSON_BENCHMARK_DATA_DIR "/twitter.json"; - padded_string p = get_corpus(filename); dom::parser parser; - auto [doc, error] = parser.parse(p); - if(error) { + dom::element doc; + auto error = parser.load(filename).get(doc); + if (error) { std::cerr << error << std::endl; return EXIT_FAILURE; } + std::cout << doc << std::endl; return EXIT_SUCCESS; } diff --git a/tests/stringparsingcheck.cpp b/tests/stringparsingcheck.cpp index eeec5900b..cefaa637b 100644 --- a/tests/stringparsingcheck.cpp +++ b/tests/stringparsingcheck.cpp @@ -337,7 +337,8 @@ bool validate(const char *dirname) { } else { strcpy(fullpath + dirlen, name); } - auto [p, error] = simdjson::padded_string::load(fullpath); + simdjson::padded_string p; + auto error = simdjson::padded_string::load(fullpath).get(p); if (error) { std::cerr << "Could not load the file " << fullpath << std::endl; return EXIT_FAILURE; diff --git a/tests/test_macros.h b/tests/test_macros.h index a44855534..661e50b81 100644 --- a/tests/test_macros.h +++ b/tests/test_macros.h @@ -27,6 +27,14 @@ bool equals_expected(const char *actual, const char return !strcmp(actual, expected); } +simdjson::error_code to_error_code(simdjson::error_code error) { + return error; +} +template +simdjson::error_code to_error_code(const simdjson::simdjson_result &result) { + return result.error(); +} + #define TEST_START() { cout << "Running " << __func__ << " ..." << endl; } #define ASSERT_EQUAL(ACTUAL, EXPECTED) \ do { \ @@ -37,10 +45,10 @@ do { \ return false; \ } \ } while(0); -#define ASSERT_ERROR(ACTUAL, EXPECTED) do { auto _actual = (ACTUAL); auto _expected = (EXPECTED); if (_actual != _expected) { std::cerr << "FAIL: Unexpected error \"" << _actual << "\" (expected \"" << _expected << "\")" << std::endl; return false; } } while (0); +#define ASSERT_ERROR(ACTUAL, EXPECTED) do { auto _actual = to_error_code(ACTUAL); auto _expected = to_error_code(EXPECTED); if (_actual != _expected) { std::cerr << "FAIL: Unexpected error \"" << _actual << "\" (expected \"" << _expected << "\")" << std::endl; return false; } } while (0); #define ASSERT(RESULT, MESSAGE) if (!(RESULT)) { std::cerr << MESSAGE << std::endl; return false; } #define RUN_TEST(RESULT) if (!RESULT) { return false; } -#define ASSERT_SUCCESS(ERROR) do { auto _error = (ERROR); if (_error) { std::cerr << _error << std::endl; return false; } } while(0); +#define ASSERT_SUCCESS(ERROR) do { auto _error = to_error_code(ERROR); if (_error) { std::cerr << _error << std::endl; return false; } } while(0); #define TEST_FAIL(MESSAGE) { std::cerr << "FAIL: " << (MESSAGE) << std::endl; return false; } #define TEST_SUCCEED() { return true; } diff --git a/tools/json2json.cpp b/tools/json2json.cpp index d02ba74c5..084ad3ca2 100644 --- a/tools/json2json.cpp +++ b/tools/json2json.cpp @@ -49,10 +49,10 @@ int main(int argc, char *argv[]) { const char *filename = result["file"].as().c_str(); simdjson::dom::parser parser; - auto [doc, error] = parser.load(filename); // do the parsing, return false on error - if (error != simdjson::SUCCESS) { - std::cerr << " Parsing failed. Error is '" << simdjson::error_message(error) - << "'." << std::endl; + simdjson::dom::element doc; + auto error = parser.load(filename).get(doc); // do the parsing, return false on error + if (error) { + std::cerr << " Parsing failed. Error is '" << error << "'." << std::endl; return EXIT_FAILURE; } if(rawdump) { diff --git a/tools/jsonpointer.cpp b/tools/jsonpointer.cpp index c5f0c6f55..ca6311a1b 100644 --- a/tools/jsonpointer.cpp +++ b/tools/jsonpointer.cpp @@ -20,16 +20,17 @@ int main(int argc, char *argv[]) { const char *filename = argv[1]; simdjson::dom::parser parser; - auto [doc, error] = parser.load(filename); + simdjson::dom::element doc; + auto error = parser.load(filename).get(doc); if (error) { std::cerr << "Error parsing " << filename << ": " << error << std::endl; } std::cout << "[" << std::endl; for (int idx = 2; idx < argc; idx++) { const char *json_pointer = argv[idx]; - auto [value, pointer_error] = doc[json_pointer]; + simdjson::dom::element value; std::cout << "{\"jsonpath\": \"" << json_pointer << "\""; - if (pointer_error) { - std::cout << ",\"error\":\"" << pointer_error << "\""; + if ((error = doc[json_pointer].get(value))) { + std::cout << ",\"error\":\"" << error << "\""; } else { std::cout << ",\"value\":" << value; } diff --git a/tools/jsonstats.cpp b/tools/jsonstats.cpp index 4f2aa4208..977d6a480 100644 --- a/tools/jsonstats.cpp +++ b/tools/jsonstats.cpp @@ -166,7 +166,8 @@ void recurse(simdjson::dom::element element, stat_t &s, size_t depth) { stat_t simdjson_compute_stats(const simdjson::padded_string &p) { stat_t s{}; simdjson::dom::parser parser; - auto [doc, error] = parser.parse(p); + simdjson::dom::element doc; + auto error = parser.parse(p).get(doc); if (error) { s.valid = false; std::cerr << error << std::endl; @@ -217,7 +218,8 @@ int main(int argc, char *argv[]) { const char *filename = result["file"].as().c_str(); - auto [p, error] = simdjson::padded_string::load(filename); + simdjson::padded_string p; + auto error = simdjson::padded_string::load(filename).get(p); if (error) { std::cerr << "Could not load the file " << filename << std::endl; return EXIT_FAILURE; diff --git a/tools/minify.cpp b/tools/minify.cpp index f9f264c42..2e2acebd2 100644 --- a/tools/minify.cpp +++ b/tools/minify.cpp @@ -56,7 +56,8 @@ int main(int argc, char *argv[]) { std::string filename = result["file"].as(); - auto [p, error] = simdjson::padded_string::load(filename); + simdjson::padded_string p; + auto error = simdjson::padded_string::load(filename).get(p); if (error) { std::cerr << "Could not load the file " << filename << std::endl; return EXIT_FAILURE; From 0c9dc11550949c7a9f63ee1ed1a39c8ac1b4c91c Mon Sep 17 00:00:00 2001 From: John Keiser Date: Sun, 21 Jun 2020 16:16:27 -0700 Subject: [PATCH 07/10] Use really_inline to help g++ detect initialized variable --- tests/test_macros.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_macros.h b/tests/test_macros.h index 661e50b81..5c46c0765 100644 --- a/tests/test_macros.h +++ b/tests/test_macros.h @@ -19,19 +19,19 @@ const char *TRUENULL_JSON = SIMDJSON_BENCHMARK_SMALLDATA_DIR "truenull.json"; // For the ASSERT_EQUAL macro template -bool equals_expected(T actual, S expected) { +really_inline bool equals_expected(T actual, S expected) { return actual == T(expected); } template<> -bool equals_expected(const char *actual, const char *expected) { +really_inline bool equals_expected(const char *actual, const char *expected) { return !strcmp(actual, expected); } -simdjson::error_code to_error_code(simdjson::error_code error) { +really_inline simdjson::error_code to_error_code(simdjson::error_code error) { return error; } template -simdjson::error_code to_error_code(const simdjson::simdjson_result &result) { +really_inline simdjson::error_code to_error_code(const simdjson::simdjson_result &result) { return result.error(); } From 12ccdcf858c630992b5396f5f109e54032f89477 Mon Sep 17 00:00:00 2001 From: John Keiser Date: Tue, 23 Jun 2020 08:49:47 -0700 Subject: [PATCH 08/10] Include document_stream line in parse_many docs --- doc/basics.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/basics.md b/doc/basics.md index e1bb0708c..59c0e8d9c 100644 --- a/doc/basics.md +++ b/doc/basics.md @@ -489,7 +489,8 @@ Here is a simple example, given "x.json" with this content: ```c++ dom::parser parser; -for (dom::element doc : parser.load_many(filename)) { +dom::document_stream docs = parser.load_many(filename); +for (dom::element doc : docs) { cout << doc["foo"] << endl; } // Prints 1 2 3 From 2d84b6f6d90c29967393b67c71683b4d6f298ae9 Mon Sep 17 00:00:00 2001 From: John Keiser Date: Sun, 21 Jun 2020 11:32:11 -0700 Subject: [PATCH 09/10] Make simdjson_result.is() return bool --- include/simdjson/dom/element.h | 18 ++++++------- include/simdjson/inline/element.h | 45 +++++++++++++------------------ tests/basictests.cpp | 22 ++------------- tests/cast_tester.h | 42 ++++++++--------------------- 4 files changed, 40 insertions(+), 87 deletions(-) diff --git a/include/simdjson/dom/element.h b/include/simdjson/dom/element.h index 6d9af51dd..acd466fca 100644 --- a/include/simdjson/dom/element.h +++ b/include/simdjson/dom/element.h @@ -455,7 +455,7 @@ public: really_inline simdjson_result type() const noexcept; template - really_inline simdjson_result is() const noexcept; + really_inline bool is() const noexcept; template really_inline simdjson_result get() const noexcept; template @@ -470,14 +470,14 @@ public: really_inline simdjson_result get_double() const noexcept; really_inline simdjson_result get_bool() const noexcept; - really_inline simdjson_result is_array() const noexcept; - really_inline simdjson_result is_object() const noexcept; - really_inline simdjson_result is_string() const noexcept; - really_inline simdjson_result is_int64_t() const noexcept; - really_inline simdjson_result is_uint64_t() const noexcept; - really_inline simdjson_result is_double() const noexcept; - really_inline simdjson_result is_bool() const noexcept; - really_inline simdjson_result is_null() const noexcept; + really_inline bool is_array() const noexcept; + really_inline bool is_object() const noexcept; + really_inline bool is_string() const noexcept; + really_inline bool is_int64_t() const noexcept; + really_inline bool is_uint64_t() const noexcept; + really_inline bool is_double() const noexcept; + really_inline bool is_bool() const noexcept; + really_inline bool is_null() const noexcept; really_inline simdjson_result operator[](const std::string_view &key) const noexcept; really_inline simdjson_result operator[](const char *key) const noexcept; diff --git a/include/simdjson/inline/element.h b/include/simdjson/inline/element.h index 690f40ff6..fc983eedb 100644 --- a/include/simdjson/inline/element.h +++ b/include/simdjson/inline/element.h @@ -24,9 +24,8 @@ inline simdjson_result simdjson_result::type() } template -really_inline simdjson_result simdjson_result::is() const noexcept { - if (error()) { return error(); } - return first.is(); +really_inline bool simdjson_result::is() const noexcept { + return !error() && first.is(); } template really_inline simdjson_result simdjson_result::get() const noexcept { @@ -72,38 +71,30 @@ really_inline simdjson_result simdjson_result::get_bool() co return first.get_bool(); } -really_inline simdjson_result simdjson_result::is_array() const noexcept { - if (error()) { return error(); } - return first.is_array(); +really_inline bool simdjson_result::is_array() const noexcept { + return !error() && first.is_array(); } -really_inline simdjson_result simdjson_result::is_object() const noexcept { - if (error()) { return error(); } - return first.is_object(); +really_inline bool simdjson_result::is_object() const noexcept { + return !error() && first.is_object(); } -really_inline simdjson_result simdjson_result::is_string() const noexcept { - if (error()) { return error(); } - return first.is_string(); +really_inline bool simdjson_result::is_string() const noexcept { + return !error() && first.is_string(); } -really_inline simdjson_result simdjson_result::is_int64_t() const noexcept { - if (error()) { return error(); } - return first.is_int64_t(); +really_inline bool simdjson_result::is_int64_t() const noexcept { + return !error() && first.is_int64_t(); } -really_inline simdjson_result simdjson_result::is_uint64_t() const noexcept { - if (error()) { return error(); } - return first.is_uint64_t(); +really_inline bool simdjson_result::is_uint64_t() const noexcept { + return !error() && first.is_uint64_t(); } -really_inline simdjson_result simdjson_result::is_double() const noexcept { - if (error()) { return error(); } - return first.is_double(); +really_inline bool simdjson_result::is_double() const noexcept { + return !error() && first.is_double(); } -really_inline simdjson_result simdjson_result::is_bool() const noexcept { - if (error()) { return error(); } - return first.is_bool(); +really_inline bool simdjson_result::is_bool() const noexcept { + return !error() && first.is_bool(); } -really_inline simdjson_result simdjson_result::is_null() const noexcept { - if (error()) { return error(); } - return first.is_null(); +really_inline bool simdjson_result::is_null() const noexcept { + return !error() && first.is_null(); } really_inline simdjson_result simdjson_result::operator[](const std::string_view &key) const noexcept { diff --git a/tests/basictests.cpp b/tests/basictests.cpp index d89eb5a9c..ec7fe553b 100644 --- a/tests/basictests.cpp +++ b/tests/basictests.cpp @@ -1503,27 +1503,9 @@ namespace type_tests { std::cout << " test_is_null() expecting " << expected_is_null << std::endl; // Grab the element out and check success dom::element element = result.first; - bool actual_is_null; - auto error = result.is_null().get(actual_is_null); - ASSERT_SUCCESS(error); - ASSERT_EQUAL(actual_is_null, expected_is_null); + ASSERT_EQUAL(result.is_null(), expected_is_null); - actual_is_null = element.is_null(); - ASSERT_EQUAL(actual_is_null, expected_is_null); - -#if SIMDJSON_EXCEPTIONS - - try { - - actual_is_null = result.is_null(); - ASSERT_EQUAL(actual_is_null, expected_is_null); - - } catch(simdjson_error &e) { - std::cerr << e.error() << std::endl; - return false; - } - -#endif // SIMDJSON_EXCEPTIONS + ASSERT_EQUAL(element.is_null(), expected_is_null); return true; } diff --git a/tests/cast_tester.h b/tests/cast_tester.h index 00e90de74..55eda96ae 100644 --- a/tests/cast_tester.h +++ b/tests/cast_tester.h @@ -40,7 +40,6 @@ public: bool test_is(element element, bool expected); bool test_is(simdjson_result element, bool expected); - bool test_is_error(simdjson_result element, error_code expected_error); bool test_named_get(element element, T expected = {}); bool test_named_get(simdjson_result element, T expected = {}); @@ -49,13 +48,12 @@ public: bool test_named_is(element element, bool expected); bool test_named_is(simdjson_result element, bool expected); - bool test_named_is_error(simdjson_result element, error_code expected_error); private: simdjson_result named_get(element element); simdjson_result named_get(simdjson_result element); bool named_is(element element); - simdjson_result named_is(simdjson_result element); + bool named_is(simdjson_result element); bool assert_equal(const T& expected, const T& actual); }; @@ -206,16 +204,7 @@ bool cast_tester::test_is(element element, bool expected) { template bool cast_tester::test_is(simdjson_result element, bool expected) { - bool actual; - ASSERT_SUCCESS(element.is().get(actual)); - ASSERT_EQUAL(actual, expected); - return true; -} - -template -bool cast_tester::test_is_error(simdjson_result element, error_code expected_error) { - UNUSED bool actual; - ASSERT_EQUAL(element.is().get(actual), expected_error); + ASSERT_EQUAL(element.is(), expected); return true; } @@ -227,16 +216,7 @@ bool cast_tester::test_named_is(element element, bool expected) { template bool cast_tester::test_named_is(simdjson_result element, bool expected) { - bool actual; - ASSERT_SUCCESS(named_is(element).get(actual)); - ASSERT_EQUAL(actual, expected); - return true; -} - -template -bool cast_tester::test_named_is_error(simdjson_result element, error_code expected_error) { - bool actual; - ASSERT_EQUAL(named_is(element).get(actual), expected_error); + ASSERT_EQUAL(named_is(element), expected); return true; } @@ -267,14 +247,14 @@ template<> bool cast_tester::named_is(element element) { return element template<> bool cast_tester::named_is(element element) { return element.is_double(); } template<> bool cast_tester::named_is(element element) { return element.is_bool(); } -template<> simdjson_result cast_tester::named_is(simdjson_result element) { return element.is_array(); } -template<> simdjson_result cast_tester::named_is(simdjson_result element) { return element.is_object(); } -template<> simdjson_result cast_tester::named_is(simdjson_result element) { return element.is_string(); } -template<> simdjson_result cast_tester::named_is(simdjson_result element) { return element.is_string(); } -template<> simdjson_result cast_tester::named_is(simdjson_result element) { return element.is_uint64_t(); } -template<> simdjson_result cast_tester::named_is(simdjson_result element) { return element.is_int64_t(); } -template<> simdjson_result cast_tester::named_is(simdjson_result element) { return element.is_double(); } -template<> simdjson_result cast_tester::named_is(simdjson_result element) { return element.is_bool(); } +template<> bool cast_tester::named_is(simdjson_result element) { return element.is_array(); } +template<> bool cast_tester::named_is(simdjson_result element) { return element.is_object(); } +template<> bool cast_tester::named_is(simdjson_result element) { return element.is_string(); } +template<> bool cast_tester::named_is(simdjson_result element) { return element.is_string(); } +template<> bool cast_tester::named_is(simdjson_result element) { return element.is_uint64_t(); } +template<> bool cast_tester::named_is(simdjson_result element) { return element.is_int64_t(); } +template<> bool cast_tester::named_is(simdjson_result element) { return element.is_double(); } +template<> bool cast_tester::named_is(simdjson_result element) { return element.is_bool(); } template bool cast_tester::assert_equal(const T& expected, const T& actual) { ASSERT_EQUAL(expected, actual); From e369d45b9c2093b04861cfba97dbe716c798e545 Mon Sep 17 00:00:00 2001 From: John Keiser Date: Tue, 23 Jun 2020 09:48:17 -0700 Subject: [PATCH 10/10] Fix non-compileable examples --- include/simdjson/dom/array.h | 2 +- include/simdjson/dom/element.h | 14 +++++++------- include/simdjson/dom/object.h | 14 +++++++------- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/include/simdjson/dom/array.h b/include/simdjson/dom/array.h index 55d3ec8a8..a54383b41 100644 --- a/include/simdjson/dom/array.h +++ b/include/simdjson/dom/array.h @@ -68,7 +68,7 @@ public: * Get the value associated with the given JSON pointer. * * dom::parser parser; - * array a = parser.parse(R"([ { "foo": { "a": [ 10, 20, 30 ] }} ])"); + * array a = parser.parse(R"([ { "foo": { "a": [ 10, 20, 30 ] }} ])"_padded); * a.at("0/foo/a/1") == 20 * a.at("0")["foo"]["a"].at(1) == 20 * diff --git a/include/simdjson/dom/element.h b/include/simdjson/dom/element.h index 9a9ce6f13..c979ba661 100644 --- a/include/simdjson/dom/element.h +++ b/include/simdjson/dom/element.h @@ -336,8 +336,8 @@ public: * The key will be matched against **unescaped** JSON: * * dom::parser parser; - * parser.parse(R"({ "a\n": 1 })")["a\n"].get().first == 1 - * parser.parse(R"({ "a\n": 1 })")["a\\n"].get().error() == NO_SUCH_FIELD + * parser.parse(R"({ "a\n": 1 })"_padded)["a\n"].get().first == 1 + * parser.parse(R"({ "a\n": 1 })"_padded)["a\\n"].get().error() == NO_SUCH_FIELD * * @return The value associated with this field, or: * - NO_SUCH_FIELD if the field does not exist in the object @@ -351,8 +351,8 @@ public: * The key will be matched against **unescaped** JSON: * * dom::parser parser; - * parser.parse(R"({ "a\n": 1 })")["a\n"].get().first == 1 - * parser.parse(R"({ "a\n": 1 })")["a\\n"].get().error() == NO_SUCH_FIELD + * parser.parse(R"({ "a\n": 1 })"_padded)["a\n"].get().first == 1 + * parser.parse(R"({ "a\n": 1 })"_padded)["a\\n"].get().error() == NO_SUCH_FIELD * * @return The value associated with this field, or: * - NO_SUCH_FIELD if the field does not exist in the object @@ -364,7 +364,7 @@ public: * Get the value associated with the given JSON pointer. * * dom::parser parser; - * element doc = parser.parse(R"({ "foo": { "a": [ 10, 20, 30 ] }})"); + * element doc = parser.parse(R"({ "foo": { "a": [ 10, 20, 30 ] }})"_padded); * doc.at("/foo/a/1") == 20 * doc.at("/")["foo"]["a"].at(1) == 20 * doc.at("")["foo"]["a"].at(1) == 20 @@ -391,8 +391,8 @@ public: * The key will be matched against **unescaped** JSON: * * dom::parser parser; - * parser.parse(R"({ "a\n": 1 })")["a\n"].get().first == 1 - * parser.parse(R"({ "a\n": 1 })")["a\\n"].get().error() == NO_SUCH_FIELD + * parser.parse(R"({ "a\n": 1 })"_padded)["a\n"].get().first == 1 + * parser.parse(R"({ "a\n": 1 })"_padded)["a\\n"].get().error() == NO_SUCH_FIELD * * @return The value associated with this field, or: * - NO_SUCH_FIELD if the field does not exist in the object diff --git a/include/simdjson/dom/object.h b/include/simdjson/dom/object.h index 301dad945..3be058eb4 100644 --- a/include/simdjson/dom/object.h +++ b/include/simdjson/dom/object.h @@ -101,8 +101,8 @@ public: * The key will be matched against **unescaped** JSON: * * dom::parser parser; - * parser.parse(R"({ "a\n": 1 })")["a\n"].get().first == 1 - * parser.parse(R"({ "a\n": 1 })")["a\\n"].get().error() == NO_SUCH_FIELD + * parser.parse(R"({ "a\n": 1 })"_padded)["a\n"].get().first == 1 + * parser.parse(R"({ "a\n": 1 })"_padded)["a\\n"].get().error() == NO_SUCH_FIELD * * This function has linear-time complexity: the keys are checked one by one. * @@ -118,8 +118,8 @@ public: * The key will be matched against **unescaped** JSON: * * dom::parser parser; - * parser.parse(R"({ "a\n": 1 })")["a\n"].get().first == 1 - * parser.parse(R"({ "a\n": 1 })")["a\\n"].get().error() == NO_SUCH_FIELD + * parser.parse(R"({ "a\n": 1 })"_padded)["a\n"].get().first == 1 + * parser.parse(R"({ "a\n": 1 })"_padded)["a\\n"].get().error() == NO_SUCH_FIELD * * This function has linear-time complexity: the keys are checked one by one. * @@ -133,7 +133,7 @@ public: * Get the value associated with the given JSON pointer. * * dom::parser parser; - * object obj = parser.parse(R"({ "foo": { "a": [ 10, 20, 30 ] }})"); + * object obj = parser.parse(R"({ "foo": { "a": [ 10, 20, 30 ] }})"_padded); * obj.at("foo/a/1") == 20 * obj.at("foo")["a"].at(1) == 20 * @@ -151,8 +151,8 @@ public: * The key will be matched against **unescaped** JSON: * * dom::parser parser; - * parser.parse(R"({ "a\n": 1 })")["a\n"].get().first == 1 - * parser.parse(R"({ "a\n": 1 })")["a\\n"].get().error() == NO_SUCH_FIELD + * parser.parse(R"({ "a\n": 1 })"_padded)["a\n"].get().first == 1 + * parser.parse(R"({ "a\n": 1 })"_padded)["a\\n"].get().error() == NO_SUCH_FIELD * * This function has linear-time complexity: the keys are checked one by one. *