From 0b21203141bd31229c77e4b1e8b22f523a4a69e0 Mon Sep 17 00:00:00 2001 From: John Keiser Date: Mon, 24 Feb 2020 20:59:38 -0800 Subject: [PATCH] Document navigation API --- .gitignore | 1 + benchmark/CMakeLists.txt | 6 +- benchmark/bench_dom_api.cpp | 233 +++++++ include/simdjson/document.h | 1042 ++++++++++++++++++++++++---- include/simdjson/error.h | 5 +- include/simdjson/inline/document.h | 587 ++++++++++++++-- src/document.cpp | 6 +- src/error.cpp | 3 + src/generic/stage1_find_marks.h | 2 +- src/haswell/simd.h | 5 +- src/westmere/simd.h | 5 +- tests/CMakeLists.txt | 3 +- tests/basictests.cpp | 207 ++++++ tests/readme_examples.cpp | 10 + 14 files changed, 1926 insertions(+), 189 deletions(-) create mode 100644 benchmark/bench_dom_api.cpp diff --git a/.gitignore b/.gitignore index 33bbca04e..0f46c1496 100644 --- a/.gitignore +++ b/.gitignore @@ -57,6 +57,7 @@ objs /allparsingcompetition /basictests /benchfeatures +/benchmark/bench_dom_api /benchmark/bench_parse_call /benchmark/get_corpus_benchmark /benchmark/parse diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index 5bc552e21..3e4f95f9a 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -12,6 +12,10 @@ add_executable(perfdiff perfdiff.cpp) # Google Benchmarks if (SIMDJSON_GOOGLE_BENCHMARKS) - add_cpp_benchmark(bench_parse_call bench_parse_call.cpp) + add_cpp_benchmark(bench_parse_call) target_link_libraries(bench_parse_call benchmark::benchmark) + + add_cpp_benchmark(bench_dom_api) + target_link_libraries(bench_dom_api benchmark::benchmark) + target_compile_definitions(bench_dom_api PRIVATE JSON_TEST_PATH="${PROJECT_SOURCE_DIR}/jsonexamples/twitter.json") endif() \ No newline at end of file diff --git a/benchmark/bench_dom_api.cpp b/benchmark/bench_dom_api.cpp new file mode 100644 index 000000000..dcc01a9e8 --- /dev/null +++ b/benchmark/bench_dom_api.cpp @@ -0,0 +1,233 @@ +#include +#include "simdjson/document.h" +#include "simdjson/jsonparser.h" +using namespace simdjson; +using namespace benchmark; +using namespace std; + +#ifndef JSON_TEST_PATH +#define JSON_TEST_PATH "jsonexamples/twitter.json" +#endif + +const padded_string EMPTY_ARRAY("[]", 2); + +static void twitter_count(State& state) { + // Prints the number of results in twitter.json + document doc = document::parse(get_corpus(JSON_TEST_PATH)); + for (auto _ : state) { + uint64_t result_count = doc["search_metadata"]["count"]; + if (result_count != 100) { return; } + } +} +BENCHMARK(twitter_count); + +static void error_code_twitter_count(State& state) noexcept { + // Prints the number of results in twitter.json + document doc = document::parse(get_corpus(JSON_TEST_PATH)); + for (auto _ : state) { + auto [value, error] = doc["search_metadata"]["count"]; + if (error) { return; } + if (uint64_t(value) != 100) { return; } + } +} +BENCHMARK(error_code_twitter_count); + +static void iterator_twitter_count(State& state) { + // Prints the number of results in twitter.json + document doc = document::parse(get_corpus(JSON_TEST_PATH)); + for (auto _ : state) { + document::iterator iter(doc); + // uint64_t result_count = doc["search_metadata"]["count"]; + if (!iter.move_to_key("search_metadata")) { return; } + if (!iter.move_to_key("count")) { return; } + if (!iter.is_integer()) { return; } + int64_t result_count = iter.get_integer(); + + if (result_count != 100) { return; } + } +} +BENCHMARK(iterator_twitter_count); + +static void twitter_default_profile(State& state) { + // Count unique users with a default profile. + document doc = document::parse(get_corpus(JSON_TEST_PATH)); + for (auto _ : state) { + set default_users; + for (document::object tweet : doc["statuses"].as_array()) { + document::object user = tweet["user"]; + if (user["default_profile"]) { + default_users.insert(user["screen_name"]); + } + } + if (default_users.size() != 86) { return; } + } +} +BENCHMARK(twitter_default_profile); + +static void error_code_twitter_default_profile(State& state) noexcept { + // Count unique users with a default profile. + document doc = document::parse(get_corpus(JSON_TEST_PATH)); + for (auto _ : state) { + set default_users; + + auto [tweets, error] = doc["statuses"].as_array(); + if (error) { return; } + for (document::element tweet : tweets) { + auto [user, error2] = tweet["user"].as_object(); + if (error2) { return; } + auto [default_profile, error3] = user["default_profile"].as_bool(); + if (error3) { return; } + if (default_profile) { + auto [screen_name, error4] = user["screen_name"].as_string(); + if (error4) { return; } + default_users.insert(screen_name); + } + } + + if (default_users.size() != 86) { return; } + } +} +BENCHMARK(error_code_twitter_default_profile); + +static void iterator_twitter_default_profile(State& state) { + // Count unique users with a default profile. + document doc = document::parse(get_corpus(JSON_TEST_PATH)); + for (auto _ : state) { + set default_users; + document::iterator iter(doc); + + // for (document::object tweet : doc["statuses"].as_array()) { + if (!(iter.move_to_key("statuses") && iter.is_array())) { return; } + if (iter.down()) { // first status + do { + + // document::object user = tweet["user"]; + if (!(iter.move_to_key("user") && iter.is_object())) { return; } + + // if (user["default_profile"]) { + if (iter.move_to_key("default_profile")) { + if (iter.is_true()) { + if (!iter.up()) { return; } // back to user + + // default_users.insert(user["screen_name"]); + if (!(iter.move_to_key("screen_name") && iter.is_string())) { return; } + default_users.insert(string_view(iter.get_string(), iter.get_string_length())); + } + if (!iter.up()) { return; } // back to user + } + + if (!iter.up()) { return; } // back to status + + } while (iter.next()); // next status + } + + if (default_users.size() != 86) { return; } + } +} +BENCHMARK(iterator_twitter_default_profile); + +static void twitter_image_sizes(State& state) { + // Count unique image sizes + document doc = document::parse(get_corpus(JSON_TEST_PATH)); + for (auto _ : state) { + set> image_sizes; + for (document::object tweet : doc["statuses"].as_array()) { + auto [media, not_found] = tweet["entities"]["media"]; + if (!not_found) { + for (document::object image : media.as_array()) { + for (auto [key, size] : image["sizes"].as_object()) { + image_sizes.insert({ size["w"], size["h"] }); + } + } + } + } + if (image_sizes.size() != 15) { return; }; + } +} +BENCHMARK(twitter_image_sizes); + +static void error_code_twitter_image_sizes(State& state) noexcept { + // Count unique image sizes + document doc = document::parse(get_corpus(JSON_TEST_PATH)); + for (auto _ : state) { + set> image_sizes; + auto [statuses, error] = doc["statuses"].as_array(); + if (error) { return; } + for (document::element tweet : statuses) { + auto [images, not_found] = tweet["entities"]["media"].as_array(); + if (!not_found) { + for (document::element image : images) { + auto [sizes, error2] = image["sizes"].as_object(); + if (error2) { return; } + for (auto [key, size] : sizes) { + auto [width, error3] = size["w"].as_uint64_t(); + auto [height, error4] = size["h"].as_uint64_t(); + if (error3 || error4) { return; } + image_sizes.insert({ width, height }); + } + } + } + } + if (image_sizes.size() != 15) { return; }; + } +} +BENCHMARK(error_code_twitter_image_sizes); + +static void iterator_twitter_image_sizes(State& state) { + // Count unique image sizes + document doc = document::parse(get_corpus(JSON_TEST_PATH)); + for (auto _ : state) { + set> image_sizes; + document::iterator iter(doc); + + // for (document::object tweet : doc["statuses"].as_array()) { + if (!(iter.move_to_key("statuses") && iter.is_array())) { return; } + if (iter.down()) { // first status + do { + + // auto [media, not_found] = tweet["entities"]["media"]; + // if (!not_found) { + if (iter.move_to_key("entities")) { + if (!iter.is_object()) { return; } + if (iter.move_to_key("media")) { + if (!iter.is_array()) { return; } + + // for (document::object image : media.as_array()) { + if (iter.down()) { // first media + do { + + // for (auto [key, size] : image["sizes"].as_object()) { + if (!(iter.move_to_key("sizes") && iter.is_object())) { return; } + if (iter.down()) { // first size + do { + iter.move_to_value(); + + // image_sizes.insert({ size["w"], size["h"] }); + if (!(iter.move_to_key("w")) && !iter.is_integer()) { return; } + uint64_t width = iter.get_integer(); + if (!iter.up()) { return; } // back to size + if (!(iter.move_to_key("h")) && !iter.is_integer()) { return; } + uint64_t height = iter.get_integer(); + if (!iter.up()) { return; } // back to size + image_sizes.insert({ width, height }); + + } while (iter.next()); // next size + if (!iter.up()) { return; } // back to sizes + } + if (!iter.up()) { return; } // back to image + } while (iter.next()); // next image + if (!iter.up()) { return; } // back to media + } + if (!iter.up()) { return; } // back to entities + } + if (!iter.up()) { return; } // back to status + } + } while (iter.next()); // next status + } + + if (image_sizes.size() != 15) { return; }; + } +} +BENCHMARK(iterator_twitter_image_sizes); + +BENCHMARK_MAIN(); \ No newline at end of file diff --git a/include/simdjson/document.h b/include/simdjson/document.h index 563e7a691..82eb19f48 100644 --- a/include/simdjson/document.h +++ b/include/simdjson/document.h @@ -4,121 +4,823 @@ #include #include #include +#include #include "simdjson/common_defs.h" #include "simdjson/simdjson.h" #include "simdjson/padded_string.h" -#define JSON_VALUE_MASK 0xFFFFFFFFFFFFFF +#define JSON_VALUE_MASK 0x00FFFFFFFFFFFFFF #define DEFAULT_MAX_DEPTH 1024 // a JSON document with a depth exceeding 1024 is probably de facto invalid namespace simdjson { template class document_iterator; -class document_parser; +/** + * A parsed JSON document. + * + * This class cannot be copied, only moved, to avoid unintended allocations. + */ class document { public: - // create a document container with zero capacity, parser will allocate capacity as needed - document()=default; - ~document()=default; + /** + * Create a document container with zero capacity. + * + * The parser will allocate capacity as needed. + */ + document() noexcept=default; + ~document() noexcept=default; - // this is a move only class - document(document &&p) = default; - document(const document &p) = delete; - document &operator=(document &&o) = default; - document &operator=(const document &o) = delete; + /** + * Take another document's buffers. + * + * @param other The document to take. Its capacity is zeroed and it is invalidated. + */ + document(document &&other) noexcept = default; + document(const document &) = delete; // Disallow copying + /** + * Take another document's buffers. + * + * @param other The document to take. Its capacity is zeroed. + */ + document &operator=(document &&other) noexcept = default; + document &operator=(const document &) = delete; // Disallow copying - // Nested classes. See definitions later in file. - using iterator = document_iterator; + // Nested classes + class element; + class array; + class object; + class key_value_pair; class parser; + + template + class element_result; class doc_result; class doc_ref_result; - // - // Tell whether this document has been parsed, or is just empty. - // - bool is_initialized() { - return tape && string_buf; - } + // Nested classes. See definitions later in file. + using iterator = document_iterator; - // print the json to std::ostream (should be valid) - // return false if the tape is likely wrong (e.g., you did not parse a valid - // JSON). - WARN_UNUSED - bool print_json(std::ostream &os, size_t max_depth=DEFAULT_MAX_DEPTH) const; - WARN_UNUSED - bool dump_raw_tape(std::ostream &os) const; + /** + * Get the root element of this document as a JSON array. + */ + element root() const noexcept; + /** + * Get the root element of this document as a JSON array. + */ + element_result as_array() const noexcept; + /** + * Get the root element of this document as a JSON object. + */ + element_result as_object() const noexcept; + /** + * Get the root element of this document. + */ + operator element() const noexcept; + /** + * Read the root element of this document as a JSON array. + * + * @return The JSON array. + * @exception invalid_json(UNEXPECTED_TYPE) if the JSON element is not an array + */ + operator array() const noexcept(false); + /** + * Read this element as a JSON object (key/value pairs). + * + * @return The JSON object. + * @exception invalid_json(UNEXPECTED_TYPE) if the JSON element is not an object + */ + operator object() const noexcept(false); - // - // Parse a JSON document. - // - // If you will be parsing more than one JSON document, it's recommended to create a - // document::parser object instead, keeping internal buffers around for efficiency reasons. - // - // Throws invalid_json if the JSON is invalid. - // + /** + * Get the value associated with the given key. + * + * The key will be matched against **unescaped** JSON: + * + * document::parse(R"({ "a\n": 1 })")["a\n"].as_uint64_t().value == 1 + * document::parse(R"({ "a\n": 1 })")["a\\n"].as_uint64_t().error == NO_SUCH_FIELD + * + * @return The value associated with the given key, or: + * - NO_SUCH_FIELD if the field does not exist in the object + * - UNEXPECTED_TYPE if the document is not an object + */ + element_result operator[](const std::string_view &s) const noexcept; + /** + * Get the value associated with the given key. + * + * The key will be matched against **unescaped** JSON: + * + * document::parse(R"({ "a\n": 1 })")["a\n"].as_uint64_t().value == 1 + * document::parse(R"({ "a\n": 1 })")["a\\n"].as_uint64_t().error == NO_SUCH_FIELD + * + * @return The value associated with this field, or: + * - NO_SUCH_FIELD if the field does not exist in the object + * - UNEXPECTED_TYPE if the document is not an object + */ + element_result operator[](const char *s) const noexcept; + + /** + * Print this JSON to a std::ostream. + * + * @param os the stream to output to. + * @param max_depth the maximum JSON depth to output. + * @return false if the tape is likely wrong (e.g., you did not parse a valid JSON). + */ + bool print_json(std::ostream &os, size_t max_depth=DEFAULT_MAX_DEPTH) const noexcept; + /** + * Dump the raw tape for debugging. + * + * @param os the stream to output to. + * @return false if the tape is likely wrong (e.g., you did not parse a valid JSON). + */ + bool dump_raw_tape(std::ostream &os) const noexcept; + + /** + * Parse a JSON document and return a reference to it. + * + * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what + * those bytes are initialized to, as long as they are allocated. If realloc_if_needed is true, + * it is assumed that the buffer does *not* have enough padding, and it is reallocated, enlarged + * and copied before parsing. + * + * @param buf The JSON to parse. Must have at least len + SIMDJSON_PADDING allocated bytes, unless + * realloc_if_needed is true. + * @param len The length of the JSON. + * @param realloc_if_needed Whether to reallocate and enlarge the JSON buffer to add padding. + * @return the document, or an error if the JSON is invalid. + */ static doc_result parse(const uint8_t *buf, size_t len, bool realloc_if_needed = true) noexcept; + + /** + * Parse a JSON document. + * + * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what + * those bytes are initialized to, as long as they are allocated. If realloc_if_needed is true, + * it is assumed that the buffer does *not* have enough padding, and it is reallocated, enlarged + * and copied before parsing. + * + * @param buf The JSON to parse. Must have at least len + SIMDJSON_PADDING allocated bytes, unless + * realloc_if_needed is true. + * @param len The length of the JSON. + * @param realloc_if_needed Whether to reallocate and enlarge the JSON buffer to add padding. + * @return the document, or an error if the JSON is invalid. + */ static doc_result parse(const char *buf, size_t len, bool realloc_if_needed = true) noexcept; - static doc_result parse(const std::string &s, bool realloc_if_needed = true) noexcept; + + /** + * Parse a JSON document. + * + * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what + * those bytes are initialized to, as long as they are allocated. If `str.capacity() - str.size() + * < SIMDJSON_PADDING`, the string will be copied to a string with larger capacity before parsing. + * + * @param s The JSON to parse. Must have at least len + SIMDJSON_PADDING allocated bytes, or + * a new string will be created with the extra padding. + * @return the document, or an error if the JSON is invalid. + */ + static doc_result parse(const std::string &s) noexcept; + + /** + * Parse a JSON document. + * + * @param s The JSON to parse. + * @return the document, or an error if the JSON is invalid. + */ static doc_result parse(const padded_string &s) noexcept; + // We do not want to allow implicit conversion from C string to std::string. - doc_result parse(const char *buf, bool realloc_if_needed = true) noexcept = delete; + doc_ref_result parse(const char *buf, bool realloc_if_needed = true) noexcept = delete; std::unique_ptr tape; std::unique_ptr string_buf;// should be at least byte_capacity private: + class tape_ref; + enum class tape_type; bool set_capacity(size_t len); }; // class document +/** + * A parsed, *owned* document, or an error if the parse failed. + * + * document &doc = document::parse(json); + * + * Returns an owned `document`. When the doc_result (or the document retrieved from it) goes out of + * scope, the document's memory is deallocated. + * + * ## Error Codes vs. Exceptions + * + * This result type allows the user to pick whether to use exceptions or not. + * + * Use like this to avoid exceptions: + * + * auto [doc, error] = document::parse(json); + * if (error) { exit(1); } + * + * Use like this if you'd prefer to use exceptions: + * + * document doc = document::parse(json); + * + */ class document::doc_result { -private: - doc_result(document &&_doc, error_code _error) : doc(std::move(_doc)), error(_error) { } - doc_result(document &&_doc) : doc(std::move(_doc)), error(SUCCESS) { } - doc_result(error_code _error) : doc(), error(_error) { } - friend class document; public: - ~doc_result()=default; - - operator bool() noexcept { return error == SUCCESS; } - operator document() { - if (!*this) { - throw invalid_json(error); - } - return std::move(doc); - } + /** + * The parsed document. This is *invalid* if there is an error. + */ document doc; + /** + * The error code, or SUCCESS (0) if there is no error. + */ error_code error; - const std::string &get_error_message() { - return error_message(error); - } -}; // class doc_result + + /** + * Return the document, or throw an exception if it is invalid. + * + * @return the document. + * @exception invalid_json if the document is invalid or there was an error parsing it. + */ + operator document() noexcept(false); + + /** + * Get the error message for the error. + */ + const std::string &get_error_message() const noexcept; + + ~doc_result() noexcept=default; + +private: + doc_result(document &&_doc, error_code _error) noexcept; + doc_result(document &&_doc) noexcept; + doc_result(error_code _error) noexcept; + friend class document; +}; // class document::doc_result /** - * The result of document::parser::parse(). Stores an error code and a document reference. - * - * Designed so that you can either check the error code before using the document, or use - * exceptions and use thedirectly and parse it, or - * - */ + * A parsed document reference, or an error if the parse failed. + * + * document &doc = document::parse(json); + * + * ## Document Ownership + * + * The `document &` refers to an internal document the parser reuses on each `parse()` call. It will + * become invalidated on the next `parse()`. + * + * This is more efficient for common cases where documents are parsed and used one at a time. If you + * need to keep the document around longer, you may *take* it from the parser by casting it: + * + * document doc = parser.parse(); // take ownership + * + * If you do this, the parser will automatically allocate a new document on the next `parse()` call. + * + * ## Error Codes vs. Exceptions + * + * This result type allows the user to pick whether to use exceptions or not. + * + * Use like this to avoid exceptions: + * + * auto [doc, error] = parser.parse(json); + * if (error) { exit(1); } + * + * Use like this if you'd prefer to use exceptions: + * + * document &doc = document::parse(json); + * + */ class document::doc_ref_result { public: - doc_ref_result(document &_doc, error_code _error) : doc(_doc), error(_error) { } + /** + * The parsed document. This is *invalid* if there is an error. + */ + document &doc; + /** + * The error code, or SUCCESS (0) if there is no error. + */ + error_code error; + + /** + * A reference to the document, or throw an exception if it is invalid. + * + * @return the document. + * @exception invalid_json if the document is invalid or there was an error parsing it. + */ + operator document&() noexcept(false); + + /** + * Get the error message for the error. + */ + const std::string &get_error_message() const noexcept; + ~doc_ref_result()=default; - operator bool() noexcept { return error == SUCCESS; } - operator document&() { - if (!*this) { - throw invalid_json(error); - } - return doc; - } - document& doc; +private: + doc_ref_result(document &_doc, error_code _error) noexcept; + friend class document::parser; +}; // class document::doc_ref_result + +/** + * The possible types in the tape. Internal only. + */ +enum class document::tape_type { + ROOT = 'r', + START_ARRAY = '[', + START_OBJECT = '{', + END_ARRAY = ']', + END_OBJECT = '}', + STRING = '"', + INT64 = 'l', + UINT64 = 'u', + DOUBLE = 'd', + TRUE_VALUE = 't', + FALSE_VALUE = 'f', + NULL_VALUE = 'n' +}; + +/** + * A reference to an element on the tape. Internal only. + */ +class document::tape_ref { +protected: + tape_ref() noexcept; + tape_ref(const document *_doc, size_t _json_index) noexcept; + size_t after_element() const noexcept; + tape_type type() const noexcept; + uint64_t tape_value() const noexcept; + template + T next_tape_value() const noexcept; + + /** The document this element references. */ + const document *doc; + + /** The index of this element on `doc.tape[]` */ + size_t json_index; + + friend class document::key_value_pair; +}; + +/** + * A JSON element. + * + * References an element in a JSON document, representing a JSON null, boolean, string, number, + * array or object. + */ +class document::element : protected document::tape_ref { +public: + /** Whether this element is a json `null`. */ + bool is_null() const noexcept; + /** Whether this is a JSON `true` or `false` */ + bool is_bool() const noexcept; + /** Whether this is a JSON number (e.g. 1, 1.0 or 1e2) */ + bool is_number() const noexcept; + /** Whether this is a JSON integer (e.g. 1 or -1, but *not* 1.0 or 1e2) */ + bool is_integer() const noexcept; + /** Whether this is a JSON string (e.g. "abc") */ + bool is_string() const noexcept; + /** Whether this is a JSON array (e.g. []) */ + bool is_array() const noexcept; + /** Whether this is a JSON array (e.g. []) */ + bool is_object() const noexcept; + + /** + * Read this element as a boolean (json `true` or `false`). + * + * @return The boolean value, or: + * - UNEXPECTED_TYPE error if the JSON element is not a boolean + */ + element_result as_bool() const noexcept; + + /** + * Read this element as a null-terminated string. + * + * Does *not* convert other types to a string; requires that the JSON type of the element was + * an actual string. + * + * @return A `string_view` into the string, or: + * - UNEXPECTED_TYPE error if the JSON element is not a string + */ + element_result as_c_str() const noexcept; + + /** + * Read this element as a C++ string_view (string with length). + * + * Does *not* convert other types to a string; requires that the JSON type of the element was + * an actual string. + * + * @return A `string_view` into the string, or: + * - UNEXPECTED_TYPE error if the JSON element is not a string + */ + element_result as_string() const noexcept; + + /** + * Read this element as an unsigned integer. + * + * @return The uninteger value, or: + * - UNEXPECTED_TYPE if the JSON element is not an integer + * - NUMBER_OUT_OF_RANGE if the integer doesn't fit in 64 bits or is negative + */ + element_result as_uint64_t() const noexcept; + + /** + * Read this element as a signed integer. + * + * @return The integer value, or: + * - UNEXPECTED_TYPE if the JSON element is not an integer + * - NUMBER_OUT_OF_RANGE if the integer doesn't fit in 64 bits + */ + element_result as_int64_t() const noexcept; + + /** + * Read this element as a floating point value. + * + * @return The double value, or: + * - UNEXPECTED_TYPE if the JSON element is not a number + */ + element_result as_double() const noexcept; + + /** + * Read this element as a JSON array. + * + * @return The array value, or: + * - UNEXPECTED_TYPE if the JSON element is not an array + */ + element_result as_array() const noexcept; + + /** + * Read this element as a JSON object (key/value pairs). + * + * @return The object value, or: + * - UNEXPECTED_TYPE if the JSON element is not an object + */ + element_result as_object() const noexcept; + + /** + * Read this element as a boolean. + * + * @return The boolean value + * @exception invalid_json(UNEXPECTED_TYPE) if the JSON element is not a boolean. + */ + operator bool() const noexcept(false); + + /** + * Read this element as a null-terminated string. + * + * Does *not* convert other types to a string; requires that the JSON type of the element was + * an actual string. + * + * @return The string value. + * @exception invalid_json(UNEXPECTED_TYPE) if the JSON element is not a string. + */ + explicit operator const char*() const noexcept(false); + + /** + * Read this element as a null-terminated string. + * + * Does *not* convert other types to a string; requires that the JSON type of the element was + * an actual string. + * + * @return The string value. + * @exception invalid_json(UNEXPECTED_TYPE) if the JSON element is not a string. + */ + operator std::string_view() const noexcept(false); + + /** + * Read this element as an unsigned integer. + * + * @return The integer value. + * @exception invalid_json(UNEXPECTED_TYPE) if the JSON element is not an integer + * @exception invalid_json(NUMBER_OUT_OF_RANGE) if the integer doesn't fit in 64 bits or is negative + */ + operator uint64_t() const noexcept(false); + /** + * Read this element as an signed integer. + * + * @return The integer value. + * @exception invalid_json(UNEXPECTED_TYPE) if the JSON element is not an integer + * @exception invalid_json(NUMBER_OUT_OF_RANGE) if the integer doesn't fit in 64 bits + */ + operator int64_t() const noexcept(false); + /** + * Read this element as an double. + * + * @return The double value. + * @exception invalid_json(UNEXPECTED_TYPE) if the JSON element is not a number + * @exception invalid_json(NUMBER_OUT_OF_RANGE) if the integer doesn't fit in 64 bits or is negative + */ + operator double() const noexcept(false); + /** + * Read this element as a JSON array. + * + * @return The JSON array. + * @exception invalid_json(UNEXPECTED_TYPE) if the JSON element is not an array + */ + operator document::array() const noexcept(false); + /** + * Read this element as a JSON object (key/value pairs). + * + * @return The JSON object. + * @exception invalid_json(UNEXPECTED_TYPE) if the JSON element is not an object + */ + operator document::object() const noexcept(false); + + /** + * Get the value associated with the given key. + * + * The key will be matched against **unescaped** JSON: + * + * document::parse(R"({ "a\n": 1 })")["a\n"].as_uint64_t().value == 1 + * document::parse(R"({ "a\n": 1 })")["a\\n"].as_uint64_t().error == NO_SUCH_FIELD + * + * @return The value associated with this field, or: + * - NO_SUCH_FIELD if the field does not exist in the object + * - UNEXPECTED_TYPE if the document is not an object + */ + element_result operator[](const std::string_view &s) const noexcept; + /** + * Get the value associated with the given key. + * + * Note: The key will be matched against **unescaped** JSON: + * + * document::parse(R"({ "a\n": 1 })")["a\n"].as_uint64_t().value == 1 + * document::parse(R"({ "a\n": 1 })")["a\\n"].as_uint64_t().error == NO_SUCH_FIELD + * + * @return The value associated with this field, or: + * - NO_SUCH_FIELD if the field does not exist in the object + * - UNEXPECTED_TYPE if the document is not an object + */ + element_result operator[](const char *s) const noexcept; + +private: + element() noexcept; + element(const document *_doc, size_t _json_index) noexcept; + friend class document; + template + friend class document::element_result; +}; + +/** + * Represents a JSON array. + */ +class document::array : protected document::tape_ref { +public: + class iterator : tape_ref { + public: + /** + * Get the actual value + */ + element operator*() const noexcept; + /** + * Get the next value. + * + * Part of the std::iterator interface. + */ + void operator++() noexcept; + /** + * Check if these values come from the same place in the JSON. + * + * Part of the std::iterator interface. + */ + bool operator!=(const iterator& other) const noexcept; + private: + iterator(const document *_doc, size_t _json_index) noexcept; + friend class array; + }; + + /** + * Return the first array element. + * + * Part of the std::iterable interface. + */ + iterator begin() const noexcept; + /** + * One past the last array element. + * + * Part of the std::iterable interface. + */ + iterator end() const noexcept; + +private: + array() noexcept; + array(const document *_doc, size_t _json_index) noexcept; + friend class document::element; + template + friend class document::element_result; +}; + +/** + * Represents a JSON object. + */ +class document::object : protected document::tape_ref { +public: + class iterator : protected document::tape_ref { + public: + /** + * Get the actual key/value pair + */ + const document::key_value_pair operator*() const noexcept; + /** + * Get the next key/value pair. + * + * Part of the std::iterator interface. + */ + void operator++() noexcept; + /** + * Check if these key value pairs come from the same place in the JSON. + * + * Part of the std::iterator interface. + */ + bool operator!=(const iterator& other) const noexcept; + /** + * Get the key of this key/value pair. + */ + std::string_view key() const noexcept; + /** + * Get the key of this key/value pair. + */ + const char *key_c_str() const noexcept; + /** + * Get the value of this key/value pair. + */ + element value() const noexcept; + private: + iterator(const document *_doc, size_t _json_index) noexcept; + friend class document::object; + }; + + /** + * Return the first key/value pair. + * + * Part of the std::iterable interface. + */ + iterator begin() const noexcept; + /** + * One past the last key/value pair. + * + * Part of the std::iterable interface. + */ + iterator end() const noexcept; + + /** + * Get the value associated with the given key. + * + * The key will be matched against **unescaped** JSON: + * + * document::parse(R"({ "a\n": 1 })")["a\n"].as_uint64_t().value == 1 + * document::parse(R"({ "a\n": 1 })")["a\\n"].as_uint64_t().error == NO_SUCH_FIELD + * + * @return The value associated with this field, or: + * - NO_SUCH_FIELD if the field does not exist in the object + */ + element_result operator[](const std::string_view &s) const noexcept; + /** + * Get the value associated with the given key. + * + * Note: The key will be matched against **unescaped** JSON: + * + * document::parse(R"({ "a\n": 1 })")["a\n"].as_uint64_t().value == 1 + * document::parse(R"({ "a\n": 1 })")["a\\n"].as_uint64_t().error == NO_SUCH_FIELD + * + * @return The value associated with this field, or: + * - NO_SUCH_FIELD if the field does not exist in the object + */ + element_result operator[](const char *s) const noexcept; + +private: + object() noexcept; + object(const document *_doc, size_t _json_index) noexcept; + friend class document::element; + template + friend class document::element_result; +}; + +/** + * Key/value pair in an object. + */ +class document::key_value_pair { +public: + std::string_view key; + document::element value; + +private: + key_value_pair(std::string_view _key, document::element _value) noexcept; + friend class document::object; +}; + + +/** + * The result of a JSON navigation or conversion, or an error (if the navigation or conversion + * failed). Allows the user to pick whether to use exceptions or not. + * + * Use like this to avoid exceptions: + * + * auto [str, error] = document::parse(json).root().as_string(); + * if (error) { exit(1); } + * cout << str; + * + * Use like this if you'd prefer to use exceptions: + * + * string str = document::parse(json).root(); + * cout << str; + * + */ +template +class document::element_result { +public: + /** The value */ + T value; + /** The error code (or 0 if there is no error) */ error_code error; - const std::string &get_error_message() noexcept { - return error_message(error); - } -}; // class document::doc_result + + operator T() const noexcept(false); + +private: + element_result(T value) noexcept; + element_result(error_code _error) noexcept; + friend class document; + friend class element; +}; + +// Add exception-throwing navigation / conversion methods to element_result +template<> +class document::element_result { +public: + /** The value */ + element value; + /** The error code (or 0 if there is no error) */ + error_code error; + + /** Whether this is a JSON `null` */ + element_result is_null() const noexcept; + element_result as_bool() const noexcept; + element_result as_string() const noexcept; + element_result as_c_str() const noexcept; + element_result as_uint64_t() const noexcept; + element_result as_int64_t() const noexcept; + element_result as_double() const noexcept; + element_result as_array() const noexcept; + element_result as_object() const noexcept; + + operator bool() const noexcept(false); + explicit operator const char*() const noexcept(false); + operator std::string_view() const noexcept(false); + operator uint64_t() const noexcept(false); + operator int64_t() const noexcept(false); + operator double() const noexcept(false); + operator array() const noexcept(false); + operator object() const noexcept(false); + + element_result operator[](const std::string_view &s) const noexcept; + element_result operator[](const char *s) const noexcept; + +private: + element_result(element value) noexcept; + element_result(error_code _error) noexcept; + friend class document; + friend class element; +}; + +// Add exception-throwing navigation methods to element_result +template<> +class document::element_result { +public: + /** The value */ + array value; + /** The error code (or 0 if there is no error) */ + error_code error; + + operator array() const noexcept(false); + + array::iterator begin() const noexcept(false); + array::iterator end() const noexcept(false); + +private: + element_result(array value) noexcept; + element_result(error_code _error) noexcept; + friend class document; + friend class element; +}; + +// Add exception-throwing navigation methods to element_result +template<> +class document::element_result { +public: + /** The value */ + object value; + /** The error code (or 0 if there is no error) */ + error_code error; + + operator object() const noexcept(false); + + object::iterator begin() const noexcept(false); + object::iterator end() const noexcept(false); + + element_result operator[](const std::string_view &s) const noexcept; + element_result operator[](const char *s) const noexcept; + +private: + element_result(object value) noexcept; + element_result(error_code _error) noexcept; + friend class document; + friend class element; +}; /** * A persistent document parser. @@ -126,6 +828,8 @@ public: * Use this if you intend to parse more than one document. It holds the internal memory necessary * to do parsing, as well as memory for a single document that is overwritten on each parse. * + * This class cannot be copied, only moved, to avoid unintended allocations. + * * @note This is not thread safe: one parser cannot produce two documents at the same time! */ class document::parser { @@ -136,40 +840,107 @@ public: parser()=default; ~parser()=default; - // this is a move only class - parser(document::parser &&p) = default; - parser(const document::parser &p) = delete; - parser &operator=(document::parser &&o) = default; - parser &operator=(const document::parser &o) = delete; + /** + * Take another parser's buffers and state. + * + * @param other The parser to take. Its capacity is zeroed. + */ + parser(document::parser &&other) = default; + parser(const document::parser &) = delete; // Disallow copying + /** + * Take another parser's buffers and state. + * + * @param other The parser to take. Its capacity is zeroed. + */ + parser &operator=(document::parser &&other) = default; + parser &operator=(const document::parser &) = delete; // Disallow copying + + /** + * Parse a JSON document and return a reference to it. + * + * The JSON document still lives in the parser: this is the most efficient way to parse JSON + * documents because it reuses the same buffers, but you *must* use the document before you + * destroy the parser or call parse() again. + * + * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what + * those bytes are initialized to, as long as they are allocated. If realloc_if_needed is true, + * it is assumed that the buffer does *not* have enough padding, and it is reallocated, enlarged + * and copied before parsing. + * + * @param buf The JSON to parse. Must have at least len + SIMDJSON_PADDING allocated bytes, unless + * realloc_if_needed is true. + * @param len The length of the JSON. + * @param realloc_if_needed Whether to reallocate and enlarge the JSON buffer to add padding. + * @return the document, or an error if the JSON is invalid. + */ + doc_ref_result parse(const uint8_t *buf, size_t len, bool realloc_if_needed = true) noexcept; + + /** + * Parse a JSON document and return a reference to it. + * + * The JSON document still lives in the parser: this is the most efficient way to parse JSON + * documents because it reuses the same buffers, but you *must* use the document before you + * destroy the parser or call parse() again. + * + * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what + * those bytes are initialized to, as long as they are allocated. If realloc_if_needed is true, + * it is assumed that the buffer does *not* have enough padding, and it is reallocated, enlarged + * and copied before parsing. + * + * @param buf The JSON to parse. Must have at least len + SIMDJSON_PADDING allocated bytes, unless + * realloc_if_needed is true. + * @param len The length of the JSON. + * @param realloc_if_needed Whether to reallocate and enlarge the JSON buffer to add padding. + * @return the document, or an error if the JSON is invalid. + */ + doc_ref_result parse(const char *buf, size_t len, bool realloc_if_needed = true) noexcept; + + /** + * Parse a JSON document and return a reference to it. + * + * The JSON document still lives in the parser: this is the most efficient way to parse JSON + * documents because it reuses the same buffers, but you *must* use the document before you + * destroy the parser or call parse() again. + * + * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what + * those bytes are initialized to, as long as they are allocated. If `str.capacity() - str.size() + * < SIMDJSON_PADDING`, the string will be copied to a string with larger capacity before parsing. + * + * @param s The JSON to parse. Must have at least len + SIMDJSON_PADDING allocated bytes, or + * a new string will be created with the extra padding. + * @return the document, or an error if the JSON is invalid. + */ + doc_ref_result parse(const std::string &s) noexcept; + + /** + * Parse a JSON document and return a reference to it. + * + * The JSON document still lives in the parser: this is the most efficient way to parse JSON + * documents because it reuses the same buffers, but you *must* use the document before you + * destroy the parser or call parse() again. + * + * @param s The JSON to parse. + * @return the document, or an error if the JSON is invalid. + */ + doc_ref_result parse(const padded_string &s) noexcept; - // - // Parse a JSON document and return a reference to it. - // - // The JSON document still lives in the parser: this is the most efficient way to parse JSON - // documents because it reuses the same buffers, but you *must* use the document before you - // destroy the parser or call parse() again. - // - // Throws invalid_json if the JSON is invalid. - // - inline doc_ref_result parse(const uint8_t *buf, size_t len, bool realloc_if_needed = true) noexcept; - inline doc_ref_result parse(const char *buf, size_t len, bool realloc_if_needed = true) noexcept; - inline doc_ref_result parse(const std::string &s, bool realloc_if_needed = true) noexcept; - inline doc_ref_result parse(const padded_string &s) noexcept; // We do not want to allow implicit conversion from C string to std::string. - doc_ref_result parse(const char *buf, bool realloc_if_needed = true) noexcept = delete; + doc_ref_result parse(const char *buf) noexcept = delete; - // - // Current capacity: the largest document this parser can support without reallocating. - // - size_t capacity() { return _capacity; } + /** + * Current capacity: the largest document this parser can support without reallocating. + */ + size_t capacity() const noexcept { return _capacity; } - // - // The maximum level of nested object and arrays supported by this parser. - // - size_t max_depth() { return _max_depth; } + /** + * The maximum level of nested object and arrays supported by this parser. + */ + size_t max_depth() const noexcept { return _max_depth; } - // if needed, allocate memory so that the object is able to process JSON - // documents having up to capacity bytes and max_depth "depth" + /** + * Ensure this parser has enough memory to process JSON documents up to `capacity` bytes in length + * and `max_depth` depth. + */ WARN_UNUSED bool allocate_capacity(size_t capacity, size_t max_depth = DEFAULT_MAX_DEPTH) { return set_capacity(capacity) && set_max_depth(max_depth); } @@ -207,22 +978,20 @@ public: // // returns true if the document parsed was valid - bool is_valid() const { return valid; } + bool is_valid() const noexcept; // return an error code corresponding to the last parsing attempt, see // simdjson.h will return UNITIALIZED if no parsing was attempted - int get_error_code() const { return error; } + int get_error_code() const noexcept; // return the string equivalent of "get_error_code" - std::string get_error_message() const { return error_message(error); } + std::string get_error_message() const noexcept; // print the json to std::ostream (should be valid) // return false if the tape is likely wrong (e.g., you did not parse a valid // JSON). - WARN_UNUSED - inline bool print_json(std::ostream &os) const { return is_valid() ? doc.print_json(os) : false; } - WARN_UNUSED - inline bool dump_raw_tape(std::ostream &os) const { return is_valid() ? doc.dump_raw_tape(os) : false; } + bool print_json(std::ostream &os) const noexcept; + bool dump_raw_tape(std::ostream &os) const noexcept; // // Parser callbacks: these are internal! @@ -231,38 +1000,31 @@ public: // // this should be called when parsing (right before writing the tapes) - really_inline void init_stage2(); - really_inline error_code on_error(error_code new_error_code); - really_inline error_code on_success(error_code success_code); - really_inline bool on_start_document(uint32_t depth); - really_inline bool on_start_object(uint32_t depth); - really_inline bool on_start_array(uint32_t depth); + void init_stage2() noexcept; + error_code on_error(error_code new_error_code) noexcept; + error_code on_success(error_code success_code) noexcept; + bool on_start_document(uint32_t depth) noexcept; + bool on_start_object(uint32_t depth) noexcept; + bool on_start_array(uint32_t depth) noexcept; // TODO we're not checking this bool - really_inline bool on_end_document(uint32_t depth); - really_inline bool on_end_object(uint32_t depth); - really_inline bool on_end_array(uint32_t depth); - really_inline bool on_true_atom(); - really_inline bool on_false_atom(); - really_inline bool on_null_atom(); - really_inline uint8_t *on_start_string(); - really_inline bool on_end_string(uint8_t *dst); - really_inline bool on_number_s64(int64_t value); - really_inline bool on_number_u64(uint64_t value); - really_inline bool on_number_double(double value); + bool on_end_document(uint32_t depth) noexcept; + bool on_end_object(uint32_t depth) noexcept; + bool on_end_array(uint32_t depth) noexcept; + bool on_true_atom() noexcept; + bool on_false_atom() noexcept; + bool on_null_atom() noexcept; + uint8_t *on_start_string() noexcept; + bool on_end_string(uint8_t *dst) noexcept; + bool on_number_s64(int64_t value) noexcept; + bool on_number_u64(uint64_t value) noexcept; + bool on_number_double(double value) noexcept; // // Called before a parse is initiated. // // - Returns CAPACITY if the document is too large // - Returns MEMALLOC if we needed to allocate memory and could not // - WARN_UNUSED really_inline error_code init_parse(size_t len); - - const document &get_document() const noexcept(false) { - if (!is_valid()) { - throw invalid_json(error); - } - return doc; - } + WARN_UNUSED error_code init_parse(size_t len) noexcept; private: // @@ -293,13 +1055,8 @@ private: // // - really_inline void write_tape(uint64_t val, uint8_t c) { - doc.tape[current_loc++] = val | ((static_cast(c)) << 56); - } - - really_inline void annotate_previous_loc(uint32_t saved_loc, uint64_t val) { - doc.tape[saved_loc] |= val; - } + void write_tape(uint64_t val, tape_type t) noexcept; + void annotate_previous_loc(uint32_t saved_loc, uint64_t val) noexcept; // // Set the current capacity: the largest document this parser can support without reallocating. @@ -318,6 +1075,11 @@ private: // Returns false if allocation fails. // WARN_UNUSED bool set_max_depth(size_t max_depth); + + // Used internally to get the document + const document &get_document() const noexcept(false); + + template friend class document_iterator; }; // class parser } // namespace simdjson diff --git a/include/simdjson/error.h b/include/simdjson/error.h index 8f0d91ea7..b48f9eabd 100644 --- a/include/simdjson/error.h +++ b/include/simdjson/error.h @@ -24,13 +24,16 @@ enum error_code { UNESCAPED_CHARS, // found unescaped characters in a string. UNCLOSED_STRING, // missing quote at the end UNSUPPORTED_ARCHITECTURE, // unsupported architecture + INCORRECT_TYPE, // JSON element has a different type than user expected + NUMBER_OUT_OF_RANGE, // JSON number does not fit in 64 bits + NO_SUCH_FIELD, // JSON field not found in object UNEXPECTED_ERROR // indicative of a bug in simdjson }; const std::string &error_message(error_code error) noexcept; struct invalid_json : public std::exception { - invalid_json(error_code _error) : error{_error} {} + invalid_json(error_code _error) : error{_error} { } const char *what() const noexcept { return error_message(error).c_str(); } error_code error; }; diff --git a/include/simdjson/inline/document.h b/include/simdjson/inline/document.h index a96d36d66..f47d57762 100644 --- a/include/simdjson/inline/document.h +++ b/include/simdjson/inline/document.h @@ -10,11 +10,243 @@ // implementation. #include "simdjson/implementation.h" - +#include namespace simdjson { -// TODO inline? -document::doc_ref_result document::parser::parse(const uint8_t *buf, size_t len, bool realloc_if_needed) noexcept { +// +// document::element_result inline implementation +// +template +inline document::element_result::element_result(T _value) noexcept : value(_value), error{SUCCESS} {} +template +inline document::element_result::element_result(error_code _error) noexcept : value(), error{_error} {} +template<> +inline document::element_result::operator std::string_view() const noexcept(false) { + if (error) { throw invalid_json(error); } + return value; +} +template<> +inline document::element_result::operator const char *() const noexcept(false) { + if (error) { throw invalid_json(error); } + return value; +} +template<> +inline document::element_result::operator bool() const noexcept(false) { + if (error) { throw invalid_json(error); } + return value; +} +template<> +inline document::element_result::operator uint64_t() const noexcept(false) { + if (error) { throw invalid_json(error); } + return value; +} +template<> +inline document::element_result::operator int64_t() const noexcept(false) { + if (error) { throw invalid_json(error); } + return value; +} +template<> +inline document::element_result::operator double() const noexcept(false) { + if (error) { throw invalid_json(error); } + return value; +} + +// +// document::element_result inline implementation +// +inline document::element_result::element_result(document::array _value) noexcept : value(_value), error{SUCCESS} {} +inline document::element_result::element_result(error_code _error) noexcept : value(), error{_error} {} +inline document::element_result::operator document::array() const noexcept(false) { + if (error) { throw invalid_json(error); } + return value; +} +inline document::array::iterator document::element_result::begin() const noexcept(false) { + if (error) { throw invalid_json(error); } + return value.begin(); +} +inline document::array::iterator document::element_result::end() const noexcept(false) { + if (error) { throw invalid_json(error); } + return value.end(); +} + +// +// document::element_result inline implementation +// +inline document::element_result::element_result(document::object _value) noexcept : value(_value), error{SUCCESS} {} +inline document::element_result::element_result(error_code _error) noexcept : value(), error{_error} {} +inline document::element_result::operator document::object() const noexcept(false) { + if (error) { throw invalid_json(error); } + return value; +} +inline document::element_result document::element_result::operator[](const std::string_view &key) const noexcept { + if (error) { return error; } + return value[key]; +} +inline document::element_result document::element_result::operator[](const char *key) const noexcept { + if (error) { return error; } + return value[key]; +} +inline document::object::iterator document::element_result::begin() const noexcept(false) { + if (error) { throw invalid_json(error); } + return value.begin(); +} +inline document::object::iterator document::element_result::end() const noexcept(false) { + if (error) { throw invalid_json(error); } + return value.end(); +} + +// +// document::element_result inline implementation +// +inline document::element_result::element_result(document::element _value) noexcept : value(_value), error{SUCCESS} {} +inline document::element_result::element_result(error_code _error) noexcept : value(), error{_error} {} +inline document::element_result document::element_result::is_null() const noexcept { + if (error) { return error; } + return value.is_null(); +} +inline document::element_result document::element_result::as_bool() const noexcept { + if (error) { return error; } + return value.as_bool(); +} +inline document::element_result document::element_result::as_c_str() const noexcept { + if (error) { return error; } + return value.as_c_str(); +} +inline document::element_result document::element_result::as_string() const noexcept { + if (error) { return error; } + return value.as_string(); +} +inline document::element_result document::element_result::as_uint64_t() const noexcept { + if (error) { return error; } + return value.as_uint64_t(); +} +inline document::element_result document::element_result::as_int64_t() const noexcept { + if (error) { return error; } + return value.as_int64_t(); +} +inline document::element_result document::element_result::as_double() const noexcept { + if (error) { return error; } + return value.as_double(); +} +inline document::element_result document::element_result::as_array() const noexcept { + if (error) { return error; } + return value.as_array(); +} +inline document::element_result document::element_result::as_object() const noexcept { + if (error) { return error; } + return value.as_object(); +} + +inline document::element_result::operator bool() const noexcept(false) { + return as_bool(); +} +inline document::element_result::operator const char *() const noexcept(false) { + return as_c_str(); +} +inline document::element_result::operator std::string_view() const noexcept(false) { + return as_string(); +} +inline document::element_result::operator uint64_t() const noexcept(false) { + return as_uint64_t(); +} +inline document::element_result::operator int64_t() const noexcept(false) { + return as_int64_t(); +} +inline document::element_result::operator double() const noexcept(false) { + return as_double(); +} +inline document::element_result::operator document::array() const noexcept(false) { + return as_array(); +} +inline document::element_result::operator document::object() const noexcept(false) { + return as_object(); +} +inline document::element_result document::element_result::operator[](const std::string_view &key) const noexcept { + if (error) { return *this; } + return value[key]; +} +inline document::element_result document::element_result::operator[](const char *key) const noexcept { + if (error) { return *this; } + return value[key]; +} + +// +// document inline implementation +// +inline document::element document::root() const noexcept { + return document::element(this, 1); +} +inline document::element_result document::as_array() const noexcept { + return root().as_array(); +} +inline document::element_result document::as_object() const noexcept { + return root().as_object(); +} +inline document::operator document::element() const noexcept { + return root(); +} +inline document::operator document::array() const noexcept(false) { + return root(); +} +inline document::operator document::object() const noexcept(false) { + return root(); +} +inline document::element_result document::operator[](const std::string_view &key) const noexcept { + return root()[key]; +} +inline document::element_result document::operator[](const char *key) const noexcept { + return root()[key]; +} + +// +// document::doc_ref_result inline implementation +// +inline document::doc_ref_result::doc_ref_result(document &_doc, error_code _error) noexcept : doc(_doc), error(_error) { } +inline document::doc_ref_result::operator document&() noexcept(false) { + if (error) { + throw invalid_json(error); + } + return doc; +} +inline const std::string &document::doc_ref_result::get_error_message() const noexcept { + return error_message(error); +} + +// +// document::doc_result inline implementation +// +inline document::doc_result::doc_result(document &&_doc, error_code _error) noexcept : doc(std::move(_doc)), error(_error) { } +inline document::doc_result::doc_result(document &&_doc) noexcept : doc(std::move(_doc)), error(SUCCESS) { } +inline document::doc_result::doc_result(error_code _error) noexcept : doc(), error(_error) { } +inline document::doc_result::operator document() noexcept(false) { + if (error) { + throw invalid_json(error); + } + return std::move(doc); +} +inline const std::string &document::doc_result::get_error_message() const noexcept { + return error_message(error); +} + +// +// document::parser inline implementation +// +inline bool document::parser::is_valid() const noexcept { return valid; } +inline int document::parser::get_error_code() const noexcept { return error; } +inline std::string document::parser::get_error_message() const noexcept { return error_message(error); } +inline bool document::parser::print_json(std::ostream &os) const noexcept { + return is_valid() ? doc.print_json(os) : false; +} +inline bool document::parser::dump_raw_tape(std::ostream &os) const noexcept { + return is_valid() ? doc.dump_raw_tape(os) : false; +} +inline const document &document::parser::get_document() const noexcept(false) { + if (!is_valid()) { + throw invalid_json(error); + } + return doc; +} +inline document::doc_ref_result document::parser::parse(const uint8_t *buf, size_t len, bool realloc_if_needed) noexcept { error_code code = init_parse(len); if (code) { return document::doc_ref_result(doc, code); } @@ -39,14 +271,13 @@ document::doc_ref_result document::parser::parse(const uint8_t *buf, size_t len, really_inline document::doc_ref_result document::parser::parse(const char *buf, size_t len, bool realloc_if_needed) noexcept { return parse((const uint8_t *)buf, len, realloc_if_needed); } -really_inline document::doc_ref_result document::parser::parse(const std::string &s, bool realloc_if_needed) noexcept { - return parse(s.data(), s.length(), realloc_if_needed); +really_inline document::doc_ref_result document::parser::parse(const std::string &s) noexcept { + return parse(s.data(), s.length(), s.capacity() - s.length() < SIMDJSON_PADDING); } really_inline document::doc_ref_result document::parser::parse(const padded_string &s) noexcept { return parse(s.data(), s.length(), false); } -// TODO really_inline? inline document::doc_result document::parse(const uint8_t *buf, size_t len, bool realloc_if_needed) noexcept { document::parser parser; if (!parser.allocate_capacity(len)) { @@ -58,8 +289,8 @@ inline document::doc_result document::parse(const uint8_t *buf, size_t len, bool really_inline document::doc_result document::parse(const char *buf, size_t len, bool realloc_if_needed) noexcept { return parse((const uint8_t *)buf, len, realloc_if_needed); } -really_inline document::doc_result document::parse(const std::string &s, bool realloc_if_needed) noexcept { - return parse(s.data(), s.length(), realloc_if_needed); +really_inline document::doc_result document::parse(const std::string &s) noexcept { + return parse(s.data(), s.length(), s.capacity() - s.length() < SIMDJSON_PADDING); } really_inline document::doc_result document::parse(const padded_string &s) noexcept { return parse(s.data(), s.length(), false); @@ -70,7 +301,7 @@ really_inline document::doc_result document::parse(const padded_string &s) noexc // WARN_UNUSED -inline error_code document::parser::init_parse(size_t len) { +inline error_code document::parser::init_parse(size_t len) noexcept { if (len > capacity()) { return error = CAPACITY; } @@ -83,79 +314,79 @@ inline error_code document::parser::init_parse(size_t len) { return SUCCESS; } -inline void document::parser::init_stage2() { +inline void document::parser::init_stage2() noexcept { current_string_buf_loc = doc.string_buf.get(); current_loc = 0; valid = false; error = UNINITIALIZED; } -really_inline error_code document::parser::on_error(error_code new_error_code) { +really_inline error_code document::parser::on_error(error_code new_error_code) noexcept { error = new_error_code; return new_error_code; } -really_inline error_code document::parser::on_success(error_code success_code) { +really_inline error_code document::parser::on_success(error_code success_code) noexcept { error = success_code; valid = true; return success_code; } -really_inline bool document::parser::on_start_document(uint32_t depth) { +really_inline bool document::parser::on_start_document(uint32_t depth) noexcept { containing_scope_offset[depth] = current_loc; - write_tape(0, 'r'); + write_tape(0, tape_type::ROOT); return true; } -really_inline bool document::parser::on_start_object(uint32_t depth) { +really_inline bool document::parser::on_start_object(uint32_t depth) noexcept { containing_scope_offset[depth] = current_loc; - write_tape(0, '{'); + write_tape(0, tape_type::START_OBJECT); return true; } -really_inline bool document::parser::on_start_array(uint32_t depth) { +really_inline bool document::parser::on_start_array(uint32_t depth) noexcept { containing_scope_offset[depth] = current_loc; - write_tape(0, '['); + write_tape(0, tape_type::START_ARRAY); return true; } // TODO we're not checking this bool -really_inline bool document::parser::on_end_document(uint32_t depth) { +really_inline bool document::parser::on_end_document(uint32_t depth) noexcept { // write our doc.tape location to the header scope // The root scope gets written *at* the previous location. annotate_previous_loc(containing_scope_offset[depth], current_loc); - write_tape(containing_scope_offset[depth], 'r'); + write_tape(containing_scope_offset[depth], tape_type::ROOT); return true; } -really_inline bool document::parser::on_end_object(uint32_t depth) { +really_inline bool document::parser::on_end_object(uint32_t depth) noexcept { // write our doc.tape location to the header scope - write_tape(containing_scope_offset[depth], '}'); + write_tape(containing_scope_offset[depth], tape_type::END_OBJECT); annotate_previous_loc(containing_scope_offset[depth], current_loc); return true; } -really_inline bool document::parser::on_end_array(uint32_t depth) { +really_inline bool document::parser::on_end_array(uint32_t depth) noexcept { // write our doc.tape location to the header scope - write_tape(containing_scope_offset[depth], ']'); + write_tape(containing_scope_offset[depth], tape_type::END_ARRAY); annotate_previous_loc(containing_scope_offset[depth], current_loc); return true; } -really_inline bool document::parser::on_true_atom() { - write_tape(0, 't'); +really_inline bool document::parser::on_true_atom() noexcept { + write_tape(0, tape_type::TRUE_VALUE); return true; } -really_inline bool document::parser::on_false_atom() { - write_tape(0, 'f'); +really_inline bool document::parser::on_false_atom() noexcept { + write_tape(0, tape_type::FALSE_VALUE); return true; } -really_inline bool document::parser::on_null_atom() { - write_tape(0, 'n'); +really_inline bool document::parser::on_null_atom() noexcept { + write_tape(0, tape_type::NULL_VALUE); return true; } -really_inline uint8_t *document::parser::on_start_string() { +really_inline uint8_t *document::parser::on_start_string() noexcept { /* we advance the point, accounting for the fact that we have a NULL * termination */ - write_tape(current_string_buf_loc - doc.string_buf.get(), '"'); + write_tape(current_string_buf_loc - doc.string_buf.get(), tape_type::STRING); return current_string_buf_loc + sizeof(uint32_t); } -really_inline bool document::parser::on_end_string(uint8_t *dst) { +really_inline bool document::parser::on_end_string(uint8_t *dst) noexcept { uint32_t str_length = dst - (current_string_buf_loc + sizeof(uint32_t)); // TODO check for overflow in case someone has a crazy string (>=4GB?) // But only add the overflow check when the document itself exceeds 4GB @@ -168,25 +399,303 @@ really_inline bool document::parser::on_end_string(uint8_t *dst) { return true; } -really_inline bool document::parser::on_number_s64(int64_t value) { - write_tape(0, 'l'); +really_inline bool document::parser::on_number_s64(int64_t value) noexcept { + write_tape(0, tape_type::INT64); std::memcpy(&doc.tape[current_loc], &value, sizeof(value)); ++current_loc; return true; } -really_inline bool document::parser::on_number_u64(uint64_t value) { - write_tape(0, 'u'); +really_inline bool document::parser::on_number_u64(uint64_t value) noexcept { + write_tape(0, tape_type::UINT64); doc.tape[current_loc++] = value; return true; } -really_inline bool document::parser::on_number_double(double value) { - write_tape(0, 'd'); +really_inline bool document::parser::on_number_double(double value) noexcept { + write_tape(0, tape_type::DOUBLE); static_assert(sizeof(value) == sizeof(doc.tape[current_loc]), "mismatch size"); memcpy(&doc.tape[current_loc++], &value, sizeof(double)); // doc.tape[doc.current_loc++] = *((uint64_t *)&d); return true; } +really_inline void document::parser::write_tape(uint64_t val, document::tape_type t) noexcept { + doc.tape[current_loc++] = val | ((static_cast(static_cast(t))) << 56); +} + +really_inline void document::parser::annotate_previous_loc(uint32_t saved_loc, uint64_t val) noexcept { + doc.tape[saved_loc] |= val; +} + +// +// document::tape_ref inline implementation +// +really_inline document::tape_ref::tape_ref() noexcept : doc{nullptr}, json_index{0} {} +really_inline document::tape_ref::tape_ref(const document *_doc, size_t _json_index) noexcept : doc{_doc}, json_index{_json_index} {} + +inline size_t document::tape_ref::after_element() const noexcept { + switch (type()) { + case tape_type::START_ARRAY: + case tape_type::START_OBJECT: + return tape_value(); + case tape_type::UINT64: + case tape_type::INT64: + case tape_type::DOUBLE: + return json_index + 2; + default: + return json_index + 1; + } +} +really_inline document::tape_type document::tape_ref::type() const noexcept { + return static_cast(doc->tape[json_index] >> 56); +} +really_inline uint64_t document::tape_ref::tape_value() const noexcept { + return doc->tape[json_index] & JSON_VALUE_MASK; +} +template +really_inline T document::tape_ref::next_tape_value() const noexcept { + static_assert(sizeof(T) == sizeof(uint64_t)); + return *reinterpret_cast(&doc->tape[json_index + 1]); +} + +// +// document::array inline implementation +// +really_inline document::array::array() noexcept : tape_ref() {} +really_inline document::array::array(const document *_doc, size_t _json_index) noexcept : tape_ref(_doc, _json_index) {} +inline document::array::iterator document::array::begin() const noexcept { + return iterator(doc, json_index + 1); +} +inline document::array::iterator document::array::end() const noexcept { + return iterator(doc, after_element() - 1); +} + + +// +// document::array::iterator inline implementation +// +really_inline document::array::iterator::iterator(const document *_doc, size_t _json_index) noexcept : tape_ref(_doc, _json_index) { } +inline document::element document::array::iterator::operator*() const noexcept { + return element(doc, json_index); +} +inline bool document::array::iterator::operator!=(const document::array::iterator& other) const noexcept { + return json_index != other.json_index; +} +inline void document::array::iterator::operator++() noexcept { + json_index = after_element(); +} + +// +// document::object inline implementation +// +really_inline document::object::object() noexcept : tape_ref() {} +really_inline document::object::object(const document *_doc, size_t _json_index) noexcept : tape_ref(_doc, _json_index) { }; +inline document::object::iterator document::object::begin() const noexcept { + return iterator(doc, json_index + 1); +} +inline document::object::iterator document::object::end() const noexcept { + return iterator(doc, after_element() - 1); +} +inline document::element_result document::object::operator[](const std::string_view &key) const noexcept { + iterator end_field = end(); + for (iterator field = begin(); field != end_field; ++field) { + if (key == field.key()) { + return field.value(); + } + } + return NO_SUCH_FIELD; +} +inline document::element_result document::object::operator[](const char *key) const noexcept { + iterator end_field = end(); + for (iterator field = begin(); field != end_field; ++field) { + if (!strcmp(key, field.key_c_str())) { + return field.value(); + } + } + return NO_SUCH_FIELD; +} + +// +// document::object::iterator inline implementation +// +really_inline document::object::iterator::iterator(const document *_doc, size_t _json_index) noexcept : tape_ref(_doc, _json_index) { } +inline const document::key_value_pair document::object::iterator::operator*() const noexcept { + return key_value_pair(key(), value()); +} +inline bool document::object::iterator::operator!=(const document::object::iterator& other) const noexcept { + return json_index != other.json_index; +} +inline void document::object::iterator::operator++() noexcept { + json_index++; + json_index = after_element(); +} +inline std::string_view document::object::iterator::key() const noexcept { + size_t string_buf_index = tape_value(); + uint32_t len; + memcpy(&len, &doc->string_buf[string_buf_index], sizeof(len)); + return std::string_view( + reinterpret_cast(&doc->string_buf[string_buf_index + sizeof(uint32_t)]), + len + ); +} +inline const char* document::object::iterator::key_c_str() const noexcept { + return reinterpret_cast(&doc->string_buf[tape_value() + sizeof(uint32_t)]); +} +inline document::element document::object::iterator::value() const noexcept { + return element(doc, json_index + 1); +} + +// +// document::key_value_pair inline implementation +// +inline document::key_value_pair::key_value_pair(std::string_view _key, document::element _value) noexcept : + key(_key), value(_value) {} + +// +// document::element inline implementation +// +really_inline document::element::element() noexcept : tape_ref() {} +really_inline document::element::element(const document *_doc, size_t _json_index) noexcept : tape_ref(_doc, _json_index) { } +really_inline bool document::element::is_null() const noexcept { + return type() == tape_type::NULL_VALUE; +} +really_inline bool document::element::is_bool() const noexcept { + return type() == tape_type::TRUE_VALUE || type() == tape_type::FALSE_VALUE; +} +really_inline bool document::element::is_number() const noexcept { + return type() == tape_type::UINT64 || type() == tape_type::INT64 || type() == tape_type::DOUBLE; +} +really_inline bool document::element::is_integer() const noexcept { + return type() == tape_type::UINT64 || type() == tape_type::INT64; +} +really_inline bool document::element::is_string() const noexcept { + return type() == tape_type::STRING; +} +really_inline bool document::element::is_array() const noexcept { + return type() == tape_type::START_ARRAY; +} +really_inline bool document::element::is_object() const noexcept { + return type() == tape_type::START_OBJECT; +} +inline document::element::operator bool() const noexcept(false) { return as_bool(); } +inline document::element::operator const char*() const noexcept(false) { return as_c_str(); } +inline document::element::operator std::string_view() const noexcept(false) { return as_string(); } +inline document::element::operator uint64_t() const noexcept(false) { return as_uint64_t(); } +inline document::element::operator int64_t() const noexcept(false) { return as_int64_t(); } +inline document::element::operator double() const noexcept(false) { return as_double(); } +inline document::element::operator document::array() const noexcept(false) { return as_array(); } +inline document::element::operator document::object() const noexcept(false) { return as_object(); } +inline document::element_result document::element::as_bool() const noexcept { + switch (type()) { + case tape_type::TRUE_VALUE: + return true; + case tape_type::FALSE_VALUE: + return false; + default: + return INCORRECT_TYPE; + } +} +inline document::element_result document::element::as_c_str() const noexcept { + switch (type()) { + case tape_type::STRING: { + size_t string_buf_index = tape_value(); + return reinterpret_cast(&doc->string_buf[string_buf_index + sizeof(uint32_t)]); + } + default: + return INCORRECT_TYPE; + } +} +inline document::element_result document::element::as_string() const noexcept { + switch (type()) { + case tape_type::STRING: { + size_t string_buf_index = tape_value(); + uint32_t len; + memcpy(&len, &doc->string_buf[string_buf_index], sizeof(len)); + return std::string_view( + reinterpret_cast(&doc->string_buf[string_buf_index + sizeof(uint32_t)]), + len + ); + } + default: + return INCORRECT_TYPE; + } +} +inline document::element_result document::element::as_uint64_t() const noexcept { + switch (type()) { + case tape_type::UINT64: + return next_tape_value(); + case tape_type::INT64: { + int64_t result = next_tape_value(); + if (result < 0) { + return NUMBER_OUT_OF_RANGE; + } + return static_cast(result); + } + default: + return INCORRECT_TYPE; + } +} +inline document::element_result document::element::as_int64_t() const noexcept { + switch (type()) { + case tape_type::UINT64: { + uint64_t result = next_tape_value(); + // Wrapping max in parens to handle Windows issue: https://stackoverflow.com/questions/11544073/how-do-i-deal-with-the-max-macro-in-windows-h-colliding-with-max-in-std + if (result > (std::numeric_limits::max)()) { + return NUMBER_OUT_OF_RANGE; + } + return static_cast(result); + } + case tape_type::INT64: + return next_tape_value(); + default: + std::cout << "Incorrect " << json_index << " = " << char(type()) << std::endl; + return INCORRECT_TYPE; + } +} +inline document::element_result document::element::as_double() const noexcept { + switch (type()) { + case tape_type::UINT64: + return next_tape_value(); + case tape_type::INT64: { + return next_tape_value(); + int64_t result = tape_value(); + if (result < 0) { + return NUMBER_OUT_OF_RANGE; + } + return result; + } + case tape_type::DOUBLE: + return next_tape_value(); + default: + return INCORRECT_TYPE; + } +} +inline document::element_result document::element::as_array() const noexcept { + switch (type()) { + case tape_type::START_ARRAY: + return array(doc, json_index); + default: + return INCORRECT_TYPE; + } +} +inline document::element_result document::element::as_object() const noexcept { + switch (type()) { + case tape_type::START_OBJECT: + return object(doc, json_index); + default: + return INCORRECT_TYPE; + } +} +inline document::element_result document::element::operator[](const std::string_view &key) const noexcept { + auto [obj, error] = as_object(); + if (error) { return error; } + return obj[key]; +} +inline document::element_result document::element::operator[](const char *key) const noexcept { + auto [obj, error] = as_object(); + if (error) { return error; } + return obj[key]; +} + } // namespace simdjson #endif // SIMDJSON_INLINE_DOCUMENT_H diff --git a/src/document.cpp b/src/document.cpp index 638076187..19a2411b2 100644 --- a/src/document.cpp +++ b/src/document.cpp @@ -25,8 +25,7 @@ bool document::set_capacity(size_t capacity) { return string_buf && tape; } -WARN_UNUSED -bool document::print_json(std::ostream &os, size_t max_depth) const { +bool document::print_json(std::ostream &os, size_t max_depth) const noexcept { uint32_t string_length; size_t tape_idx = 0; uint64_t tape_val = tape[tape_idx]; @@ -132,8 +131,7 @@ bool document::print_json(std::ostream &os, size_t max_depth) const { return true; } -WARN_UNUSED -bool document::dump_raw_tape(std::ostream &os) const { +bool document::dump_raw_tape(std::ostream &os) const noexcept { uint32_t string_length; size_t tape_idx = 0; uint64_t tape_val = tape[tape_idx]; diff --git a/src/error.cpp b/src/error.cpp index b9899c96c..80d4a1c55 100644 --- a/src/error.cpp +++ b/src/error.cpp @@ -23,6 +23,9 @@ const std::map error_strings = { {UNSUPPORTED_ARCHITECTURE, "simdjson does not have an implementation" " supported by this CPU architecture (perhaps" " it's a non-SIMD CPU?)."}, + {INCORRECT_TYPE, "The JSON element does not have the requested type."}, + {NUMBER_OUT_OF_RANGE, "The JSON number is too large or too small to fit within the requested type."}, + {NO_SUCH_FIELD, "The JSON field referenced does not exist in this object."}, {UNEXPECTED_ERROR, "Unexpected error, consider reporting this problem as" " you may have found a bug in simdjson"}, }; diff --git a/src/generic/stage1_find_marks.h b/src/generic/stage1_find_marks.h index 2c7ab37d5..d5670d225 100644 --- a/src/generic/stage1_find_marks.h +++ b/src/generic/stage1_find_marks.h @@ -209,7 +209,7 @@ really_inline uint64_t follows(const uint64_t match, uint64_t &overflow) { really_inline uint64_t follows(const uint64_t match, const uint64_t filler, uint64_t &overflow) { uint64_t follows_match = follows(match, overflow); uint64_t result; - overflow |= add_overflow(follows_match, filler, &result); + overflow |= uint64_t(add_overflow(follows_match, filler, &result)); return result; } diff --git a/src/haswell/simd.h b/src/haswell/simd.h index 3c23ab9f7..d9791253a 100644 --- a/src/haswell/simd.h +++ b/src/haswell/simd.h @@ -31,7 +31,6 @@ namespace simdjson::haswell::simd { really_inline Child operator&(const Child other) const { return _mm256_and_si256(*this, other); } really_inline Child operator^(const Child other) const { return _mm256_xor_si256(*this, other); } really_inline Child bit_andnot(const Child other) const { return _mm256_andnot_si256(other, *this); } - really_inline Child operator~() const { return *this ^ 0xFFu; } really_inline Child& operator|=(const Child other) { auto this_cast = (Child*)this; *this_cast = *this_cast | other; return *this_cast; } really_inline Child& operator&=(const Child other) { auto this_cast = (Child*)this; *this_cast = *this_cast & other; return *this_cast; } really_inline Child& operator^=(const Child other) { auto this_cast = (Child*)this; *this_cast = *this_cast ^ other; return *this_cast; } @@ -71,6 +70,7 @@ namespace simdjson::haswell::simd { really_inline int to_bitmask() const { return _mm256_movemask_epi8(*this); } really_inline bool any() const { return !_mm256_testz_si256(*this, *this); } + really_inline simd8 operator~() const { return *this ^ true; } }; template @@ -105,6 +105,9 @@ namespace simdjson::haswell::simd { really_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *(simd8*)this; } really_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *(simd8*)this; } + // Override to distinguish from bool version + really_inline simd8 operator~() const { return *this ^ 0xFFu; } + // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) template really_inline simd8 lookup_16(simd8 lookup_table) const { diff --git a/src/westmere/simd.h b/src/westmere/simd.h index 6a52f156b..c2b752fc8 100644 --- a/src/westmere/simd.h +++ b/src/westmere/simd.h @@ -31,7 +31,6 @@ namespace simdjson::westmere::simd { really_inline Child operator&(const Child other) const { return _mm_and_si128(*this, other); } really_inline Child operator^(const Child other) const { return _mm_xor_si128(*this, other); } really_inline Child bit_andnot(const Child other) const { return _mm_andnot_si128(other, *this); } - really_inline Child operator~() const { return *this ^ 0xFFu; } really_inline Child& operator|=(const Child other) { auto this_cast = (Child*)this; *this_cast = *this_cast | other; return *this_cast; } really_inline Child& operator&=(const Child other) { auto this_cast = (Child*)this; *this_cast = *this_cast & other; return *this_cast; } really_inline Child& operator^=(const Child other) { auto this_cast = (Child*)this; *this_cast = *this_cast ^ other; return *this_cast; } @@ -71,6 +70,7 @@ namespace simdjson::westmere::simd { really_inline int to_bitmask() const { return _mm_movemask_epi8(*this); } really_inline bool any() const { return !_mm_testz_si128(*this, *this); } + really_inline simd8 operator~() const { return *this ^ true; } }; template @@ -97,6 +97,9 @@ namespace simdjson::westmere::simd { // Store to array really_inline void store(T dst[16]) const { return _mm_storeu_si128(reinterpret_cast<__m128i *>(dst), *this); } + // Override to distinguish from bool version + really_inline simd8 operator~() const { return *this ^ 0xFFu; } + // Addition/subtraction are the same for signed and unsigned really_inline simd8 operator+(const simd8 other) const { return _mm_add_epi8(*this, other); } really_inline simd8 operator-(const simd8 other) const { return _mm_sub_epi8(*this, other); } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index b00f56d77..b43d20b70 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -18,9 +18,10 @@ add_cpp_test(jsonstream_test) add_cpp_test(pointercheck) add_cpp_test(integer_tests) +target_compile_definitions(basictests PRIVATE JSON_TEST_PATH="${PROJECT_SOURCE_DIR}/jsonexamples/twitter.json") + ## This causes problems # add_executable(singleheader ./singleheadertest.cpp ${PROJECT_SOURCE_DIR}/singleheader/simdjson.cpp) -# target_compile_definitions(singleheader PRIVATE JSON_TEST_PATH="${PROJECT_SOURCE_DIR}/jsonexamples/twitter.json") # target_link_libraries(singleheader ${SIMDJSON_LIB_NAME}) # add_test(singleheader singleheader) diff --git a/tests/basictests.cpp b/tests/basictests.cpp index 94215ff91..baa2d94c6 100644 --- a/tests/basictests.cpp +++ b/tests/basictests.cpp @@ -7,11 +7,17 @@ #include #include #include +#include +#include #include "simdjson/jsonparser.h" #include "simdjson/jsonstream.h" #include "simdjson/document.h" +#ifndef JSON_TEST_PATH +#define JSON_TEST_PATH "jsonexamples/twitter.json" +#endif + // ulp distance // Marc B. Reynolds, 2016-2019 // Public Domain under http://unlicense.org, see link for details. @@ -569,6 +575,205 @@ bool skyprophet_test() { return true; } +namespace dom_api { + using namespace std; + using namespace simdjson; + bool object_iterator() { + string json(R"({ "a": 1, "b": 2, "c": 3 })"); + const char* expected_key[] = { "a", "b", "c" }; + uint64_t expected_value[] = { 1, 2, 3 }; + int i = 0; + + document doc = document::parse(json); + for (auto [key, value] : document::object(doc)) { + if (key != expected_key[i] || uint64_t(value) != expected_value[i]) { cerr << "Expected " << expected_key[i] << " = " << expected_value[i] << ", got " << key << "=" << uint64_t(value) << endl; return false; } + i++; + } + if (i*sizeof(uint64_t) != sizeof(expected_value)) { cout << "Expected " << sizeof(expected_value) << " values, got " << i << endl; return false; } + return true; + } + + bool array_iterator() { + string json(R"([ 1, 10, 100 ])"); + uint64_t expected_value[] = { 1, 10, 100 }; + int i=0; + + document doc = document::parse(json); + for (uint64_t value : doc.as_array()) { + if (value != expected_value[i]) { cerr << "Expected " << expected_value[i] << ", got " << value << endl; return false; } + i++; + } + if (i*sizeof(uint64_t) != sizeof(expected_value)) { cout << "Expected " << sizeof(expected_value) << " values, got " << i << endl; return false; } + return true; + } + + bool object_iterator_empty() { + string json(R"({})"); + int i = 0; + + document doc = document::parse(json); + for (auto [key, value] : doc.as_object()) { + cout << "Unexpected " << key << " = " << uint64_t(value) << endl; + i++; + } + if (i > 0) { cout << "Expected 0 values, got " << i << endl; return false; } + return true; + } + + bool array_iterator_empty() { + string json(R"([])"); + int i=0; + + document doc = document::parse(json); + for (uint64_t value : doc.as_array()) { + cout << "Unexpected value " << value << endl; + i++; + } + if (i > 0) { cout << "Expected 0 values, got " << i << endl; return false; } + return true; + } + + bool string_value() { + string json(R"([ "hi", "has backslash\\" ])"); + document doc = document::parse(json); + auto val = document::array(doc).begin(); + if (strcmp((const char*)*val, "hi")) { cerr << "Expected const char*(\"hi\") to be \"hi\", was " << (const char*)*val << endl; return false; } + if (string_view(*val) != "hi") { cerr << "Expected string_view(\"hi\") to be \"hi\", was " << string_view(*val) << endl; return false; } + ++val; + if (strcmp((const char*)*val, "has backslash\\")) { cerr << "Expected const char*(\"has backslash\\\\\") to be \"has backslash\\\", was " << (const char*)*val << endl; return false; } + if (string_view(*val) != "has backslash\\") { cerr << "Expected string_view(\"has backslash\\\\\") to be \"has backslash\\\", was " << string_view(*val) << endl; return false; } + return true; + } + + bool numeric_values() { + string json(R"([ 0, 1, -1, 1.1 ])"); + document doc = document::parse(json); + auto val = document::array(doc).begin(); + if (uint64_t(*val) != 0) { cerr << "Expected uint64_t(0) to be 0, was " << uint64_t(*val) << endl; return false; } + if (int64_t(*val) != 0) { cerr << "Expected int64_t(0) to be 0, was " << int64_t(*val) << endl; return false; } + if (double(*val) != 0) { cerr << "Expected double(0) to be 0, was " << double(*val) << endl; return false; } + ++val; + if (uint64_t(*val) != 1) { cerr << "Expected uint64_t(1) to be 1, was " << uint64_t(*val) << endl; return false; } + if (int64_t(*val) != 1) { cerr << "Expected int64_t(1) to be 1, was " << int64_t(*val) << endl; return false; } + if (double(*val) != 1) { cerr << "Expected double(1) to be 1, was " << double(*val) << endl; return false; } + ++val; + if (int64_t(*val) != -1) { cerr << "Expected int64_t(-1) to be -1, was " << int64_t(*val) << endl; return false; } + if (double(*val) != -1) { cerr << "Expected double(-1) to be -1, was " << double(*val) << endl; return false; } + ++val; + if (double(*val) != 1.1) { cerr << "Expected double(1.1) to be 1.1, was " << double(*val) << endl; return false; } + return true; + } + + bool boolean_values() { + string json(R"([ true, false ])"); + document doc = document::parse(json); + auto val = document::array(doc).begin(); + if (bool(*val) != true) { cerr << "Expected bool(true) to be true, was " << bool(*val) << endl; return false; } + ++val; + if (bool(*val) != false) { cerr << "Expected bool(false) to be false, was " << bool(*val) << endl; return false; } + return true; + } + + bool null_value() { + string json(R"([ null ])"); + document doc = document::parse(json); + auto val = document::array(doc).begin(); + if (!(*val).is_null()) { cerr << "Expected null to be null!" << endl; return false; } + return true; + } + + bool document_object_index() { + string json(R"({ "a": 1, "b": 2, "c": 3})"); + document doc = document::parse(json); + if (uint64_t(doc["a"]) != 1) { cerr << "Expected uint64_t(doc[\"a\"]) to be 1, was " << uint64_t(doc["a"]) << endl; return false; } + if (uint64_t(doc["b"]) != 2) { cerr << "Expected uint64_t(doc[\"b\"]) to be 2, was " << uint64_t(doc["b"]) << endl; return false; } + if (uint64_t(doc["c"]) != 3) { cerr << "Expected uint64_t(doc[\"c\"]) to be 3, was " << uint64_t(doc["c"]) << endl; return false; } + // Check all three again in backwards order, to ensure we can go backwards + if (uint64_t(doc["c"]) != 3) { cerr << "Expected uint64_t(doc[\"c\"]) to be 3, was " << uint64_t(doc["c"]) << endl; return false; } + if (uint64_t(doc["b"]) != 2) { cerr << "Expected uint64_t(doc[\"b\"]) to be 2, was " << uint64_t(doc["b"]) << endl; return false; } + if (uint64_t(doc["a"]) != 1) { cerr << "Expected uint64_t(doc[\"a\"]) to be 1, was " << uint64_t(doc["a"]) << endl; return false; } + + auto [val, error] = doc["d"]; + if (error != simdjson::NO_SUCH_FIELD) { cerr << "Expected NO_SUCH_FIELD error for uint64_t(doc[\"d\"]), got " << error_message(error) << endl; return false; } + return true; + } + + bool object_index() { + string json(R"({ "obj": { "a": 1, "b": 2, "c": 3 } })"); + document doc = document::parse(json); + if (uint64_t(doc["obj"]["a"]) != 1) { cerr << "Expected uint64_t(doc[\"obj\"][\"a\"]) to be 1, was " << uint64_t(doc["obj"]["a"]) << endl; return false; } + document::object obj = doc["obj"]; + if (uint64_t(obj["a"]) != 1) { cerr << "Expected uint64_t(obj[\"a\"]) to be 1, was " << uint64_t(obj["a"]) << endl; return false; } + if (uint64_t(obj["b"]) != 2) { cerr << "Expected uint64_t(obj[\"b\"]) to be 2, was " << uint64_t(obj["b"]) << endl; return false; } + if (uint64_t(obj["c"]) != 3) { cerr << "Expected uint64_t(obj[\"c\"]) to be 3, was " << uint64_t(obj["c"]) << endl; return false; } + // Check all three again in backwards order, to ensure we can go backwards + if (uint64_t(obj["c"]) != 3) { cerr << "Expected uint64_t(obj[\"c\"]) to be 3, was " << uint64_t(obj["c"]) << endl; return false; } + if (uint64_t(obj["b"]) != 2) { cerr << "Expected uint64_t(obj[\"b\"]) to be 2, was " << uint64_t(obj["b"]) << endl; return false; } + if (uint64_t(obj["a"]) != 1) { cerr << "Expected uint64_t(obj[\"a\"]) to be 1, was " << uint64_t(obj["a"]) << endl; return false; } + + auto [val, error] = obj["d"]; + if (error != simdjson::NO_SUCH_FIELD) { cerr << "Expected NO_SUCH_FIELD error for uint64_t(obj[\"d\"]), got " << error_message(error) << endl; return false; } + return true; + } + + bool twitter_count() { + // Prints the number of results in twitter.json + document doc = document::parse(get_corpus(JSON_TEST_PATH)); + uint64_t result_count = doc["search_metadata"]["count"]; + if (result_count != 100) { cerr << "Expected twitter.json[metadata_count][count] = 100, got " << result_count << endl; return false; } + return true; + } + + bool twitter_default_profile() { + // Print users with a default profile. + set default_users; + document doc = document::parse(get_corpus(JSON_TEST_PATH)); + for (document::object tweet : doc["statuses"].as_array()) { + document::object user = tweet["user"]; + if (user["default_profile"]) { + default_users.insert(user["screen_name"]); + } + } + if (default_users.size() != 86) { cerr << "Expected twitter.json[statuses][user] to contain 86 default_profile users, got " << default_users.size() << endl; return false; } + return true; + } + + bool twitter_image_sizes() { + // Print image names and sizes + set> image_sizes; + document doc = document::parse(get_corpus(JSON_TEST_PATH)); + for (document::object tweet : doc["statuses"].as_array()) { + auto [media, not_found] = tweet["entities"]["media"]; + if (!not_found) { + for (document::object image : media.as_array()) { + for (auto [key, size] : image["sizes"].as_object()) { + image_sizes.insert({ size["w"], size["h"] }); + } + } + } + } + if (image_sizes.size() != 15) { cerr << "Expected twitter.json[statuses][entities][media][sizes] to contain 15 different sizes, got " << image_sizes.size() << endl; return false; } + return true; + } + + bool run_tests() { + if (!object_iterator()) { return false; } + if (!array_iterator()) { return false; } + if (!object_iterator_empty()) { return false; } + if (!array_iterator_empty()) { return false; } + if (!string_value()) { return false; } + if (!numeric_values()) { return false; } + if (!boolean_values()) { return false; } + if (!null_value()) { return false; } + if (!document_object_index()) { return false; } + if (!object_index()) { return false; } + if (!twitter_count()) { return false; } + if (!twitter_default_profile()) { return false; } + if (!twitter_image_sizes()) { return false; } + return true; + } +} + int main() { // this is put here deliberately to check that the documentation is correct (README), // should this fail to compile, you should update the documentation: @@ -596,6 +801,8 @@ int main() { return EXIT_FAILURE; if (!skyprophet_test()) return EXIT_FAILURE; + if (!dom_api::run_tests()) + return EXIT_FAILURE; std::cout << "Basic tests are ok." << std::endl; return EXIT_SUCCESS; } diff --git a/tests/readme_examples.cpp b/tests/readme_examples.cpp index 2e5dcd8bf..422022b14 100644 --- a/tests/readme_examples.cpp +++ b/tests/readme_examples.cpp @@ -5,6 +5,8 @@ using namespace std; using namespace simdjson; void document_parse_error_code() { + cout << __func__ << endl; + string json("[ 1, 2, 3 ]"); auto [doc, error] = document::parse(json); if (error) { cerr << "Error: " << error_message(error) << endl; exit(1); } @@ -13,6 +15,8 @@ void document_parse_error_code() { } void document_parse_exception() { + cout << __func__ << endl; + string json("[ 1, 2, 3 ]"); document doc = document::parse(json); doc.print_json(cout); @@ -20,6 +24,8 @@ void document_parse_exception() { } void document_parse_padded_string() { + cout << __func__ << endl; + padded_string json(string("[ 1, 2, 3 ]")); document doc = document::parse(json); doc.print_json(cout); @@ -27,6 +33,8 @@ void document_parse_padded_string() { } void document_parse_get_corpus() { + cout << __func__ << endl; + padded_string json(get_corpus("jsonexamples/small/demo.json")); document doc = document::parse(json); doc.print_json(cout); @@ -34,6 +42,8 @@ void document_parse_get_corpus() { } void parser_parse() { + cout << __func__ << endl; + // Allocate a parser big enough for all files document::parser parser; if (!parser.allocate_capacity(1024*1024)) { exit(1); }