diff --git a/doc/basics.md b/doc/basics.md index f18ab175b..1d1ffd6e0 100644 --- a/doc/basics.md +++ b/doc/basics.md @@ -209,6 +209,25 @@ std::string data = "my data"; simdjson::padded_string my_padded_data(data); // copies to a padded buffer ``` +You can then parse the JSON data from the `simdjson::padded_string` instance: + + +```c++ +ondemand::document doc = parser.iterate(my_padded_data); +``` + +Whenever you pass an `std::string` reference to `parser::iterate`, +the parser will access the bytes beyond the end of +the string but before the end of the allocated memory (`std::string::capacity()`). +If you are using a sanitizer that checks for reading uninitialized bytes or `std::string`'s +container-overflow checks, you may encounter sanitizer warnings. +You can safely ignore these warnings. Or you can call `simdjson::pad(std::string&)` to pad the +string with `SIMDJSON_PADDING` spaces: this function returns a `simdjson::padding_string_view` which can be be passed to the parser's iterator function: + +```c++ +std::string json = "[1]"; +ondemand::document doc = parser.iterate(simdjson::pad(json)); +``` We recommend against creating many `std::string` or many `std::padding_string` instances in your application to store your JSON data. Consider reusing the same buffers and limiting memory allocations. diff --git a/doc/dom.md b/doc/dom.md index 6b8cfd259..0c9272a84 100644 --- a/doc/dom.md +++ b/doc/dom.md @@ -60,6 +60,26 @@ std::string data = "my data"; simdjson::padded_string my_padded_data(data); // copies to a padded buffer ``` +You can then parse the JSON document from the `simdjson::padded_string` instance: + +```cpp +simdjson::dom::parser parser; +simdjson::dom::element doc = parser.parse(my_padded_data); +``` + +Whenever you pass an `std::string` reference to `parser::parse`, +the parser will access the bytes beyond the end of +the string but before the end of the allocated memory (`std::string::capacity()`). +If you are using a sanitizer that checks for reading uninitialized bytes or `std::string`'s +container-overflow checks, you may encounter sanitizer warnings. +You can safely ignore these warnings. Or you can call `simdjson::pad(std::string&)` to pad the +string with `SIMDJSON_PADDING` spaces: this function returns a `simdjson::padding_string_view` which can be be passed to the parser's iterator function: + +```c++ +std::string json = "[1]"; +dom::element doc = parser.parse(simdjson::pad(json)); +``` + The parsed document resulting from the `parser.load` and `parser.parse` calls depends on the `parser` instance. Thus the `parser` instance must remain in scope. Furthermore, you must have at most one parsed document in play per `parser` instance. You cannot copy a `parser` instance, you may only move it. diff --git a/include/simdjson/dom/element.h b/include/simdjson/dom/element.h index d222ca72e..d39c83576 100644 --- a/include/simdjson/dom/element.h +++ b/include/simdjson/dom/element.h @@ -372,6 +372,8 @@ public: * - INCORRECT_TYPE if this is not an object */ inline simdjson_result operator[](const char *key) const noexcept; + simdjson_result operator[](int) const noexcept = delete; + /** * Get the value associated with the given JSON pointer. We use the RFC 6901 @@ -540,6 +542,7 @@ public: simdjson_inline simdjson_result operator[](std::string_view key) const noexcept; simdjson_inline simdjson_result operator[](const char *key) const noexcept; + simdjson_result operator[](int) const noexcept = delete; simdjson_inline simdjson_result at_pointer(const std::string_view json_pointer) const noexcept; simdjson_inline simdjson_result at_path(const std::string_view json_path) const noexcept; [[deprecated("For standard compliance, use at_pointer instead, and prefix your pointers with a slash '/', see RFC6901 ")]] diff --git a/include/simdjson/dom/object.h b/include/simdjson/dom/object.h index 1c72c11fc..83f1392d9 100644 --- a/include/simdjson/dom/object.h +++ b/include/simdjson/dom/object.h @@ -145,6 +145,7 @@ public: * - INCORRECT_TYPE if this is not an object */ inline simdjson_result operator[](const char *key) const noexcept; + simdjson_result operator[](int) const noexcept = delete; /** * Get the value associated with the given JSON pointer. We use the RFC 6901 @@ -258,6 +259,7 @@ public: inline simdjson_result operator[](std::string_view key) const noexcept; inline simdjson_result operator[](const char *key) const noexcept; + simdjson_result operator[](int) const noexcept = delete; inline simdjson_result at_pointer(std::string_view json_pointer) const noexcept; inline simdjson_result at_path(std::string_view json_path) const noexcept; inline simdjson_result at_key(std::string_view key) const noexcept; diff --git a/include/simdjson/dom/parser.h b/include/simdjson/dom/parser.h index bbfe3cbf2..7c28a712f 100644 --- a/include/simdjson/dom/parser.h +++ b/include/simdjson/dom/parser.h @@ -202,6 +202,22 @@ public: * simdjson::dom::parser parser; * simdjson::dom::element element = parser.parse(padded_json_copy.get(), json_len, false); * + * ### std::string references + * + * If you pass a mutable std::string reference (std::string&), the parser will seek to extend + * its capacity to SIMDJSON_PADDING bytes beyond the end of the string. + * + * Whenever you pass an std::string reference, the parser will access the bytes beyond the end of + * the string but before the end of the allocated memory (std::string::capacity()). + * If you are using a sanitizer that checks for reading uninitialized bytes or std::string's + * container-overflow checks, you may encounter sanitizer warnings. + * You can safely ignore these warnings. Or you can call simdjson::pad(std::string&) to pad the + * string with SIMDJSON_PADDING spaces: this function returns a simdjson::padding_string_view + * which can be be passed to the parser's parse function: + * + * std::string json = R"({ "foo": 1 } { "foo": 2 } { "foo": 3 } )"; + * element doc = parser.parse(simdjson::pad(json)); + * * ### Parser Capacity * * If the parser's current capacity is less than len, it will allocate enough capacity diff --git a/include/simdjson/generic/ondemand/document.h b/include/simdjson/generic/ondemand/document.h index 7cac3dca3..b802fbb01 100644 --- a/include/simdjson/generic/ondemand/document.h +++ b/include/simdjson/generic/ondemand/document.h @@ -466,6 +466,7 @@ public: simdjson_inline simdjson_result operator[](std::string_view key) & noexcept; /** @overload simdjson_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; */ simdjson_inline simdjson_result operator[](const char *key) & noexcept; + simdjson_result operator[](int) & noexcept = delete; /** * Get the type of this JSON value. It does not validate or consume the value. @@ -852,6 +853,7 @@ public: simdjson_inline simdjson_result find_field(const char *key) & noexcept; simdjson_inline simdjson_result operator[](std::string_view key) & noexcept; simdjson_inline simdjson_result operator[](const char *key) & noexcept; + simdjson_result operator[](int) & noexcept = delete; simdjson_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; simdjson_inline simdjson_result find_field_unordered(const char *key) & noexcept; @@ -930,6 +932,7 @@ public: simdjson_inline simdjson_result find_field(const char *key) & noexcept; simdjson_inline simdjson_result operator[](std::string_view key) & noexcept; simdjson_inline simdjson_result operator[](const char *key) & noexcept; + simdjson_result operator[](int) & noexcept = delete; simdjson_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; simdjson_inline simdjson_result find_field_unordered(const char *key) & noexcept; simdjson_inline simdjson_result type() noexcept; @@ -1007,6 +1010,7 @@ public: simdjson_inline simdjson_result find_field(const char *key) & noexcept; simdjson_inline simdjson_result operator[](std::string_view key) & noexcept; simdjson_inline simdjson_result operator[](const char *key) & noexcept; + simdjson_result operator[](int) & noexcept = delete; simdjson_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; simdjson_inline simdjson_result find_field_unordered(const char *key) & noexcept; simdjson_inline simdjson_result type() noexcept; diff --git a/include/simdjson/generic/ondemand/parser.h b/include/simdjson/generic/ondemand/parser.h index 7b4caa2db..5e22f5554 100644 --- a/include/simdjson/generic/ondemand/parser.h +++ b/include/simdjson/generic/ondemand/parser.h @@ -84,6 +84,22 @@ public: * using a sanitizer that verifies that no uninitialized byte is read, then you should initialize the * SIMDJSON_PADDING bytes to avoid runtime warnings. * + * ### std::string references + * + * If you pass a mutable std::string reference (std::string&), the parser will seek to extend + * its capacity to SIMDJSON_PADDING bytes beyond the end of the string. + * + * Whenever you pass an std::string reference, the parser will access the bytes beyond the end of + * the string but before the end of the allocated memory (std::string::capacity()). + * If you are using a sanitizer that checks for reading uninitialized bytes or std::string's + * container-overflow checks, you may encounter sanitizer warnings. + * You can safely ignore these warnings. Or you can call simdjson::pad(std::string&) to pad the + * string with SIMDJSON_PADDING spaces: this function returns a simdjson::padding_string_view + * which can be be passed to the parser's iterate function: + * + * std::string json = R"({ "foo": 1 } { "foo": 2 } { "foo": 3 } )"; + * document doc = parser.iterate(simdjson::pad(json)); + * * @param json The JSON to parse. * @param len The length of the JSON. * @param capacity The number of bytes allocated in the JSON (must be at least len+SIMDJSON_PADDING). diff --git a/include/simdjson/generic/ondemand/value.h b/include/simdjson/generic/ondemand/value.h index 5fa7e2d16..7fa9d1ad2 100644 --- a/include/simdjson/generic/ondemand/value.h +++ b/include/simdjson/generic/ondemand/value.h @@ -163,6 +163,17 @@ public: * Important: a value should be consumed once. Calling get_string() twice on the same value * is an error. * + * In some instances, you may want to allow replacement of invalid Unicode sequences. + * You may do so by passing the allow_replacement parameter as true. In the following + * example, the string "431924697b\udff0L\u0001Y" is not valid Unicode. By passing true + * to get_string, we allow the replacement of the invalid Unicode sequences with the Unicode + * replacement character (U+FFFD). + * + * simdjson::ondemand::parser parser; + * auto json = R"({"deviceId":"431924697b\udff0L\u0001Y"})"_padded; + * simdjson::ondemand::document doc = parser.iterate(json); + * auto view = doc["deviceId"].get_string(true); + * * @returns An UTF-8 string. The string is stored in the parser and will be invalidated the next * time it parses a document or when it is destroyed. * @returns INCORRECT_TYPE if the JSON value is not a string. @@ -414,6 +425,7 @@ public: simdjson_inline simdjson_result operator[](std::string_view key) noexcept; /** @overload simdjson_inline simdjson_result find_field_unordered(std::string_view key) noexcept; */ simdjson_inline simdjson_result operator[](const char *key) noexcept; + simdjson_result operator[](int) noexcept = delete; /** * Get the type of this JSON value. It does not validate or consume the value. @@ -781,6 +793,7 @@ public: simdjson_inline simdjson_result operator[](std::string_view key) noexcept; /** @overload simdjson_inline simdjson_result find_field_unordered(std::string_view key) noexcept; */ simdjson_inline simdjson_result operator[](const char *key) noexcept; + simdjson_result operator[](int) noexcept = delete; /** * Get the type of this JSON value. diff --git a/include/simdjson/padded_string_view-inl.h b/include/simdjson/padded_string_view-inl.h index 1c1811de4..3c727dbcd 100644 --- a/include/simdjson/padded_string_view-inl.h +++ b/include/simdjson/padded_string_view-inl.h @@ -53,6 +53,11 @@ inline bool padded_string_view::remove_utf8_bom() noexcept { inline std::ostream& operator<<(std::ostream& out, simdjson_result &s) noexcept(false) { return out << s.value(); } #endif +inline padded_string_view pad(std::string& s) noexcept { + const auto len = s.size(); + s.append(SIMDJSON_PADDING, ' '); + return padded_string_view(s.data(), len, s.size()); +} } // namespace simdjson diff --git a/include/simdjson/padded_string_view.h b/include/simdjson/padded_string_view.h index f8dc59a79..190c98fec 100644 --- a/include/simdjson/padded_string_view.h +++ b/include/simdjson/padded_string_view.h @@ -83,6 +83,15 @@ public: inline std::ostream& operator<<(std::ostream& out, simdjson_result &s) noexcept(false); #endif +/** + * Create a padded_string_view from a string. The string will be padded with SIMDJSON_PADDING + * space characters. The resulting padded_string_view will have a length equal to the original + * string. + * + * @param s The string. + * @return The padded string. + */ +inline padded_string_view pad(std::string& s) noexcept; } // namespace simdjson #endif // SIMDJSON_PADDED_STRING_VIEW_H diff --git a/tests/dom/readme_examples.cpp b/tests/dom/readme_examples.cpp index 3df3cf1f5..c674c7985 100644 --- a/tests/dom/readme_examples.cpp +++ b/tests/dom/readme_examples.cpp @@ -461,6 +461,13 @@ void parse_documentation_lowlevel() { (void)element; } +void simplepad() { + std::string json = "[1]"; + dom::parser parser; + dom::element doc; + auto error = parser.parse(simdjson::pad(json)).get(doc); + if(error) { exit(-1); } +} void jsondollar() { dom::parser parser; @@ -497,6 +504,7 @@ void jsonpath() { } int main() { + simplepad(); jsonpath(); jsondollar(); basics_dom_1(); diff --git a/tests/ondemand/ondemand_readme_examples.cpp b/tests/ondemand/ondemand_readme_examples.cpp index 88a8906b0..35c8ad79d 100644 --- a/tests/ondemand/ondemand_readme_examples.cpp +++ b/tests/ondemand/ondemand_readme_examples.cpp @@ -8,7 +8,15 @@ #endif using namespace std; using namespace simdjson; -using error_code=simdjson::error_code; +using error_code = simdjson::error_code; + +bool simplepad() { + std::string json = "[1]"; + ondemand::parser parser; + ondemand::document doc; + auto error = parser.iterate(simdjson::pad(json)).get(doc); + return error == SUCCESS; +} bool string1() { const char * data = "my data"; // 7 bytes @@ -1918,6 +1926,7 @@ bool run() { && using_the_parsed_json_4() && using_the_parsed_json_5() #endif + && simplepad() && using_the_parsed_json_6() && json_pointer_simple() && json_pointer_unicode()