diff --git a/CMakeLists.txt b/CMakeLists.txt index de84cb402..be1c2c90a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -190,11 +190,14 @@ endif() if(is_top_project AND NOT SIMDJSON_DEVELOPER_MODE) message(STATUS "Building only the library. Advanced users and contributors may want to turn SIMDJSON_DEVELOPER_MODE to ON, e.g., via -D SIMDJSON_DEVELOPER_MODE=ON.") - return() elseif(SIMDJSON_DEVELOPER_MODE AND NOT is_top_project) message(AUTHOR_WARNING "Developer mode in simdjson is intended for the developers of simdjson") endif() +if(NOT SIMDJSON_DEVELOPER_MODE) + return() +endif() + simdjson_apply_props(simdjson-internal-flags) set( diff --git a/doc/basics.md b/doc/basics.md index a32ae1940..f751ea805 100644 --- a/doc/basics.md +++ b/doc/basics.md @@ -208,6 +208,8 @@ For best performance, a `parser` instance should be reused over several files: o needlessly reallocate memory, an expensive process. It is also possible to avoid entirely memory allocations during parsing when using simdjson. [See our performance notes for details](performance.md). +If you need to have several documents active at once, you should have several parser instances. + C++11 Support and string_view ------------- @@ -300,7 +302,13 @@ support for users who avoid exceptions. See [the simdjson error handling documen `get_uint64()`, `get_int64()`, `get_bool()`, `get_object()` and `get_array()`. After a cast or an explicit method, the number, string or boolean will be parsed, or the initial `[` or `{` will be verified. An exception is thrown if the cast is not possible. The `get_string()` returns a valid UTF-8 string, after - unescaping characters as needed: unmatched surrogate pairs are treated as an error. When calling `get_uint64()` and `get_int64()`, if the number does not fit in a corresponding 64-bit integer type, it is also considered an error. + unescaping characters as needed: unmatched surrogate pairs are treated as an error unless you + pass `true` (`get_string(true)`) as a parameter to get replacement characters where errors + occur. If you somehow need to access non-UTF-8 strings in a lossless manner + (e.g., if you strings contain unpaired surrogates), you may use the `get_wobbly_string()` function to get a string in the [WTF-8 format](https://simonsapin.github.io/wtf-8). + Or you may pass `true` as a parameter to the + When calling `get_uint64()` and `get_int64()`, if the number does not fit in a corresponding + 64-bit integer type, it is also considered an error. > IMPORTANT NOTE: values can only be parsed once. Since documents are *iterators*, once you have > parsed a value (such as by casting to double), you cannot get at it again. It is an error to call diff --git a/include/simdjson/generic/dom_parser_implementation.h b/include/simdjson/generic/dom_parser_implementation.h index 4c98dbb2e..72823bac3 100644 --- a/include/simdjson/generic/dom_parser_implementation.h +++ b/include/simdjson/generic/dom_parser_implementation.h @@ -35,7 +35,8 @@ public: simdjson_warn_unused error_code stage1(const uint8_t *buf, size_t len, stage1_mode partial) noexcept final; simdjson_warn_unused error_code stage2(dom::document &doc) noexcept final; simdjson_warn_unused error_code stage2_next(dom::document &doc) noexcept final; - simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst) const noexcept final; + simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept final; + simdjson_warn_unused uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept final; inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final; inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final; private: diff --git a/include/simdjson/generic/ondemand/document-inl.h b/include/simdjson/generic/ondemand/document-inl.h index 7459ca363..dd8f6a3a5 100644 --- a/include/simdjson/generic/ondemand/document-inl.h +++ b/include/simdjson/generic/ondemand/document-inl.h @@ -94,8 +94,11 @@ simdjson_inline simdjson_result document::get_double() noexcept { simdjson_inline simdjson_result document::get_double_in_string() noexcept { return get_root_value_iterator().get_root_double_in_string(true); } -simdjson_inline simdjson_result document::get_string() noexcept { - return get_root_value_iterator().get_root_string(true); +simdjson_inline simdjson_result document::get_string(bool allow_replacement) noexcept { + return get_root_value_iterator().get_root_string(true, allow_replacement); +} +simdjson_inline simdjson_result document::get_wobbly_string() noexcept { + return get_root_value_iterator().get_root_wobbly_string(true); } simdjson_inline simdjson_result document::get_raw_json_string() noexcept { return get_root_value_iterator().get_root_raw_json_string(true); @@ -110,7 +113,7 @@ simdjson_inline simdjson_result document::is_null() noexcept { template<> simdjson_inline simdjson_result document::get() & noexcept { return get_array(); } template<> simdjson_inline simdjson_result document::get() & noexcept { return get_object(); } template<> simdjson_inline simdjson_result document::get() & noexcept { return get_raw_json_string(); } -template<> simdjson_inline simdjson_result document::get() & noexcept { return get_string(); } +template<> simdjson_inline simdjson_result document::get() & noexcept { return get_string(false); } template<> simdjson_inline simdjson_result document::get() & noexcept { return get_double(); } template<> simdjson_inline simdjson_result document::get() & noexcept { return get_uint64(); } template<> simdjson_inline simdjson_result document::get() & noexcept { return get_int64(); } @@ -118,7 +121,7 @@ template<> simdjson_inline simdjson_result document::get() & noexcept { re template<> simdjson_inline simdjson_result document::get() & noexcept { return get_value(); } template<> simdjson_inline simdjson_result document::get() && noexcept { return get_raw_json_string(); } -template<> simdjson_inline simdjson_result document::get() && noexcept { return get_string(); } +template<> simdjson_inline simdjson_result document::get() && noexcept { return get_string(false); } template<> simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_double(); } template<> simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_uint64(); } template<> simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_int64(); } @@ -138,7 +141,7 @@ simdjson_inline document::operator object() & noexcept(false) { return get_objec simdjson_inline document::operator uint64_t() noexcept(false) { return get_uint64(); } simdjson_inline document::operator int64_t() noexcept(false) { return get_int64(); } simdjson_inline document::operator double() noexcept(false) { return get_double(); } -simdjson_inline document::operator std::string_view() noexcept(false) { return get_string(); } +simdjson_inline document::operator std::string_view() noexcept(false) { return get_string(false); } simdjson_inline document::operator raw_json_string() noexcept(false) { return get_raw_json_string(); } simdjson_inline document::operator bool() noexcept(false) { return get_bool(); } simdjson_inline document::operator value() noexcept(false) { return get_value(); } @@ -359,9 +362,13 @@ simdjson_inline simdjson_result simdjson_result simdjson_result::get_string() noexcept { +simdjson_inline simdjson_result simdjson_result::get_string(bool allow_replacement) noexcept { if (error()) { return error(); } - return first.get_string(); + return first.get_string(allow_replacement); +} +simdjson_inline simdjson_result simdjson_result::get_wobbly_string() noexcept { + if (error()) { return error(); } + return first.get_wobbly_string(); } simdjson_inline simdjson_result simdjson_result::get_raw_json_string() noexcept { if (error()) { return error(); } @@ -538,7 +545,8 @@ simdjson_inline simdjson_result document_reference::get_int64() noexcep simdjson_inline simdjson_result document_reference::get_int64_in_string() noexcept { return doc->get_root_value_iterator().get_root_int64_in_string(false); } simdjson_inline simdjson_result document_reference::get_double() noexcept { return doc->get_root_value_iterator().get_root_double(false); } simdjson_inline simdjson_result document_reference::get_double_in_string() noexcept { return doc->get_root_value_iterator().get_root_double(false); } -simdjson_inline simdjson_result document_reference::get_string() noexcept { return doc->get_root_value_iterator().get_root_string(false); } +simdjson_inline simdjson_result document_reference::get_string(bool allow_replacement) noexcept { return doc->get_root_value_iterator().get_root_string(false, allow_replacement); } +simdjson_inline simdjson_result document_reference::get_wobbly_string() noexcept { return doc->get_root_value_iterator().get_root_wobbly_string(false); } simdjson_inline simdjson_result document_reference::get_raw_json_string() noexcept { return doc->get_root_value_iterator().get_root_raw_json_string(false); } simdjson_inline simdjson_result document_reference::get_bool() noexcept { return doc->get_root_value_iterator().get_root_bool(false); } simdjson_inline simdjson_result document_reference::get_value() noexcept { return doc->get_value(); } @@ -670,9 +678,13 @@ simdjson_inline simdjson_result simdjson_result simdjson_result::get_string() noexcept { +simdjson_inline simdjson_result simdjson_result::get_string(bool allow_replacement) noexcept { if (error()) { return error(); } - return first.get_string(); + return first.get_string(allow_replacement); +} +simdjson_inline simdjson_result simdjson_result::get_wobbly_string() noexcept { + if (error()) { return error(); } + return first.get_wobbly_string(); } simdjson_inline simdjson_result simdjson_result::get_raw_json_string() noexcept { if (error()) { return error(); } diff --git a/include/simdjson/generic/ondemand/document.h b/include/simdjson/generic/ondemand/document.h index 9a770c4c7..63d33a53b 100644 --- a/include/simdjson/generic/ondemand/document.h +++ b/include/simdjson/generic/ondemand/document.h @@ -96,11 +96,24 @@ public: * * Important: Calling get_string() twice on the same document is an error. * + * @param Whether to allow a replacement character for unmatched surrogate pairs. * @returns An UTF-8 string. The string is stored in the parser and will be invalidated the next * time it parses a document or when it is destroyed. * @returns INCORRECT_TYPE if the JSON value is not a string. */ - simdjson_inline simdjson_result get_string() noexcept; + simdjson_inline simdjson_result get_string(bool allow_replacement = false) noexcept; + /** + * Cast this JSON value to a string. + * + * The string is not guaranteed to be valid UTF-8. See https://simonsapin.github.io/wtf-8/ + * + * Important: Calling get_wobbly_string() twice on the same document is an error. + * + * @returns An UTF-8 string. The string is stored in the parser and will be invalidated the next + * time it parses a document or when it is destroyed. + * @returns INCORRECT_TYPE if the JSON value is not a string. + */ + simdjson_inline simdjson_result get_wobbly_string() noexcept; /** * Cast this JSON value to a raw_json_string. * @@ -599,7 +612,8 @@ public: simdjson_inline simdjson_result get_int64_in_string() noexcept; simdjson_inline simdjson_result get_double() noexcept; simdjson_inline simdjson_result get_double_in_string() noexcept; - simdjson_inline simdjson_result get_string() noexcept; + simdjson_inline simdjson_result get_string(bool allow_replacement = false) noexcept; + simdjson_inline simdjson_result get_wobbly_string() noexcept; simdjson_inline simdjson_result get_raw_json_string() noexcept; simdjson_inline simdjson_result get_bool() noexcept; simdjson_inline simdjson_result get_value() noexcept; @@ -667,7 +681,8 @@ public: simdjson_inline simdjson_result get_int64_in_string() noexcept; simdjson_inline simdjson_result get_double() noexcept; simdjson_inline simdjson_result get_double_in_string() noexcept; - simdjson_inline simdjson_result get_string() noexcept; + simdjson_inline simdjson_result get_string(bool allow_replacement = false) noexcept; + simdjson_inline simdjson_result get_wobbly_string() noexcept; simdjson_inline simdjson_result get_raw_json_string() noexcept; simdjson_inline simdjson_result get_bool() noexcept; simdjson_inline simdjson_result get_value() noexcept; @@ -737,7 +752,8 @@ public: simdjson_inline simdjson_result get_int64_in_string() noexcept; simdjson_inline simdjson_result get_double() noexcept; simdjson_inline simdjson_result get_double_in_string() noexcept; - simdjson_inline simdjson_result get_string() noexcept; + simdjson_inline simdjson_result get_string(bool allow_replacement = false) noexcept; + simdjson_inline simdjson_result get_wobbly_string() noexcept; simdjson_inline simdjson_result get_raw_json_string() noexcept; simdjson_inline simdjson_result get_bool() noexcept; simdjson_inline simdjson_result get_value() noexcept; diff --git a/include/simdjson/generic/ondemand/field-inl.h b/include/simdjson/generic/ondemand/field-inl.h index 9bfa5217b..6df0ecd69 100644 --- a/include/simdjson/generic/ondemand/field-inl.h +++ b/include/simdjson/generic/ondemand/field-inl.h @@ -21,9 +21,9 @@ simdjson_inline simdjson_result field::start(const value_iterator &parent return field(key, parent_iter.child()); } -simdjson_inline simdjson_warn_unused simdjson_result field::unescaped_key() noexcept { +simdjson_inline simdjson_warn_unused simdjson_result field::unescaped_key(bool allow_replacement) noexcept { SIMDJSON_ASSUME(first.buf != nullptr); // We would like to call .alive() but Visual Studio won't let us. - simdjson_result answer = first.unescape(second.iter.json_iter()); + simdjson_result answer = first.unescape(second.iter.json_iter(), allow_replacement); first.consume(); return answer; } @@ -66,9 +66,9 @@ simdjson_inline simdjson_result simdjson_result::unescaped_key() noexcept { +simdjson_inline simdjson_result simdjson_result::unescaped_key(bool allow_replacement) noexcept { if (error()) { return error(); } - return first.unescaped_key(); + return first.unescaped_key(allow_replacement); } simdjson_inline simdjson_result simdjson_result::value() noexcept { if (error()) { return error(); } diff --git a/include/simdjson/generic/ondemand/field.h b/include/simdjson/generic/ondemand/field.h index ab46c4f27..87922ca50 100644 --- a/include/simdjson/generic/ondemand/field.h +++ b/include/simdjson/generic/ondemand/field.h @@ -28,7 +28,7 @@ public: * This consumes the key: once you have called unescaped_key(), you cannot * call it again nor can you call key(). */ - simdjson_inline simdjson_warn_unused simdjson_result unescaped_key() noexcept; + simdjson_inline simdjson_warn_unused simdjson_result unescaped_key(bool allow_replacement) noexcept; /** * Get the key as a raw_json_string. Can be used for direct comparison with * an unescaped C string: e.g., key() == "test". @@ -64,7 +64,7 @@ public: simdjson_inline simdjson_result(error_code error) noexcept; ///< @private simdjson_inline simdjson_result() noexcept = default; - simdjson_inline simdjson_result unescaped_key() noexcept; + simdjson_inline simdjson_result unescaped_key(bool allow_replacement = false) noexcept; simdjson_inline simdjson_result key() noexcept; simdjson_inline simdjson_result value() noexcept; }; diff --git a/include/simdjson/generic/ondemand/json_iterator-inl.h b/include/simdjson/generic/ondemand/json_iterator-inl.h index c2adc3c2b..987b56290 100644 --- a/include/simdjson/generic/ondemand/json_iterator-inl.h +++ b/include/simdjson/generic/ondemand/json_iterator-inl.h @@ -316,8 +316,12 @@ simdjson_inline token_position json_iterator::position() const noexcept { return token.position(); } -simdjson_inline simdjson_result json_iterator::unescape(raw_json_string in) noexcept { - return parser->unescape(in, _string_buf_loc); +simdjson_inline simdjson_result json_iterator::unescape(raw_json_string in, bool allow_replacement) noexcept { + return parser->unescape(in, _string_buf_loc, allow_replacement); +} + +simdjson_inline simdjson_result json_iterator::unescape_wobbly(raw_json_string in) noexcept { + return parser->unescape_wobbly(in, _string_buf_loc); } simdjson_inline void json_iterator::reenter_child(token_position position, depth_t child_depth) noexcept { diff --git a/include/simdjson/generic/ondemand/json_iterator.h b/include/simdjson/generic/ondemand/json_iterator.h index 82d06f8f4..5b109c175 100644 --- a/include/simdjson/generic/ondemand/json_iterator.h +++ b/include/simdjson/generic/ondemand/json_iterator.h @@ -246,7 +246,8 @@ public: * Each raw_json_string should be unescaped once, or else the string buffer might * overflow. */ - simdjson_inline simdjson_result unescape(raw_json_string in) noexcept; + simdjson_inline simdjson_result unescape(raw_json_string in, bool allow_replacement) noexcept; + simdjson_inline simdjson_result unescape_wobbly(raw_json_string in) noexcept; simdjson_inline void reenter_child(token_position position, depth_t child_depth) noexcept; #if SIMDJSON_DEVELOPMENT_CHECKS diff --git a/include/simdjson/generic/ondemand/parser-inl.h b/include/simdjson/generic/ondemand/parser-inl.h index d4c17fc14..9ed454037 100644 --- a/include/simdjson/generic/ondemand/parser-inl.h +++ b/include/simdjson/generic/ondemand/parser-inl.h @@ -116,8 +116,16 @@ simdjson_inline void parser::set_max_capacity(size_t max_capacity) noexcept { } } -simdjson_inline simdjson_warn_unused simdjson_result parser::unescape(raw_json_string in, uint8_t *&dst) const noexcept { - uint8_t *end = implementation->parse_string(in.buf, dst); +simdjson_inline simdjson_warn_unused simdjson_result parser::unescape(raw_json_string in, uint8_t *&dst, bool allow_replacement) const noexcept { + uint8_t *end = implementation->parse_string(in.buf, dst, allow_replacement); + if (!end) { return STRING_ERROR; } + std::string_view result(reinterpret_cast(dst), end-dst); + dst = end; + return result; +} + +simdjson_inline simdjson_warn_unused simdjson_result parser::unescape_wobbly(raw_json_string in, uint8_t *&dst) const noexcept { + uint8_t *end = implementation->parse_wobbly_string(in.buf, dst); if (!end) { return STRING_ERROR; } std::string_view result(reinterpret_cast(dst), end-dst); dst = end; diff --git a/include/simdjson/generic/ondemand/parser.h b/include/simdjson/generic/ondemand/parser.h index f7793fd05..8ce043217 100644 --- a/include/simdjson/generic/ondemand/parser.h +++ b/include/simdjson/generic/ondemand/parser.h @@ -263,6 +263,31 @@ public: /** * Unescape this JSON string, replacing \\ with \, \n with newline, etc. to a user-provided buffer. + * The result must be valid UTF-8. + * The provided pointer is advanced to the end of the string by reference, and a string_view instance + * is returned. You can ensure that your buffer is large enough by allocating a block of memory at least + * as large as the input JSON plus SIMDJSON_PADDING and then unescape all strings to this one buffer. + * + * This unescape function is a low-level function. If you want a more user-friendly approach, you should + * avoid raw_json_string instances (e.g., by calling unescaped_key() instead of key() or get_string() + * instead of get_raw_json_string()). + * + * ## IMPORTANT: string_view lifetime + * + * The string_view is only valid as long as the bytes in dst. + * + * @param raw_json_string input + * @param dst A pointer to a buffer at least large enough to write this string as well as + * an additional SIMDJSON_PADDING bytes. + * @param allow_replacement Whether we allow a replacement if the input string contains unmatched surrogate pairs. + * @return A string_view pointing at the unescaped string in dst + * @error STRING_ERROR if escapes are incorrect. + */ + simdjson_inline simdjson_result unescape(raw_json_string in, uint8_t *&dst, bool allow_replacement = false) const noexcept; + + /** + * Unescape this JSON string, replacing \\ with \, \n with newline, etc. to a user-provided buffer. + * The result may not be valid UTF-8. See https://simonsapin.github.io/wtf-8/ * The provided pointer is advanced to the end of the string by reference, and a string_view instance * is returned. You can ensure that your buffer is large enough by allocating a block of memory at least * as large as the input JSON plus SIMDJSON_PADDING and then unescape all strings to this one buffer. @@ -281,7 +306,8 @@ public: * @return A string_view pointing at the unescaped string in dst * @error STRING_ERROR if escapes are incorrect. */ - simdjson_inline simdjson_result unescape(raw_json_string in, uint8_t *&dst) const noexcept; + simdjson_inline simdjson_result unescape_wobbly(raw_json_string in, uint8_t *&dst) const noexcept; + private: /** @private [for benchmarking access] The implementation to use */ std::unique_ptr implementation{}; diff --git a/include/simdjson/generic/ondemand/raw_json_string-inl.h b/include/simdjson/generic/ondemand/raw_json_string-inl.h index 8396024d2..f5fa610b8 100644 --- a/include/simdjson/generic/ondemand/raw_json_string-inl.h +++ b/include/simdjson/generic/ondemand/raw_json_string-inl.h @@ -143,10 +143,13 @@ simdjson_unused simdjson_inline bool operator!=(std::string_view c, const raw_js } -simdjson_inline simdjson_warn_unused simdjson_result raw_json_string::unescape(json_iterator &iter) const noexcept { - return iter.unescape(*this); +simdjson_inline simdjson_warn_unused simdjson_result raw_json_string::unescape(json_iterator &iter, bool allow_replacement) const noexcept { + return iter.unescape(*this, allow_replacement); } +simdjson_inline simdjson_warn_unused simdjson_result raw_json_string::unescape_wobbly(json_iterator &iter) const noexcept { + return iter.unescape_wobbly(*this); +} simdjson_unused simdjson_inline std::ostream &operator<<(std::ostream &out, const raw_json_string &str) noexcept { bool in_escape = false; @@ -177,9 +180,12 @@ simdjson_inline simdjson_result simdjson_result simdjson_result::unescape(SIMDJSON_IMPLEMENTATION::ondemand::json_iterator &iter) const noexcept { +simdjson_inline simdjson_warn_unused simdjson_result simdjson_result::unescape(SIMDJSON_IMPLEMENTATION::ondemand::json_iterator &iter, bool allow_replacement) const noexcept { if (error()) { return error(); } - return first.unescape(iter); + return first.unescape(iter, allow_replacement); +} +simdjson_inline simdjson_warn_unused simdjson_result simdjson_result::unescape_wobbly(SIMDJSON_IMPLEMENTATION::ondemand::json_iterator &iter) const noexcept { + if (error()) { return error(); } + return first.unescape_wobbly(iter); } - } // namespace simdjson diff --git a/include/simdjson/generic/ondemand/raw_json_string.h b/include/simdjson/generic/ondemand/raw_json_string.h index 5a13531a3..4a8375ba7 100644 --- a/include/simdjson/generic/ondemand/raw_json_string.h +++ b/include/simdjson/generic/ondemand/raw_json_string.h @@ -139,6 +139,20 @@ private: /** * Unescape this JSON string, replacing \\ with \, \n with newline, etc. + * The result will be a valid UTF-8. + * + * ## IMPORTANT: string_view lifetime + * + * The string_view is only valid until the next parse() call on the parser. + * + * @param iter A json_iterator, which contains a buffer where the string will be written. + * @param allow_replacement Whether we allow replacement of invalid surrogate pairs. + */ + simdjson_inline simdjson_warn_unused simdjson_result unescape(json_iterator &iter, bool allow_replacement) const noexcept; + + /** + * Unescape this JSON string, replacing \\ with \, \n with newline, etc. + * The result may not be a valid UTF-8. https://simonsapin.github.io/wtf-8/ * * ## IMPORTANT: string_view lifetime * @@ -146,8 +160,7 @@ private: * * @param iter A json_iterator, which contains a buffer where the string will be written. */ - simdjson_inline simdjson_warn_unused simdjson_result unescape(json_iterator &iter) const noexcept; - + simdjson_inline simdjson_warn_unused simdjson_result unescape_wobbly(json_iterator &iter) const noexcept; const uint8_t * buf{}; friend class object; friend class field; @@ -182,7 +195,8 @@ public: simdjson_inline ~simdjson_result() noexcept = default; ///< @private simdjson_inline simdjson_result raw() const noexcept; - simdjson_inline simdjson_warn_unused simdjson_result unescape(SIMDJSON_IMPLEMENTATION::ondemand::json_iterator &iter) const noexcept; + simdjson_inline simdjson_warn_unused simdjson_result unescape(SIMDJSON_IMPLEMENTATION::ondemand::json_iterator &iter, bool allow_replacement) const noexcept; + simdjson_inline simdjson_warn_unused simdjson_result unescape_wobbly(SIMDJSON_IMPLEMENTATION::ondemand::json_iterator &iter) const noexcept; }; } // namespace simdjson diff --git a/include/simdjson/generic/ondemand/value-inl.h b/include/simdjson/generic/ondemand/value-inl.h index 801bbcd7f..8fed336ca 100644 --- a/include/simdjson/generic/ondemand/value-inl.h +++ b/include/simdjson/generic/ondemand/value-inl.h @@ -30,8 +30,11 @@ simdjson_inline simdjson_result value::start_or_resume_object() noexcept simdjson_inline simdjson_result value::get_raw_json_string() noexcept { return iter.get_raw_json_string(); } -simdjson_inline simdjson_result value::get_string() noexcept { - return iter.get_string(); +simdjson_inline simdjson_result value::get_string(bool allow_replacement) noexcept { + return iter.get_string(allow_replacement); +} +simdjson_inline simdjson_result value::get_wobbly_string() noexcept { + return iter.get_wobbly_string(); } simdjson_inline simdjson_result value::get_double() noexcept { return iter.get_double(); @@ -60,7 +63,7 @@ simdjson_inline simdjson_result value::is_null() noexcept { template<> simdjson_inline simdjson_result value::get() noexcept { return get_array(); } template<> simdjson_inline simdjson_result value::get() noexcept { return get_object(); } template<> simdjson_inline simdjson_result value::get() noexcept { return get_raw_json_string(); } -template<> simdjson_inline simdjson_result value::get() noexcept { return get_string(); } +template<> simdjson_inline simdjson_result value::get() noexcept { return get_string(false); } template<> simdjson_inline simdjson_result value::get() noexcept { return get_number(); } template<> simdjson_inline simdjson_result value::get() noexcept { return get_double(); } template<> simdjson_inline simdjson_result value::get() noexcept { return get_uint64(); } @@ -88,7 +91,7 @@ simdjson_inline value::operator double() noexcept(false) { return get_double(); } simdjson_inline value::operator std::string_view() noexcept(false) { - return get_string(); + return get_string(false); } simdjson_inline value::operator raw_json_string() noexcept(false) { return get_raw_json_string(); @@ -298,9 +301,13 @@ simdjson_inline simdjson_result simdjson_result simdjson_result::get_string() noexcept { +simdjson_inline simdjson_result simdjson_result::get_string(bool allow_replacement) noexcept { if (error()) { return error(); } - return first.get_string(); + return first.get_string(allow_replacement); +} +simdjson_inline simdjson_result simdjson_result::get_wobbly_string() noexcept { + if (error()) { return error(); } + return first.get_wobbly_string(); } simdjson_inline simdjson_result simdjson_result::get_raw_json_string() noexcept { if (error()) { return error(); } diff --git a/include/simdjson/generic/ondemand/value.h b/include/simdjson/generic/ondemand/value.h index a3d5d6638..6cadad14c 100644 --- a/include/simdjson/generic/ondemand/value.h +++ b/include/simdjson/generic/ondemand/value.h @@ -129,8 +129,23 @@ public: * time it parses a document or when it is destroyed. * @returns INCORRECT_TYPE if the JSON value is not a string. */ - simdjson_inline simdjson_result get_string() noexcept; + simdjson_inline simdjson_result get_string(bool allow_replacement = false) noexcept; + + /** + * Cast this JSON value to a "wobbly" string. + * + * The string is may not be a valid UTF-8 string. + * See https://simonsapin.github.io/wtf-8/ + * + * Important: a value should be consumed once. Calling get_wobbly_string() twice on the same value + * is an error. + * + * @returns An UTF-8 string. The string is stored in the parser and will be invalidated the next + * time it parses a document or when it is destroyed. + * @returns INCORRECT_TYPE if the JSON value is not a string. + */ + simdjson_inline simdjson_result get_wobbly_string() noexcept; /** * Cast this JSON value to a raw_json_string. * @@ -585,7 +600,8 @@ public: simdjson_inline simdjson_result get_int64_in_string() noexcept; simdjson_inline simdjson_result get_double() noexcept; simdjson_inline simdjson_result get_double_in_string() noexcept; - simdjson_inline simdjson_result get_string() noexcept; + simdjson_inline simdjson_result get_string(bool allow_replacement = false) noexcept; + simdjson_inline simdjson_result get_wobbly_string() noexcept; simdjson_inline simdjson_result get_raw_json_string() noexcept; simdjson_inline simdjson_result get_bool() noexcept; simdjson_inline simdjson_result is_null() noexcept; diff --git a/include/simdjson/generic/ondemand/value_iterator-inl.h b/include/simdjson/generic/ondemand/value_iterator-inl.h index 9de9b4543..c6785994e 100644 --- a/include/simdjson/generic/ondemand/value_iterator-inl.h +++ b/include/simdjson/generic/ondemand/value_iterator-inl.h @@ -475,8 +475,11 @@ simdjson_warn_unused simdjson_inline simdjson_result value_iterator::parse return is_null_string; } -simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_string() noexcept { - return get_raw_json_string().unescape(json_iter()); +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_string(bool allow_replacement) noexcept { + return get_raw_json_string().unescape(json_iter(), allow_replacement); +} +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_wobbly_string() noexcept { + return get_raw_json_string().unescape_wobbly(json_iter()); } simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_raw_json_string() noexcept { auto json = peek_scalar("string"); @@ -592,8 +595,11 @@ simdjson_inline simdjson_result value_iterator::get_root_number(bool che advance_root_scalar("number"); return num; } -simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_string(bool check_trailing) noexcept { - return get_root_raw_json_string(check_trailing).unescape(json_iter()); +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_string(bool check_trailing, bool allow_replacement) noexcept { + return get_root_raw_json_string(check_trailing).unescape(json_iter(), allow_replacement); +} +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_wobbly_string(bool check_trailing) noexcept { + return get_root_raw_json_string(check_trailing).unescape_wobbly(json_iter()); } simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_raw_json_string(bool check_trailing) noexcept { auto json = peek_scalar("string"); diff --git a/include/simdjson/generic/ondemand/value_iterator.h b/include/simdjson/generic/ondemand/value_iterator.h index fe649675f..8a48dfbd4 100644 --- a/include/simdjson/generic/ondemand/value_iterator.h +++ b/include/simdjson/generic/ondemand/value_iterator.h @@ -280,7 +280,8 @@ public: * @{ */ - simdjson_warn_unused simdjson_inline simdjson_result get_string() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_string(bool allow_replacement) noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_wobbly_string() noexcept; simdjson_warn_unused simdjson_inline simdjson_result get_raw_json_string() noexcept; simdjson_warn_unused simdjson_inline simdjson_result get_uint64() noexcept; simdjson_warn_unused simdjson_inline simdjson_result get_uint64_in_string() noexcept; @@ -295,7 +296,8 @@ public: simdjson_warn_unused simdjson_inline simdjson_result get_number_type() noexcept; simdjson_warn_unused simdjson_inline simdjson_result get_number() noexcept; - simdjson_warn_unused simdjson_inline simdjson_result get_root_string(bool check_trailing) noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_root_string(bool check_trailing, bool allow_replacement) noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_root_wobbly_string(bool check_trailing) noexcept; simdjson_warn_unused simdjson_inline simdjson_result get_root_raw_json_string(bool check_trailing) noexcept; simdjson_warn_unused simdjson_inline simdjson_result get_root_uint64(bool check_trailing) noexcept; simdjson_warn_unused simdjson_inline simdjson_result get_root_uint64_in_string(bool check_trailing) noexcept; diff --git a/include/simdjson/internal/dom_parser_implementation.h b/include/simdjson/internal/dom_parser_implementation.h index d85c99214..7aaaeb403 100644 --- a/include/simdjson/internal/dom_parser_implementation.h +++ b/include/simdjson/internal/dom_parser_implementation.h @@ -114,9 +114,26 @@ public: * * @param str pointer to the beginning of a valid UTF-8 JSON string, must end with an unescaped quote. * @param dst pointer to a destination buffer, it must point a region in memory of sufficient size. + * @param allow_replacement whether we allow a replacement character when the UTF-8 contains unmatched surrogate pairs. * @return end of the of the written region (exclusive) or nullptr in case of error. */ - simdjson_warn_unused virtual uint8_t *parse_string(const uint8_t *src, uint8_t *dst) const noexcept = 0; + simdjson_warn_unused virtual uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept = 0; + + /** + * Unescape a NON-valid UTF-8 string from src to dst, stopping at a final unescaped quote. There + * must be an unescaped quote terminating the string. It returns the final output + * position as pointer. In case of error (e.g., the string has bad escaped codes), + * then null_nullptrptr is returned. It is assumed that the output buffer is large + * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + + * SIMDJSON_PADDING bytes. + * + * Overridden by each implementation. + * + * @param str pointer to the beginning of a possibly invalid UTF-8 JSON string, must end with an unescaped quote. + * @param dst pointer to a destination buffer, it must point a region in memory of sufficient size. + * @return end of the of the written region (exclusive) or nullptr in case of error. + */ + simdjson_warn_unused virtual uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept = 0; /** * Change the capacity of this parser. diff --git a/src/arm64/dom_parser_implementation.cpp b/src/arm64/dom_parser_implementation.cpp index 085d0a260..dca665994 100644 --- a/src/arm64/dom_parser_implementation.cpp +++ b/src/arm64/dom_parser_implementation.cpp @@ -152,8 +152,12 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu return stage2::tape_builder::parse_document(*this, _doc); } -simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst) const noexcept { - return arm64::stringparsing::parse_string(src, dst); +simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept { + return arm64::stringparsing::parse_string(src, dst, allow_replacement); +} + +simdjson_warn_unused uint8_t *dom_parser_implementation::parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept { + return arm64::stringparsing::parse_wobbly_string(src, dst); } simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { diff --git a/src/fallback/dom_parser_implementation.cpp b/src/fallback/dom_parser_implementation.cpp index 3f748f1bc..f186cda29 100644 --- a/src/fallback/dom_parser_implementation.cpp +++ b/src/fallback/dom_parser_implementation.cpp @@ -360,8 +360,12 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu return stage2::tape_builder::parse_document(*this, _doc); } -simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst) const noexcept { - return fallback::stringparsing::parse_string(src, dst); +simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool replacement_char) const noexcept { + return fallback::stringparsing::parse_string(src, dst, replacement_char); +} + +simdjson_warn_unused uint8_t *dom_parser_implementation::parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept { + return fallback::stringparsing::parse_wobbly_string(src, dst); } simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { diff --git a/src/generic/stage2/stringparsing.h b/src/generic/stage2/stringparsing.h index 757287ed5..275985fe9 100644 --- a/src/generic/stage2/stringparsing.h +++ b/src/generic/stage2/stringparsing.h @@ -41,7 +41,9 @@ static const uint8_t escape_map[256] = { // We work in little-endian then swap at write time simdjson_warn_unused simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, - uint8_t **dst_ptr) { + uint8_t **dst_ptr, bool allow_replacement) { + // Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD) + constexpr uint32_t substitution_code_point = 0xfffd; // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the // conversion isn't valid; we defer the check for this to inside the // multilingual plane check @@ -56,33 +58,79 @@ simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, const uint8_t *src_data = *src_ptr; /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ if (((src_data[0] << 8) | src_data[1]) != ((static_cast ('\\') << 8) | static_cast ('u'))) { - return false; - } - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); + if(!allow_replacement) { return false; } + code_point = substitution_code_point; + } else { + uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - // We have already checked that the high surrogate is valid and - // (code_point - 0xd800) < 1024. - // - // Check that code_point_2 is in the range 0xdc00..0xdfff - // and that code_point_2 was parsed from valid hex. - uint32_t low_bit = code_point_2 - 0xdc00; - if (low_bit >> 10) { - return false; - } + // We have already checked that the high surrogate is valid and + // (code_point - 0xd800) < 1024. + // + // Check that code_point_2 is in the range 0xdc00..0xdfff + // and that code_point_2 was parsed from valid hex. + uint32_t low_bit = code_point_2 - 0xdc00; + if (low_bit >> 10) { + if(!allow_replacement) { return false; } + code_point = substitution_code_point; + } else { + code_point = (((code_point - 0xd800) << 10) | low_bit) + 0x10000; + *src_ptr += 6; + } - code_point = - (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; + } } else if (code_point >= 0xdc00 && code_point <= 0xdfff) { // If we encounter a low surrogate (not preceded by a high surrogate) // then we have an error. - return false; + if(!allow_replacement) { return false; } + code_point = substitution_code_point; } size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); *dst_ptr += offset; return offset > 0; } + +// handle a unicode codepoint using the wobbly convention +// https://simonsapin.github.io/wtf-8/ +// write appropriate values into dest +// src will advance 6 bytes or 12 bytes +// dest will advance a variable amount (return via pointer) +// return true if the unicode codepoint was valid +// We work in little-endian then swap at write time +simdjson_warn_unused +simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t **src_ptr, + uint8_t **dst_ptr) { + // It is not ideal that this function is nearly identical to handle_unicode_codepoint. + // + // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the + // conversion isn't valid; we defer the check for this to inside the + // multilingual plane check + uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); + *src_ptr += 6; + // If we found a high surrogate, we must + // check for low surrogate for characters + // outside the Basic + // Multilingual Plane. + if (code_point >= 0xd800 && code_point < 0xdc00) { + const uint8_t *src_data = *src_ptr; + /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ + if (((src_data[0] << 8) | src_data[1]) == ((static_cast ('\\') << 8) | static_cast ('u'))) { + uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); + uint32_t low_bit = code_point_2 - 0xdc00; + if ((low_bit >> 10) == 0) { + code_point = + (((code_point - 0xd800) << 10) | low_bit) + 0x10000; + *src_ptr += 6; + } + } + } + + size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); + *dst_ptr += offset; + return offset > 0; +} + + /** * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There * must be an unescaped quote terminating the string. It returns the final output @@ -91,7 +139,7 @@ simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + * SIMDJSON_PADDING bytes. */ -simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst) { +simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) { while (1) { // Copy the next n bytes, and find the backslash and quote in them. auto bs_quote = backslash_and_quote::copy_and_find(src, dst); @@ -110,7 +158,54 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, u within the unicode codepoint handling code. */ src += bs_dist; dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst)) { + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } else { + /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and + * write bs_dist+1 characters to output + * note this may reach beyond the part of the buffer we've actually + * seen. I think this is ok */ + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; /* bogus escape value is an error */ + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } else { + /* they are the same. Since they can't co-occur, it means we + * encountered neither. */ + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } + /* can't be reached */ + return nullptr; +} + +simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { + // It is not ideal that this function is nearly identical to parse_string. + while (1) { + // Copy the next n bytes, and find the backslash and quote in them. + auto bs_quote = backslash_and_quote::copy_and_find(src, dst); + // If the next thing is the end quote, copy and return + if (bs_quote.has_quote_first()) { + // we encountered quotes first. Move dst to point to quotes and exit + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + /* find out where the backspace is */ + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + /* we encountered backslash first. Handle backslash */ + if (escape_char == 'u') { + /* move src/dst up to the start; they will be further adjusted + within the unicode codepoint handling code. */ + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint_wobbly(&src, &dst)) { return nullptr; } } else { diff --git a/src/generic/stage2/tape_builder.h b/src/generic/stage2/tape_builder.h index 04943676d..8a1039e19 100644 --- a/src/generic/stage2/tape_builder.h +++ b/src/generic/stage2/tape_builder.h @@ -145,7 +145,7 @@ simdjson_inline tape_builder::tape_builder(dom::document &doc) noexcept : tape{d simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { iter.log_value(key ? "key" : "string"); uint8_t *dst = on_start_string(iter); - dst = stringparsing::parse_string(value+1, dst); + dst = stringparsing::parse_string(value+1, dst, false); // We do not allow replacement when the escape characters are invalid. if (dst == nullptr) { iter.log_error("Invalid escape in string"); return STRING_ERROR; diff --git a/src/haswell/dom_parser_implementation.cpp b/src/haswell/dom_parser_implementation.cpp index 5ce0eaef0..413699a7d 100644 --- a/src/haswell/dom_parser_implementation.cpp +++ b/src/haswell/dom_parser_implementation.cpp @@ -153,8 +153,12 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu return stage2::tape_builder::parse_document(*this, _doc); } -simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst) const noexcept { - return haswell::stringparsing::parse_string(src, dst); +simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool replacement_char) const noexcept { + return haswell::stringparsing::parse_string(src, dst, replacement_char); +} + +simdjson_warn_unused uint8_t *dom_parser_implementation::parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept { + return haswell::stringparsing::parse_wobbly_string(src, dst); } simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { diff --git a/src/icelake/dom_parser_implementation.cpp b/src/icelake/dom_parser_implementation.cpp index a1eeb3ee1..dd6923f14 100644 --- a/src/icelake/dom_parser_implementation.cpp +++ b/src/icelake/dom_parser_implementation.cpp @@ -199,8 +199,12 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu return stage2::tape_builder::parse_document(*this, _doc); } -simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst) const noexcept { - return icelake::stringparsing::parse_string(src, dst); +simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool replacement_char) const noexcept { + return icelake::stringparsing::parse_string(src, dst, replacement_char); +} + +simdjson_warn_unused uint8_t *dom_parser_implementation::parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept { + return icelake::stringparsing::parse_wobbly_string(src, dst); } simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { diff --git a/src/ppc64/dom_parser_implementation.cpp b/src/ppc64/dom_parser_implementation.cpp index a196f5d1c..c592354c4 100644 --- a/src/ppc64/dom_parser_implementation.cpp +++ b/src/ppc64/dom_parser_implementation.cpp @@ -122,8 +122,12 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu return stage2::tape_builder::parse_document(*this, _doc); } -simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst) const noexcept { - return ppc64::stringparsing::parse_string(src, dst); +simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool replacement_char) const noexcept { + return ppc64::stringparsing::parse_string(src, dst, replacement_char); +} + +simdjson_warn_unused uint8_t *dom_parser_implementation::parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept { + return ppc64::stringparsing::parse_wobbly_string(src, dst); } simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { diff --git a/src/westmere/dom_parser_implementation.cpp b/src/westmere/dom_parser_implementation.cpp index 664108c7a..c7a85e48e 100644 --- a/src/westmere/dom_parser_implementation.cpp +++ b/src/westmere/dom_parser_implementation.cpp @@ -152,8 +152,12 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu return stage2::tape_builder::parse_document(*this, _doc); } -simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst) const noexcept { - return westmere::stringparsing::parse_string(src, dst); +simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool replacement_char) const noexcept { + return westmere::stringparsing::parse_string(src, dst, replacement_char); +} + +simdjson_warn_unused uint8_t *dom_parser_implementation::parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept { + return westmere::stringparsing::parse_wobbly_string(src, dst); } simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { diff --git a/tests/ondemand/ondemand_misc_tests.cpp b/tests/ondemand/ondemand_misc_tests.cpp index 265f4e99a..dfce3f194 100644 --- a/tests/ondemand/ondemand_misc_tests.cpp +++ b/tests/ondemand/ondemand_misc_tests.cpp @@ -5,6 +5,56 @@ using namespace simdjson; namespace misc_tests { using namespace std; + bool replacement_char() { + auto fun_phrase = R"( ["I \u2665 Unicode. Even broken \ud800 Unicode." ])"_padded; + std::string_view expected_fun = "I \xe2\x99\xa5 Unicode. Even broken \xef\xbf\xbd Unicode."; + + TEST_START(); + ondemand::parser parser; + ondemand::document doc; + ASSERT_SUCCESS(parser.iterate(fun_phrase).get(doc)); + ondemand::array arr; + ASSERT_SUCCESS( doc.get_array().get(arr)); + std::string_view view; + ASSERT_SUCCESS( arr.at(0).get_string(true).get(view)); + ASSERT_EQUAL(view, expected_fun); + TEST_SUCCEED(); + } + + bool wobbly_tests() { + auto lone_surrogate = R"( "\ud800" )"_padded; + std::string_view expected_lone = "\xed\xa0\x80"; + + + auto fun_phrase = R"( ["I \u2665 Unicode. Even broken \ud800 Unicode." ])"_padded; + std::string_view expected_fun = "I \xe2\x99\xa5 Unicode. Even broken \xed\xa0\x80 Unicode."; + + auto insane_url = R"({"input": "http://example.com/\uDC00\uD834\uDF06\uDC00"} )"_padded; + std::string_view expected_insane = "http://example.com/\xED\xB0\x80\xF0\x9D\x8C\x86\xED\xB0\x80"; + + TEST_START(); + ondemand::parser parser; + ondemand::document doc; + ASSERT_SUCCESS(parser.iterate(lone_surrogate).get(doc)); + std::string_view view; + ASSERT_SUCCESS( doc.get_wobbly_string().get(view)); + ASSERT_EQUAL(view, expected_lone); + + ASSERT_SUCCESS(parser.iterate(fun_phrase).get(doc)); + ondemand::array arr; + ASSERT_SUCCESS( doc.get_array().get(arr)); + ASSERT_SUCCESS( arr.at(0).get_wobbly_string().get(view)); + ASSERT_EQUAL(view, expected_fun); + + ASSERT_SUCCESS(parser.iterate(insane_url).get(doc)); + ondemand::object obj; + ASSERT_SUCCESS( doc.get_object().get(obj)); + ASSERT_SUCCESS( obj["input"].get_wobbly_string().get(view)); + + ASSERT_EQUAL(view, expected_insane); + TEST_SUCCEED(); + } + bool test_get_value() { TEST_START(); ondemand::parser parser; @@ -520,6 +570,8 @@ namespace misc_tests { bool run() { return + replacement_char() && + wobbly_tests() && issue_uffff() && issue_backslash() && issue1870() &&