diff --git a/src/arm64/dom_parser_implementation.cpp b/src/arm64/dom_parser_implementation.cpp index 0685c982b..f9dfa3922 100644 --- a/src/arm64/dom_parser_implementation.cpp +++ b/src/arm64/dom_parser_implementation.cpp @@ -105,7 +105,7 @@ WARN_UNUSED error_code dom_parser_implementation::stage1(const uint8_t *_buf, si this->len = _len; return arm64::stage1::json_structural_indexer::index<64>(buf, len, *this, streaming); } - +#include "generic/stage1/utf8_validator.h" } // namespace arm64 } // namespace simdjson diff --git a/src/generic/stage1/buf_block_reader.h b/src/generic/stage1/buf_block_reader.h index af47933c7..910cbfb68 100644 --- a/src/generic/stage1/buf_block_reader.h +++ b/src/generic/stage1/buf_block_reader.h @@ -6,6 +6,9 @@ public: really_inline size_t block_index(); really_inline bool has_full_block() const; really_inline const uint8_t *full_block() const; + really_inline bool has_remainder() const { + return idx < len; + } /** * Get the last block, padded with spaces. * diff --git a/src/generic/stage1/utf8_validator.h b/src/generic/stage1/utf8_validator.h new file mode 100644 index 000000000..384cab0e7 --- /dev/null +++ b/src/generic/stage1/utf8_validator.h @@ -0,0 +1,29 @@ +namespace stage1 { +/** + * Validates that the string is actual UTF-8. + */ +template +bool utf8_validate(const uint8_t * input, size_t length) { + checker c{}; + buf_block_reader<64> reader(input, length); + while (reader.has_full_block()) { + simd::simd8x64 in(reader.full_block()); + c.check_next_input(in); + reader.advance(); + } + + if (likely(reader.has_remainder())) { + uint8_t block[64]{}; + reader.get_remainder(block); + simd::simd8x64 in(block); + c.check_next_input(in); + reader.advance(); + } + return c.errors() == error_code::SUCCESS; +} + +bool utf8_validate(const char * input, size_t length) { + return utf8_validate((const uint8_t *)input,length); +} + +} // namespace stage1 \ No newline at end of file diff --git a/src/haswell/dom_parser_implementation.cpp b/src/haswell/dom_parser_implementation.cpp index f9d3461da..4554a0c79 100644 --- a/src/haswell/dom_parser_implementation.cpp +++ b/src/haswell/dom_parser_implementation.cpp @@ -93,7 +93,7 @@ WARN_UNUSED error_code dom_parser_implementation::stage1(const uint8_t *_buf, si this->len = _len; return haswell::stage1::json_structural_indexer::index<128>(_buf, _len, *this, streaming); } - +#include "generic/stage1/utf8_validator.h" } // namespace haswell } // namespace simdjson UNTARGET_REGION diff --git a/src/westmere/dom_parser_implementation.cpp b/src/westmere/dom_parser_implementation.cpp index 89017e4ba..f996277d7 100644 --- a/src/westmere/dom_parser_implementation.cpp +++ b/src/westmere/dom_parser_implementation.cpp @@ -94,7 +94,7 @@ WARN_UNUSED error_code dom_parser_implementation::stage1(const uint8_t *_buf, si this->len = _len; return westmere::stage1::json_structural_indexer::index<64>(_buf, _len, *this, streaming); } - +#include "generic/stage1/utf8_validator.h" } // namespace westmere } // namespace simdjson UNTARGET_REGION