Compare commits

...

3 Commits

Author SHA1 Message Date
Daniel Lemire 8e85352df4 Merge branch 'master' into parse_string_if_needed 2026-01-21 10:10:42 -05:00
Daniel Lemire 403b8bfb91 let us be careful and not change the API 2024-07-11 19:36:12 -04:00
Daniel Lemire 76f45a0c4b fix: add parse_string_if_needed function 2024-07-11 17:57:05 -04:00
22 changed files with 219 additions and 15 deletions
@@ -17,6 +17,7 @@ using namespace simd;
struct backslash_and_quote { struct backslash_and_quote {
public: public:
static constexpr uint32_t BYTES_PROCESSED = 32; static constexpr uint32_t BYTES_PROCESSED = 32;
// We only copy if dst is non-null.
simdjson_inline backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); simdjson_inline backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst);
simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; }
@@ -34,8 +35,10 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin
static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes");
simd8<uint8_t> v0(src); simd8<uint8_t> v0(src);
simd8<uint8_t> v1(src + sizeof(v0)); simd8<uint8_t> v1(src + sizeof(v0));
if(dst != nullptr) {
v0.store(dst); v0.store(dst);
v1.store(dst + sizeof(v0)); v1.store(dst + sizeof(v0));
}
// Getting a 64-bit bitmask is much cheaper than multiple 16-bit bitmasks on ARM; therefore, we // Getting a 64-bit bitmask is much cheaper than multiple 16-bit bitmasks on ARM; therefore, we
// smash them together into a 64-byte mask and get the bitmask from there. // smash them together into a 64-byte mask and get the bitmask from there.
@@ -13,6 +13,7 @@ namespace {
struct backslash_and_quote { struct backslash_and_quote {
public: public:
static constexpr uint32_t BYTES_PROCESSED = 1; static constexpr uint32_t BYTES_PROCESSED = 1;
// We only copy if dst is non-null.
simdjson_inline backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); simdjson_inline backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst);
simdjson_inline bool has_quote_first() { return c == '"'; } simdjson_inline bool has_quote_first() { return c == '"'; }
@@ -25,7 +26,9 @@ public:
simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) {
// store to dest unconditionally - we can overwrite the bits we don't like later // store to dest unconditionally - we can overwrite the bits we don't like later
if(dst != nullptr) {
dst[0] = src[0]; dst[0] = src[0];
}
return { src[0] }; return { src[0] };
} }
@@ -40,6 +40,7 @@ public:
simdjson_warn_unused error_code stage1(const uint8_t *buf, size_t len, stage1_mode partial) noexcept final; simdjson_warn_unused error_code stage1(const uint8_t *buf, size_t len, stage1_mode partial) noexcept final;
simdjson_warn_unused error_code stage2(dom::document &doc) noexcept final; simdjson_warn_unused error_code stage2(dom::document &doc) noexcept final;
simdjson_warn_unused error_code stage2_next(dom::document &doc) noexcept final; simdjson_warn_unused error_code stage2_next(dom::document &doc) noexcept final;
simdjson_warn_unused std::pair<const uint8_t *,bool> parse_string_if_needed(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept final;
simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept final; simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept final;
simdjson_warn_unused uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept final; simdjson_warn_unused uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept final;
inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final; inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final;
@@ -364,7 +364,7 @@ simdjson_inline simdjson_result<std::string_view> json_iterator::unescape(raw_js
#endif // !defined(SIMDJSON_VISUAL_STUDIO) && !defined(SIMDJSON_CLANG_VISUAL_STUDIO) #endif // !defined(SIMDJSON_VISUAL_STUDIO) && !defined(SIMDJSON_CLANG_VISUAL_STUDIO)
return result; return result;
#else #else
return parser->unescape(in, _string_buf_loc, allow_replacement); return parser->unescape_maybe(in, _string_buf_loc, allow_replacement);
#endif #endif
} }
@@ -179,6 +179,20 @@ simdjson_inline void parser::set_max_capacity(size_t max_capacity) noexcept {
} }
} }
simdjson_inline simdjson_warn_unused simdjson_result<std::string_view> parser::unescape_maybe(raw_json_string in, uint8_t *&dst, bool allow_replacement) const noexcept {
std::pair<const uint8_t *, bool> result = implementation->parse_string_if_needed(in.buf, dst, allow_replacement);
const uint8_t *end = result.first;
bool copied = result.second;
if (!end) { return STRING_ERROR; }
if(copied) {
std::string_view strresult(reinterpret_cast<const char *>(dst), end-dst);
dst = const_cast<uint8_t *>(end);
return strresult;
}
// fast path, no copy was made!!!
return std::string_view(reinterpret_cast<const char *>(in.buf), end-in.buf);
}
simdjson_inline simdjson_warn_unused simdjson_result<std::string_view> parser::unescape(raw_json_string in, uint8_t *&dst, bool allow_replacement) const noexcept { simdjson_inline simdjson_warn_unused simdjson_result<std::string_view> parser::unescape(raw_json_string in, uint8_t *&dst, bool allow_replacement) const noexcept {
uint8_t *end = implementation->parse_string(in.buf, dst, allow_replacement); uint8_t *end = implementation->parse_string(in.buf, dst, allow_replacement);
if (!end) { return STRING_ERROR; } if (!end) { return STRING_ERROR; }
@@ -334,6 +334,32 @@ public:
*/ */
simdjson_inline simdjson_result<std::string_view> unescape(raw_json_string in, uint8_t *&dst, bool allow_replacement = false) const noexcept; simdjson_inline simdjson_result<std::string_view> unescape(raw_json_string in, uint8_t *&dst, bool allow_replacement = false) const noexcept;
/**
* Unescape this JSON string, replacing \\ with \, \n with newline, etc. to a user-provided buffer if
* needed. If no escaping is done, the string is returned as is and dst is not not changed.
* The result must be valid UTF-8.
* The provided pointer is advanced to the end of the string by reference if a copy is needed,
* and a string_view instance
* is returned. You can ensure that your buffer is large enough by allocating a block of memory at least
* as large as the input JSON plus SIMDJSON_PADDING and then unescape all strings to this one buffer.
*
* This unescape_maybe function is a low-level function. If you want a more user-friendly approach, you should
* avoid raw_json_string instances (e.g., by calling unescaped_key() instead of key() or get_string()
* instead of get_raw_json_string()).
*
* ## IMPORTANT: string_view lifetime
*
* The string_view is only valid as long as the bytes in dst.
*
* @param raw_json_string input
* @param dst A pointer to a buffer at least large enough to write this string as well as
* an additional SIMDJSON_PADDING bytes.
* @param allow_replacement Whether we allow a replacement if the input string contains unmatched surrogate pairs.
* @return A string_view pointing at the unescaped string in dst
* @error STRING_ERROR if escapes are incorrect.
*/
simdjson_inline simdjson_result<std::string_view> unescape_maybe(raw_json_string in, uint8_t *&dst, bool allow_replacement = false) const noexcept;
/** /**
* Unescape this JSON string, replacing \\ with \, \n with newline, etc. to a user-provided buffer. * Unescape this JSON string, replacing \\ with \, \n with newline, etc. to a user-provided buffer.
* The result may not be valid UTF-8. See https://simonsapin.github.io/wtf-8/ * The result may not be valid UTF-8. See https://simonsapin.github.io/wtf-8/
@@ -17,6 +17,7 @@ using namespace simd;
struct backslash_and_quote { struct backslash_and_quote {
public: public:
static constexpr uint32_t BYTES_PROCESSED = 32; static constexpr uint32_t BYTES_PROCESSED = 32;
// We only copy if dst is non-null.
simdjson_inline backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); simdjson_inline backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst);
simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; }
@@ -34,7 +35,9 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin
static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes");
simd8<uint8_t> v(src); simd8<uint8_t> v(src);
// store to dest unconditionally - we can overwrite the bits we don't like later // store to dest unconditionally - we can overwrite the bits we don't like later
if(dst != nullptr) {
v.store(dst); v.store(dst);
}
return { return {
static_cast<uint32_t>((v == '\\').to_bitmask()), // bs_bits static_cast<uint32_t>((v == '\\').to_bitmask()), // bs_bits
static_cast<uint32_t>((v == '"').to_bitmask()), // quote_bits static_cast<uint32_t>((v == '"').to_bitmask()), // quote_bits
@@ -17,6 +17,7 @@ using namespace simd;
struct backslash_and_quote { struct backslash_and_quote {
public: public:
static constexpr uint32_t BYTES_PROCESSED = 64; static constexpr uint32_t BYTES_PROCESSED = 64;
// We only copy if dst is non-null.
simdjson_inline backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); simdjson_inline backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst);
simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; }
@@ -34,7 +35,9 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin
static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes");
simd8<uint8_t> v(src); simd8<uint8_t> v(src);
// store to dest unconditionally - we can overwrite the bits we don't like later // store to dest unconditionally - we can overwrite the bits we don't like later
if(dst != nullptr) {
v.store(dst); v.store(dst);
}
return { return {
static_cast<uint64_t>(v == '\\'), // bs_bits static_cast<uint64_t>(v == '\\'), // bs_bits
static_cast<uint64_t>(v == '"'), // quote_bits static_cast<uint64_t>(v == '"'), // quote_bits
@@ -4,6 +4,7 @@
#include "simdjson/base.h" #include "simdjson/base.h"
#include "simdjson/error.h" #include "simdjson/error.h"
#include <memory> #include <memory>
#include <utility>
namespace simdjson { namespace simdjson {
@@ -102,6 +103,24 @@ public:
*/ */
simdjson_warn_unused virtual error_code stage2_next(dom::document &doc) noexcept = 0; simdjson_warn_unused virtual error_code stage2_next(dom::document &doc) noexcept = 0;
/**
* Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There
* must be an unescaped quote terminating the string. It returns the final output
* position as pointer. In case of error (e.g., the string has bad escaped codes),
* then null_ptr is returned. If no escaping was required, then no copy is made.
* It is assumed that the output buffer is large
* enough to store the unescapedstring + SIMDJSON_PADDING bytes.
*
* Overridden by each implementation.
*
* @param str pointer to the beginning of a valid UTF-8 JSON string, must end with an unescaped quote.
* @param dst pointer to a destination buffer, it must point a region in memory of sufficient size.
* @param allow_replacement whether we allow a replacement character when the UTF-8 contains unmatched surrogate pairs.
* @return end of the of the written region (exclusive) or nullptr in case of error coupled with a Boolean telling you if a copy was made
*/
simdjson_warn_unused virtual std::pair<const uint8_t *,bool> parse_string_if_needed(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept = 0;
/** /**
* Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There
* must be an unescaped quote terminating the string. It returns the final output * must be an unescaped quote terminating the string. It returns the final output
@@ -17,6 +17,7 @@ using namespace simd;
struct backslash_and_quote { struct backslash_and_quote {
public: public:
static constexpr uint32_t BYTES_PROCESSED = 32; static constexpr uint32_t BYTES_PROCESSED = 32;
// We only copy if dst is non-null.
simdjson_inline backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); simdjson_inline backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst);
simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; }
@@ -33,7 +34,9 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin
// SIMDJSON_PADDING of padding // SIMDJSON_PADDING of padding
static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes");
simd8<uint8_t> v(src); simd8<uint8_t> v(src);
if(dst != nullptr) {
v.store(dst); v.store(dst);
}
return { return {
static_cast<uint32_t>((v == '\\').to_bitmask()), // bs_bits static_cast<uint32_t>((v == '\\').to_bitmask()), // bs_bits
static_cast<uint32_t>((v == '"').to_bitmask()), // quote_bits static_cast<uint32_t>((v == '"').to_bitmask()), // quote_bits
@@ -17,6 +17,7 @@ using namespace simd;
struct backslash_and_quote { struct backslash_and_quote {
public: public:
static constexpr uint32_t BYTES_PROCESSED = 32; static constexpr uint32_t BYTES_PROCESSED = 32;
// We only copy if dst is non-null.
simdjson_inline backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); simdjson_inline backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst);
simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; }
@@ -34,8 +35,10 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin
static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes");
simd8<uint8_t> v0(src); simd8<uint8_t> v0(src);
simd8<uint8_t> v1(src + sizeof(v0)); simd8<uint8_t> v1(src + sizeof(v0));
if(dst != nullptr) {
v0.store(dst); v0.store(dst);
v1.store(dst + sizeof(v0)); v1.store(dst + sizeof(v0));
}
// Getting a 64-bit bitmask is much cheaper than multiple 16-bit bitmasks on LSX; therefore, we // Getting a 64-bit bitmask is much cheaper than multiple 16-bit bitmasks on LSX; therefore, we
// smash them together into a 64-byte mask and get the bitmask from there. // smash them together into a 64-byte mask and get the bitmask from there.
@@ -17,6 +17,7 @@ using namespace simd;
struct backslash_and_quote { struct backslash_and_quote {
public: public:
static constexpr uint32_t BYTES_PROCESSED = 32; static constexpr uint32_t BYTES_PROCESSED = 32;
// We only copy if dst is non-null.
simdjson_inline backslash_and_quote simdjson_inline backslash_and_quote
copy_and_find(const uint8_t *src, uint8_t *dst); copy_and_find(const uint8_t *src, uint8_t *dst);
@@ -44,8 +45,10 @@ backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) {
"SIMDJSON_PADDING bytes"); "SIMDJSON_PADDING bytes");
simd8<uint8_t> v0(src); simd8<uint8_t> v0(src);
simd8<uint8_t> v1(src + sizeof(v0)); simd8<uint8_t> v1(src + sizeof(v0));
if(dst != nullptr) {
v0.store(dst); v0.store(dst);
v1.store(dst + sizeof(v0)); v1.store(dst + sizeof(v0));
}
// Getting a 64-bit bitmask is much cheaper than multiple 16-bit bitmasks on // Getting a 64-bit bitmask is much cheaper than multiple 16-bit bitmasks on
// PPC; therefore, we smash them together into a 64-byte mask and get the // PPC; therefore, we smash them together into a 64-byte mask and get the
@@ -14,6 +14,7 @@ using namespace simd;
struct backslash_and_quote { struct backslash_and_quote {
public: public:
static constexpr uint32_t BYTES_PROCESSED = 32; static constexpr uint32_t BYTES_PROCESSED = 32;
// We only copy if dst is non-null.
simdjson_inline backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); simdjson_inline backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst);
simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; }
@@ -31,8 +32,10 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin
static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes");
simd8<uint8_t> v0(src); simd8<uint8_t> v0(src);
simd8<uint8_t> v1(src + 16); simd8<uint8_t> v1(src + 16);
if(dst != nullptr) {
v0.store(dst); v0.store(dst);
v1.store(dst + 16); v1.store(dst + 16);
}
uint64_t bs_and_quote = simd8x64<bool>(v0 == '\\', v1 == '\\', v0 == '"', v1 == '"').to_bitmask(); uint64_t bs_and_quote = simd8x64<bool>(v0 == '\\', v1 == '\\', v0 == '"', v1 == '"').to_bitmask();
return { return {
uint32_t(bs_and_quote), // bs_bits uint32_t(bs_and_quote), // bs_bits
+4
View File
@@ -150,6 +150,10 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu
return stage2::tape_builder::parse_document<true>(*this, _doc); return stage2::tape_builder::parse_document<true>(*this, _doc);
} }
simdjson_warn_unused std::pair<const uint8_t *, bool> dom_parser_implementation::parse_string_if_needed(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept {
return arm64::stringparsing::parse_string_if_needed(src, dst, allow_replacement);
}
SIMDJSON_NO_SANITIZE_MEMORY SIMDJSON_NO_SANITIZE_MEMORY
simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept { simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept {
return arm64::stringparsing::parse_string(src, dst, allow_replacement); return arm64::stringparsing::parse_string(src, dst, allow_replacement);
+4
View File
@@ -444,6 +444,10 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu
return stage2::tape_builder::parse_document<true>(*this, _doc); return stage2::tape_builder::parse_document<true>(*this, _doc);
} }
simdjson_warn_unused std::pair<const uint8_t *, bool> dom_parser_implementation::parse_string_if_needed(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept {
return fallback::stringparsing::parse_string_if_needed(src, dst, allow_replacement);
}
SIMDJSON_NO_SANITIZE_MEMORY SIMDJSON_NO_SANITIZE_MEMORY
simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool replacement_char) const noexcept { simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool replacement_char) const noexcept {
return fallback::stringparsing::parse_string(src, dst, replacement_char); return fallback::stringparsing::parse_string(src, dst, replacement_char);
+89 -1
View File
@@ -1,4 +1,5 @@
#include <cstdint> #include <cstdio>
#include <cstring>
#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H #ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H
#ifndef SIMDJSON_CONDITIONAL_INCLUDE #ifndef SIMDJSON_CONDITIONAL_INCLUDE
@@ -139,6 +140,93 @@ simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t **src_ptr,
return offset > 0; return offset > 0;
} }
/**
* Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote.
* If there is no need for unescaping, it avoids copying the string.
* There
* must be an unescaped quote terminating the string. It returns the final output
* position as pointer. In case of error (e.g., the string has bad escaped codes),
* then null_ptr is returned. It is assumed that the output buffer is large
* enough. E.g., if src points at 'joe"', then dst needs to have four free bytes +
* SIMDJSON_PADDING bytes.
*/
simdjson_warn_unused simdjson_inline std::pair<const uint8_t *,bool> parse_string_if_needed(const uint8_t *src, uint8_t *dst, bool allow_replacement) {
const uint8_t *srcinit = src;
while (1) {
// Find the backslash and quote in them, we pass null because we do not copy.
auto bs_quote = backslash_and_quote::copy_and_find(src, nullptr);
// If the next thing is the end quote, copy and return
if (bs_quote.has_quote_first()) {
// we encountered quotes first.
return {src + bs_quote.quote_index(), false};
}
if (bs_quote.has_backslash()) {
std::memcpy(dst, srcinit, src - srcinit + backslash_and_quote::BYTES_PROCESSED);
dst += src - srcinit;
auto bs_dist = bs_quote.backslash_index();
uint8_t escape_char = src[bs_dist + 1];
if (escape_char == 'u') {
src += bs_dist;
dst += bs_dist;
if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) {
return {nullptr, true};
}
} else {
uint8_t escape_result = escape_map[escape_char];
if (escape_result == 0u) {
return {nullptr, true};
}
dst[bs_dist] = escape_result;
src += bs_dist + 2;
dst += bs_dist + 1;
}
break;
} else {
src += backslash_and_quote::BYTES_PROCESSED;
}
}
while (1) {
// Copy the next n bytes, and find the backslash and quote in them.
auto bs_quote = backslash_and_quote::copy_and_find(src, dst);
// If the next thing is the end quote, copy and return
if (bs_quote.has_quote_first()) {
// we encountered quotes first. Move dst to point to quotes and exit
return {dst + bs_quote.quote_index(), true};
}
if (bs_quote.has_backslash()) {
/* find out where the backspace is */
auto bs_dist = bs_quote.backslash_index();
uint8_t escape_char = src[bs_dist + 1];
/* we encountered backslash first. Handle backslash */
if (escape_char == 'u') {
/* move src/dst up to the start; they will be further adjusted
within the unicode codepoint handling code. */
src += bs_dist;
dst += bs_dist;
if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) {
return {nullptr, true};
}
} else {
/* simple 1:1 conversion. Will eat bs_dist+2 characters in input and
* write bs_dist+1 characters to output
* note this may reach beyond the part of the buffer we've actually
* seen. I think this is ok */
uint8_t escape_result = escape_map[escape_char];
if (escape_result == 0u) {
return {nullptr, true}; /* bogus escape value is an error */
}
dst[bs_dist] = escape_result;
src += bs_dist + 2;
dst += bs_dist + 1;
}
} else {
/* they are the same. Since they can't co-occur, it means we
* encountered neither. */
src += backslash_and_quote::BYTES_PROCESSED;
dst += backslash_and_quote::BYTES_PROCESSED;
}
}
}
/** /**
* Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There
+4
View File
@@ -147,6 +147,10 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu
return stage2::tape_builder::parse_document<true>(*this, _doc); return stage2::tape_builder::parse_document<true>(*this, _doc);
} }
simdjson_warn_unused std::pair<const uint8_t *, bool> dom_parser_implementation::parse_string_if_needed(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept {
return haswell::stringparsing::parse_string_if_needed(src, dst, allow_replacement);
}
SIMDJSON_NO_SANITIZE_MEMORY SIMDJSON_NO_SANITIZE_MEMORY
simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool replacement_char) const noexcept { simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool replacement_char) const noexcept {
return haswell::stringparsing::parse_string(src, dst, replacement_char); return haswell::stringparsing::parse_string(src, dst, replacement_char);
+4
View File
@@ -193,6 +193,10 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu
return stage2::tape_builder::parse_document<true>(*this, _doc); return stage2::tape_builder::parse_document<true>(*this, _doc);
} }
simdjson_warn_unused std::pair<const uint8_t *, bool> dom_parser_implementation::parse_string_if_needed(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept {
return icelake::stringparsing::parse_string_if_needed(src, dst, allow_replacement);
}
SIMDJSON_NO_SANITIZE_MEMORY SIMDJSON_NO_SANITIZE_MEMORY
simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool replacement_char) const noexcept { simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool replacement_char) const noexcept {
return icelake::stringparsing::parse_string(src, dst, replacement_char); return icelake::stringparsing::parse_string(src, dst, replacement_char);
+4
View File
@@ -110,6 +110,10 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu
return stage2::tape_builder::parse_document<true>(*this, _doc); return stage2::tape_builder::parse_document<true>(*this, _doc);
} }
simdjson_warn_unused std::pair<const uint8_t *, bool> dom_parser_implementation::parse_string_if_needed(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept {
return lasx::stringparsing::parse_string_if_needed(src, dst, allow_replacement);
}
SIMDJSON_NO_SANITIZE_MEMORY SIMDJSON_NO_SANITIZE_MEMORY
simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept { simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept {
return lasx::stringparsing::parse_string(src, dst, allow_replacement); return lasx::stringparsing::parse_string(src, dst, allow_replacement);
+4
View File
@@ -114,6 +114,10 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu
return stage2::tape_builder::parse_document<true>(*this, _doc); return stage2::tape_builder::parse_document<true>(*this, _doc);
} }
simdjson_warn_unused std::pair<const uint8_t *, bool> dom_parser_implementation::parse_string_if_needed(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept {
return lsx::stringparsing::parse_string_if_needed(src, dst, allow_replacement);
}
SIMDJSON_NO_SANITIZE_MEMORY SIMDJSON_NO_SANITIZE_MEMORY
simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept { simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept {
return lsx::stringparsing::parse_string(src, dst, allow_replacement); return lsx::stringparsing::parse_string(src, dst, allow_replacement);
+4
View File
@@ -120,6 +120,10 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu
return stage2::tape_builder::parse_document<true>(*this, _doc); return stage2::tape_builder::parse_document<true>(*this, _doc);
} }
simdjson_warn_unused std::pair<const uint8_t *, bool> dom_parser_implementation::parse_string_if_needed(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept {
return ppc64::stringparsing::parse_string_if_needed(src, dst, allow_replacement);
}
SIMDJSON_NO_SANITIZE_MEMORY SIMDJSON_NO_SANITIZE_MEMORY
simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool replacement_char) const noexcept { simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool replacement_char) const noexcept {
return ppc64::stringparsing::parse_string(src, dst, replacement_char); return ppc64::stringparsing::parse_string(src, dst, replacement_char);
+5 -1
View File
@@ -152,7 +152,11 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu
return stage2::tape_builder::parse_document<true>(*this, _doc); return stage2::tape_builder::parse_document<true>(*this, _doc);
} }
SIMDJSON_NO_SANITIZE_MEMORY simdjson_warn_unused std::pair<const uint8_t *, bool> dom_parser_implementation::parse_string_if_needed(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept {
return westmere::stringparsing::parse_string_if_needed(src, dst, allow_replacement);
}
SIMDJSON_NO_SANITIZE_MEMORY
simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool replacement_char) const noexcept { simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool replacement_char) const noexcept {
return westmere::stringparsing::parse_string(src, dst, replacement_char); return westmere::stringparsing::parse_string(src, dst, replacement_char);
} }