From ec0b48b7722112e4fa005e557aa0531756c1718a Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Thu, 2 Mar 2023 14:27:49 -0500 Subject: [PATCH] Version bump. --- CMakeLists.txt | 2 +- Doxyfile | 2 +- include/simdjson/simdjson_version.h | 4 +- singleheader/simdjson.cpp | 860 +++++++++++++++++++++++----- singleheader/simdjson.h | 257 +++++++-- 5 files changed, 930 insertions(+), 195 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index be1c2c90a..6a093f5c1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.14) project( simdjson # The version number is modified by tools/release.py - VERSION 3.1.2 + VERSION 3.1.3 DESCRIPTION "Parsing gigabytes of JSON per second" HOMEPAGE_URL "https://simdjson.org/" LANGUAGES CXX C diff --git a/Doxyfile b/Doxyfile index d545d278f..fbc5e5c92 100644 --- a/Doxyfile +++ b/Doxyfile @@ -38,7 +38,7 @@ PROJECT_NAME = simdjson # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = "3.1.2" +PROJECT_NUMBER = "3.1.3" # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/include/simdjson/simdjson_version.h b/include/simdjson/simdjson_version.h index 62aeddc3f..7cfcf893f 100644 --- a/include/simdjson/simdjson_version.h +++ b/include/simdjson/simdjson_version.h @@ -4,7 +4,7 @@ #define SIMDJSON_SIMDJSON_VERSION_H /** The version of simdjson being used (major.minor.revision) */ -#define SIMDJSON_VERSION "3.1.2" +#define SIMDJSON_VERSION "3.1.3" namespace simdjson { enum { @@ -19,7 +19,7 @@ enum { /** * The revision (major.minor.REVISION) of simdjson being used. */ - SIMDJSON_VERSION_REVISION = 2 + SIMDJSON_VERSION_REVISION = 3 }; } // namespace simdjson diff --git a/singleheader/simdjson.cpp b/singleheader/simdjson.cpp index e44e09f3a..523eccadc 100644 --- a/singleheader/simdjson.cpp +++ b/singleheader/simdjson.cpp @@ -1,4 +1,4 @@ -/* auto-generated on 2023-02-27 23:41:52 -0500. Do not edit! */ +/* auto-generated on 2023-03-02 09:39:35 -0500. Do not edit! */ /* begin file src/simdjson.cpp */ #include "simdjson.h" @@ -4186,7 +4186,9 @@ static const uint8_t escape_map[256] = { // We work in little-endian then swap at write time simdjson_warn_unused simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, - uint8_t **dst_ptr) { + uint8_t **dst_ptr, bool allow_replacement) { + // Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD) + constexpr uint32_t substitution_code_point = 0xfffd; // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the // conversion isn't valid; we defer the check for this to inside the // multilingual plane check @@ -4201,33 +4203,79 @@ simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, const uint8_t *src_data = *src_ptr; /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ if (((src_data[0] << 8) | src_data[1]) != ((static_cast ('\\') << 8) | static_cast ('u'))) { - return false; - } - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); + if(!allow_replacement) { return false; } + code_point = substitution_code_point; + } else { + uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - // We have already checked that the high surrogate is valid and - // (code_point - 0xd800) < 1024. - // - // Check that code_point_2 is in the range 0xdc00..0xdfff - // and that code_point_2 was parsed from valid hex. - uint32_t low_bit = code_point_2 - 0xdc00; - if (low_bit >> 10) { - return false; - } + // We have already checked that the high surrogate is valid and + // (code_point - 0xd800) < 1024. + // + // Check that code_point_2 is in the range 0xdc00..0xdfff + // and that code_point_2 was parsed from valid hex. + uint32_t low_bit = code_point_2 - 0xdc00; + if (low_bit >> 10) { + if(!allow_replacement) { return false; } + code_point = substitution_code_point; + } else { + code_point = (((code_point - 0xd800) << 10) | low_bit) + 0x10000; + *src_ptr += 6; + } - code_point = - (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; + } } else if (code_point >= 0xdc00 && code_point <= 0xdfff) { // If we encounter a low surrogate (not preceded by a high surrogate) // then we have an error. - return false; + if(!allow_replacement) { return false; } + code_point = substitution_code_point; } size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); *dst_ptr += offset; return offset > 0; } + +// handle a unicode codepoint using the wobbly convention +// https://simonsapin.github.io/wtf-8/ +// write appropriate values into dest +// src will advance 6 bytes or 12 bytes +// dest will advance a variable amount (return via pointer) +// return true if the unicode codepoint was valid +// We work in little-endian then swap at write time +simdjson_warn_unused +simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t **src_ptr, + uint8_t **dst_ptr) { + // It is not ideal that this function is nearly identical to handle_unicode_codepoint. + // + // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the + // conversion isn't valid; we defer the check for this to inside the + // multilingual plane check + uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); + *src_ptr += 6; + // If we found a high surrogate, we must + // check for low surrogate for characters + // outside the Basic + // Multilingual Plane. + if (code_point >= 0xd800 && code_point < 0xdc00) { + const uint8_t *src_data = *src_ptr; + /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ + if (((src_data[0] << 8) | src_data[1]) == ((static_cast ('\\') << 8) | static_cast ('u'))) { + uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); + uint32_t low_bit = code_point_2 - 0xdc00; + if ((low_bit >> 10) == 0) { + code_point = + (((code_point - 0xd800) << 10) | low_bit) + 0x10000; + *src_ptr += 6; + } + } + } + + size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); + *dst_ptr += offset; + return offset > 0; +} + + /** * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There * must be an unescaped quote terminating the string. It returns the final output @@ -4236,7 +4284,7 @@ simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + * SIMDJSON_PADDING bytes. */ -simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst) { +simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) { while (1) { // Copy the next n bytes, and find the backslash and quote in them. auto bs_quote = backslash_and_quote::copy_and_find(src, dst); @@ -4255,7 +4303,54 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, u within the unicode codepoint handling code. */ src += bs_dist; dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst)) { + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } else { + /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and + * write bs_dist+1 characters to output + * note this may reach beyond the part of the buffer we've actually + * seen. I think this is ok */ + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; /* bogus escape value is an error */ + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } else { + /* they are the same. Since they can't co-occur, it means we + * encountered neither. */ + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } + /* can't be reached */ + return nullptr; +} + +simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { + // It is not ideal that this function is nearly identical to parse_string. + while (1) { + // Copy the next n bytes, and find the backslash and quote in them. + auto bs_quote = backslash_and_quote::copy_and_find(src, dst); + // If the next thing is the end quote, copy and return + if (bs_quote.has_quote_first()) { + // we encountered quotes first. Move dst to point to quotes and exit + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + /* find out where the backspace is */ + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + /* we encountered backslash first. Handle backslash */ + if (escape_char == 'u') { + /* move src/dst up to the start; they will be further adjusted + within the unicode codepoint handling code. */ + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint_wobbly(&src, &dst)) { return nullptr; } } else { @@ -4948,7 +5043,7 @@ simdjson_inline tape_builder::tape_builder(dom::document &doc) noexcept : tape{d simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { iter.log_value(key ? "key" : "string"); uint8_t *dst = on_start_string(iter); - dst = stringparsing::parse_string(value+1, dst); + dst = stringparsing::parse_string(value+1, dst, false); // We do not allow replacement when the escape characters are invalid. if (dst == nullptr) { iter.log_error("Invalid escape in string"); return STRING_ERROR; @@ -5126,8 +5221,12 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu return stage2::tape_builder::parse_document(*this, _doc); } -simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst) const noexcept { - return arm64::stringparsing::parse_string(src, dst); +simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept { + return arm64::stringparsing::parse_string(src, dst, allow_replacement); +} + +simdjson_warn_unused uint8_t *dom_parser_implementation::parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept { + return arm64::stringparsing::parse_wobbly_string(src, dst); } simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { @@ -5662,7 +5761,9 @@ static const uint8_t escape_map[256] = { // We work in little-endian then swap at write time simdjson_warn_unused simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, - uint8_t **dst_ptr) { + uint8_t **dst_ptr, bool allow_replacement) { + // Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD) + constexpr uint32_t substitution_code_point = 0xfffd; // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the // conversion isn't valid; we defer the check for this to inside the // multilingual plane check @@ -5677,33 +5778,79 @@ simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, const uint8_t *src_data = *src_ptr; /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ if (((src_data[0] << 8) | src_data[1]) != ((static_cast ('\\') << 8) | static_cast ('u'))) { - return false; - } - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); + if(!allow_replacement) { return false; } + code_point = substitution_code_point; + } else { + uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - // We have already checked that the high surrogate is valid and - // (code_point - 0xd800) < 1024. - // - // Check that code_point_2 is in the range 0xdc00..0xdfff - // and that code_point_2 was parsed from valid hex. - uint32_t low_bit = code_point_2 - 0xdc00; - if (low_bit >> 10) { - return false; - } + // We have already checked that the high surrogate is valid and + // (code_point - 0xd800) < 1024. + // + // Check that code_point_2 is in the range 0xdc00..0xdfff + // and that code_point_2 was parsed from valid hex. + uint32_t low_bit = code_point_2 - 0xdc00; + if (low_bit >> 10) { + if(!allow_replacement) { return false; } + code_point = substitution_code_point; + } else { + code_point = (((code_point - 0xd800) << 10) | low_bit) + 0x10000; + *src_ptr += 6; + } - code_point = - (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; + } } else if (code_point >= 0xdc00 && code_point <= 0xdfff) { // If we encounter a low surrogate (not preceded by a high surrogate) // then we have an error. - return false; + if(!allow_replacement) { return false; } + code_point = substitution_code_point; } size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); *dst_ptr += offset; return offset > 0; } + +// handle a unicode codepoint using the wobbly convention +// https://simonsapin.github.io/wtf-8/ +// write appropriate values into dest +// src will advance 6 bytes or 12 bytes +// dest will advance a variable amount (return via pointer) +// return true if the unicode codepoint was valid +// We work in little-endian then swap at write time +simdjson_warn_unused +simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t **src_ptr, + uint8_t **dst_ptr) { + // It is not ideal that this function is nearly identical to handle_unicode_codepoint. + // + // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the + // conversion isn't valid; we defer the check for this to inside the + // multilingual plane check + uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); + *src_ptr += 6; + // If we found a high surrogate, we must + // check for low surrogate for characters + // outside the Basic + // Multilingual Plane. + if (code_point >= 0xd800 && code_point < 0xdc00) { + const uint8_t *src_data = *src_ptr; + /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ + if (((src_data[0] << 8) | src_data[1]) == ((static_cast ('\\') << 8) | static_cast ('u'))) { + uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); + uint32_t low_bit = code_point_2 - 0xdc00; + if ((low_bit >> 10) == 0) { + code_point = + (((code_point - 0xd800) << 10) | low_bit) + 0x10000; + *src_ptr += 6; + } + } + } + + size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); + *dst_ptr += offset; + return offset > 0; +} + + /** * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There * must be an unescaped quote terminating the string. It returns the final output @@ -5712,7 +5859,7 @@ simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + * SIMDJSON_PADDING bytes. */ -simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst) { +simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) { while (1) { // Copy the next n bytes, and find the backslash and quote in them. auto bs_quote = backslash_and_quote::copy_and_find(src, dst); @@ -5731,7 +5878,54 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, u within the unicode codepoint handling code. */ src += bs_dist; dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst)) { + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } else { + /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and + * write bs_dist+1 characters to output + * note this may reach beyond the part of the buffer we've actually + * seen. I think this is ok */ + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; /* bogus escape value is an error */ + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } else { + /* they are the same. Since they can't co-occur, it means we + * encountered neither. */ + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } + /* can't be reached */ + return nullptr; +} + +simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { + // It is not ideal that this function is nearly identical to parse_string. + while (1) { + // Copy the next n bytes, and find the backslash and quote in them. + auto bs_quote = backslash_and_quote::copy_and_find(src, dst); + // If the next thing is the end quote, copy and return + if (bs_quote.has_quote_first()) { + // we encountered quotes first. Move dst to point to quotes and exit + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + /* find out where the backspace is */ + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + /* we encountered backslash first. Handle backslash */ + if (escape_char == 'u') { + /* move src/dst up to the start; they will be further adjusted + within the unicode codepoint handling code. */ + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint_wobbly(&src, &dst)) { return nullptr; } } else { @@ -6424,7 +6618,7 @@ simdjson_inline tape_builder::tape_builder(dom::document &doc) noexcept : tape{d simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { iter.log_value(key ? "key" : "string"); uint8_t *dst = on_start_string(iter); - dst = stringparsing::parse_string(value+1, dst); + dst = stringparsing::parse_string(value+1, dst, false); // We do not allow replacement when the escape characters are invalid. if (dst == nullptr) { iter.log_error("Invalid escape in string"); return STRING_ERROR; @@ -6573,8 +6767,12 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu return stage2::tape_builder::parse_document(*this, _doc); } -simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst) const noexcept { - return fallback::stringparsing::parse_string(src, dst); +simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool replacement_char) const noexcept { + return fallback::stringparsing::parse_string(src, dst, replacement_char); +} + +simdjson_warn_unused uint8_t *dom_parser_implementation::parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept { + return fallback::stringparsing::parse_wobbly_string(src, dst); } simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { @@ -8027,7 +8225,9 @@ static const uint8_t escape_map[256] = { // We work in little-endian then swap at write time simdjson_warn_unused simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, - uint8_t **dst_ptr) { + uint8_t **dst_ptr, bool allow_replacement) { + // Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD) + constexpr uint32_t substitution_code_point = 0xfffd; // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the // conversion isn't valid; we defer the check for this to inside the // multilingual plane check @@ -8042,33 +8242,79 @@ simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, const uint8_t *src_data = *src_ptr; /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ if (((src_data[0] << 8) | src_data[1]) != ((static_cast ('\\') << 8) | static_cast ('u'))) { - return false; - } - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); + if(!allow_replacement) { return false; } + code_point = substitution_code_point; + } else { + uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - // We have already checked that the high surrogate is valid and - // (code_point - 0xd800) < 1024. - // - // Check that code_point_2 is in the range 0xdc00..0xdfff - // and that code_point_2 was parsed from valid hex. - uint32_t low_bit = code_point_2 - 0xdc00; - if (low_bit >> 10) { - return false; - } + // We have already checked that the high surrogate is valid and + // (code_point - 0xd800) < 1024. + // + // Check that code_point_2 is in the range 0xdc00..0xdfff + // and that code_point_2 was parsed from valid hex. + uint32_t low_bit = code_point_2 - 0xdc00; + if (low_bit >> 10) { + if(!allow_replacement) { return false; } + code_point = substitution_code_point; + } else { + code_point = (((code_point - 0xd800) << 10) | low_bit) + 0x10000; + *src_ptr += 6; + } - code_point = - (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; + } } else if (code_point >= 0xdc00 && code_point <= 0xdfff) { // If we encounter a low surrogate (not preceded by a high surrogate) // then we have an error. - return false; + if(!allow_replacement) { return false; } + code_point = substitution_code_point; } size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); *dst_ptr += offset; return offset > 0; } + +// handle a unicode codepoint using the wobbly convention +// https://simonsapin.github.io/wtf-8/ +// write appropriate values into dest +// src will advance 6 bytes or 12 bytes +// dest will advance a variable amount (return via pointer) +// return true if the unicode codepoint was valid +// We work in little-endian then swap at write time +simdjson_warn_unused +simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t **src_ptr, + uint8_t **dst_ptr) { + // It is not ideal that this function is nearly identical to handle_unicode_codepoint. + // + // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the + // conversion isn't valid; we defer the check for this to inside the + // multilingual plane check + uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); + *src_ptr += 6; + // If we found a high surrogate, we must + // check for low surrogate for characters + // outside the Basic + // Multilingual Plane. + if (code_point >= 0xd800 && code_point < 0xdc00) { + const uint8_t *src_data = *src_ptr; + /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ + if (((src_data[0] << 8) | src_data[1]) == ((static_cast ('\\') << 8) | static_cast ('u'))) { + uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); + uint32_t low_bit = code_point_2 - 0xdc00; + if ((low_bit >> 10) == 0) { + code_point = + (((code_point - 0xd800) << 10) | low_bit) + 0x10000; + *src_ptr += 6; + } + } + } + + size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); + *dst_ptr += offset; + return offset > 0; +} + + /** * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There * must be an unescaped quote terminating the string. It returns the final output @@ -8077,7 +8323,7 @@ simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + * SIMDJSON_PADDING bytes. */ -simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst) { +simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) { while (1) { // Copy the next n bytes, and find the backslash and quote in them. auto bs_quote = backslash_and_quote::copy_and_find(src, dst); @@ -8096,7 +8342,54 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, u within the unicode codepoint handling code. */ src += bs_dist; dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst)) { + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } else { + /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and + * write bs_dist+1 characters to output + * note this may reach beyond the part of the buffer we've actually + * seen. I think this is ok */ + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; /* bogus escape value is an error */ + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } else { + /* they are the same. Since they can't co-occur, it means we + * encountered neither. */ + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } + /* can't be reached */ + return nullptr; +} + +simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { + // It is not ideal that this function is nearly identical to parse_string. + while (1) { + // Copy the next n bytes, and find the backslash and quote in them. + auto bs_quote = backslash_and_quote::copy_and_find(src, dst); + // If the next thing is the end quote, copy and return + if (bs_quote.has_quote_first()) { + // we encountered quotes first. Move dst to point to quotes and exit + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + /* find out where the backspace is */ + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + /* we encountered backslash first. Handle backslash */ + if (escape_char == 'u') { + /* move src/dst up to the start; they will be further adjusted + within the unicode codepoint handling code. */ + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint_wobbly(&src, &dst)) { return nullptr; } } else { @@ -8789,7 +9082,7 @@ simdjson_inline tape_builder::tape_builder(dom::document &doc) noexcept : tape{d simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { iter.log_value(key ? "key" : "string"); uint8_t *dst = on_start_string(iter); - dst = stringparsing::parse_string(value+1, dst); + dst = stringparsing::parse_string(value+1, dst, false); // We do not allow replacement when the escape characters are invalid. if (dst == nullptr) { iter.log_error("Invalid escape in string"); return STRING_ERROR; @@ -8965,8 +9258,12 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu return stage2::tape_builder::parse_document(*this, _doc); } -simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst) const noexcept { - return icelake::stringparsing::parse_string(src, dst); +simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool replacement_char) const noexcept { + return icelake::stringparsing::parse_string(src, dst, replacement_char); +} + +simdjson_warn_unused uint8_t *dom_parser_implementation::parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept { + return icelake::stringparsing::parse_wobbly_string(src, dst); } simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { @@ -10374,7 +10671,9 @@ static const uint8_t escape_map[256] = { // We work in little-endian then swap at write time simdjson_warn_unused simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, - uint8_t **dst_ptr) { + uint8_t **dst_ptr, bool allow_replacement) { + // Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD) + constexpr uint32_t substitution_code_point = 0xfffd; // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the // conversion isn't valid; we defer the check for this to inside the // multilingual plane check @@ -10389,33 +10688,79 @@ simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, const uint8_t *src_data = *src_ptr; /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ if (((src_data[0] << 8) | src_data[1]) != ((static_cast ('\\') << 8) | static_cast ('u'))) { - return false; - } - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); + if(!allow_replacement) { return false; } + code_point = substitution_code_point; + } else { + uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - // We have already checked that the high surrogate is valid and - // (code_point - 0xd800) < 1024. - // - // Check that code_point_2 is in the range 0xdc00..0xdfff - // and that code_point_2 was parsed from valid hex. - uint32_t low_bit = code_point_2 - 0xdc00; - if (low_bit >> 10) { - return false; - } + // We have already checked that the high surrogate is valid and + // (code_point - 0xd800) < 1024. + // + // Check that code_point_2 is in the range 0xdc00..0xdfff + // and that code_point_2 was parsed from valid hex. + uint32_t low_bit = code_point_2 - 0xdc00; + if (low_bit >> 10) { + if(!allow_replacement) { return false; } + code_point = substitution_code_point; + } else { + code_point = (((code_point - 0xd800) << 10) | low_bit) + 0x10000; + *src_ptr += 6; + } - code_point = - (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; + } } else if (code_point >= 0xdc00 && code_point <= 0xdfff) { // If we encounter a low surrogate (not preceded by a high surrogate) // then we have an error. - return false; + if(!allow_replacement) { return false; } + code_point = substitution_code_point; } size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); *dst_ptr += offset; return offset > 0; } + +// handle a unicode codepoint using the wobbly convention +// https://simonsapin.github.io/wtf-8/ +// write appropriate values into dest +// src will advance 6 bytes or 12 bytes +// dest will advance a variable amount (return via pointer) +// return true if the unicode codepoint was valid +// We work in little-endian then swap at write time +simdjson_warn_unused +simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t **src_ptr, + uint8_t **dst_ptr) { + // It is not ideal that this function is nearly identical to handle_unicode_codepoint. + // + // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the + // conversion isn't valid; we defer the check for this to inside the + // multilingual plane check + uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); + *src_ptr += 6; + // If we found a high surrogate, we must + // check for low surrogate for characters + // outside the Basic + // Multilingual Plane. + if (code_point >= 0xd800 && code_point < 0xdc00) { + const uint8_t *src_data = *src_ptr; + /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ + if (((src_data[0] << 8) | src_data[1]) == ((static_cast ('\\') << 8) | static_cast ('u'))) { + uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); + uint32_t low_bit = code_point_2 - 0xdc00; + if ((low_bit >> 10) == 0) { + code_point = + (((code_point - 0xd800) << 10) | low_bit) + 0x10000; + *src_ptr += 6; + } + } + } + + size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); + *dst_ptr += offset; + return offset > 0; +} + + /** * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There * must be an unescaped quote terminating the string. It returns the final output @@ -10424,7 +10769,7 @@ simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + * SIMDJSON_PADDING bytes. */ -simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst) { +simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) { while (1) { // Copy the next n bytes, and find the backslash and quote in them. auto bs_quote = backslash_and_quote::copy_and_find(src, dst); @@ -10443,7 +10788,54 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, u within the unicode codepoint handling code. */ src += bs_dist; dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst)) { + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } else { + /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and + * write bs_dist+1 characters to output + * note this may reach beyond the part of the buffer we've actually + * seen. I think this is ok */ + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; /* bogus escape value is an error */ + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } else { + /* they are the same. Since they can't co-occur, it means we + * encountered neither. */ + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } + /* can't be reached */ + return nullptr; +} + +simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { + // It is not ideal that this function is nearly identical to parse_string. + while (1) { + // Copy the next n bytes, and find the backslash and quote in them. + auto bs_quote = backslash_and_quote::copy_and_find(src, dst); + // If the next thing is the end quote, copy and return + if (bs_quote.has_quote_first()) { + // we encountered quotes first. Move dst to point to quotes and exit + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + /* find out where the backspace is */ + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + /* we encountered backslash first. Handle backslash */ + if (escape_char == 'u') { + /* move src/dst up to the start; they will be further adjusted + within the unicode codepoint handling code. */ + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint_wobbly(&src, &dst)) { return nullptr; } } else { @@ -11136,7 +11528,7 @@ simdjson_inline tape_builder::tape_builder(dom::document &doc) noexcept : tape{d simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { iter.log_value(key ? "key" : "string"); uint8_t *dst = on_start_string(iter); - dst = stringparsing::parse_string(value+1, dst); + dst = stringparsing::parse_string(value+1, dst, false); // We do not allow replacement when the escape characters are invalid. if (dst == nullptr) { iter.log_error("Invalid escape in string"); return STRING_ERROR; @@ -11312,8 +11704,12 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu return stage2::tape_builder::parse_document(*this, _doc); } -simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst) const noexcept { - return haswell::stringparsing::parse_string(src, dst); +simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool replacement_char) const noexcept { + return haswell::stringparsing::parse_string(src, dst, replacement_char); +} + +simdjson_warn_unused uint8_t *dom_parser_implementation::parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept { + return haswell::stringparsing::parse_wobbly_string(src, dst); } simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { @@ -12684,7 +13080,9 @@ static const uint8_t escape_map[256] = { // We work in little-endian then swap at write time simdjson_warn_unused simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, - uint8_t **dst_ptr) { + uint8_t **dst_ptr, bool allow_replacement) { + // Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD) + constexpr uint32_t substitution_code_point = 0xfffd; // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the // conversion isn't valid; we defer the check for this to inside the // multilingual plane check @@ -12699,33 +13097,79 @@ simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, const uint8_t *src_data = *src_ptr; /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ if (((src_data[0] << 8) | src_data[1]) != ((static_cast ('\\') << 8) | static_cast ('u'))) { - return false; - } - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); + if(!allow_replacement) { return false; } + code_point = substitution_code_point; + } else { + uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - // We have already checked that the high surrogate is valid and - // (code_point - 0xd800) < 1024. - // - // Check that code_point_2 is in the range 0xdc00..0xdfff - // and that code_point_2 was parsed from valid hex. - uint32_t low_bit = code_point_2 - 0xdc00; - if (low_bit >> 10) { - return false; - } + // We have already checked that the high surrogate is valid and + // (code_point - 0xd800) < 1024. + // + // Check that code_point_2 is in the range 0xdc00..0xdfff + // and that code_point_2 was parsed from valid hex. + uint32_t low_bit = code_point_2 - 0xdc00; + if (low_bit >> 10) { + if(!allow_replacement) { return false; } + code_point = substitution_code_point; + } else { + code_point = (((code_point - 0xd800) << 10) | low_bit) + 0x10000; + *src_ptr += 6; + } - code_point = - (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; + } } else if (code_point >= 0xdc00 && code_point <= 0xdfff) { // If we encounter a low surrogate (not preceded by a high surrogate) // then we have an error. - return false; + if(!allow_replacement) { return false; } + code_point = substitution_code_point; } size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); *dst_ptr += offset; return offset > 0; } + +// handle a unicode codepoint using the wobbly convention +// https://simonsapin.github.io/wtf-8/ +// write appropriate values into dest +// src will advance 6 bytes or 12 bytes +// dest will advance a variable amount (return via pointer) +// return true if the unicode codepoint was valid +// We work in little-endian then swap at write time +simdjson_warn_unused +simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t **src_ptr, + uint8_t **dst_ptr) { + // It is not ideal that this function is nearly identical to handle_unicode_codepoint. + // + // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the + // conversion isn't valid; we defer the check for this to inside the + // multilingual plane check + uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); + *src_ptr += 6; + // If we found a high surrogate, we must + // check for low surrogate for characters + // outside the Basic + // Multilingual Plane. + if (code_point >= 0xd800 && code_point < 0xdc00) { + const uint8_t *src_data = *src_ptr; + /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ + if (((src_data[0] << 8) | src_data[1]) == ((static_cast ('\\') << 8) | static_cast ('u'))) { + uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); + uint32_t low_bit = code_point_2 - 0xdc00; + if ((low_bit >> 10) == 0) { + code_point = + (((code_point - 0xd800) << 10) | low_bit) + 0x10000; + *src_ptr += 6; + } + } + } + + size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); + *dst_ptr += offset; + return offset > 0; +} + + /** * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There * must be an unescaped quote terminating the string. It returns the final output @@ -12734,7 +13178,7 @@ simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + * SIMDJSON_PADDING bytes. */ -simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst) { +simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) { while (1) { // Copy the next n bytes, and find the backslash and quote in them. auto bs_quote = backslash_and_quote::copy_and_find(src, dst); @@ -12753,7 +13197,54 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, u within the unicode codepoint handling code. */ src += bs_dist; dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst)) { + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } else { + /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and + * write bs_dist+1 characters to output + * note this may reach beyond the part of the buffer we've actually + * seen. I think this is ok */ + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; /* bogus escape value is an error */ + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } else { + /* they are the same. Since they can't co-occur, it means we + * encountered neither. */ + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } + /* can't be reached */ + return nullptr; +} + +simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { + // It is not ideal that this function is nearly identical to parse_string. + while (1) { + // Copy the next n bytes, and find the backslash and quote in them. + auto bs_quote = backslash_and_quote::copy_and_find(src, dst); + // If the next thing is the end quote, copy and return + if (bs_quote.has_quote_first()) { + // we encountered quotes first. Move dst to point to quotes and exit + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + /* find out where the backspace is */ + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + /* we encountered backslash first. Handle backslash */ + if (escape_char == 'u') { + /* move src/dst up to the start; they will be further adjusted + within the unicode codepoint handling code. */ + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint_wobbly(&src, &dst)) { return nullptr; } } else { @@ -13446,7 +13937,7 @@ simdjson_inline tape_builder::tape_builder(dom::document &doc) noexcept : tape{d simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { iter.log_value(key ? "key" : "string"); uint8_t *dst = on_start_string(iter); - dst = stringparsing::parse_string(value+1, dst); + dst = stringparsing::parse_string(value+1, dst, false); // We do not allow replacement when the escape characters are invalid. if (dst == nullptr) { iter.log_error("Invalid escape in string"); return STRING_ERROR; @@ -13624,8 +14115,12 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu return stage2::tape_builder::parse_document(*this, _doc); } -simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst) const noexcept { - return ppc64::stringparsing::parse_string(src, dst); +simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool replacement_char) const noexcept { + return ppc64::stringparsing::parse_string(src, dst, replacement_char); +} + +simdjson_warn_unused uint8_t *dom_parser_implementation::parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept { + return ppc64::stringparsing::parse_wobbly_string(src, dst); } simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { @@ -15029,7 +15524,9 @@ static const uint8_t escape_map[256] = { // We work in little-endian then swap at write time simdjson_warn_unused simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, - uint8_t **dst_ptr) { + uint8_t **dst_ptr, bool allow_replacement) { + // Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD) + constexpr uint32_t substitution_code_point = 0xfffd; // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the // conversion isn't valid; we defer the check for this to inside the // multilingual plane check @@ -15044,33 +15541,79 @@ simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, const uint8_t *src_data = *src_ptr; /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ if (((src_data[0] << 8) | src_data[1]) != ((static_cast ('\\') << 8) | static_cast ('u'))) { - return false; - } - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); + if(!allow_replacement) { return false; } + code_point = substitution_code_point; + } else { + uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - // We have already checked that the high surrogate is valid and - // (code_point - 0xd800) < 1024. - // - // Check that code_point_2 is in the range 0xdc00..0xdfff - // and that code_point_2 was parsed from valid hex. - uint32_t low_bit = code_point_2 - 0xdc00; - if (low_bit >> 10) { - return false; - } + // We have already checked that the high surrogate is valid and + // (code_point - 0xd800) < 1024. + // + // Check that code_point_2 is in the range 0xdc00..0xdfff + // and that code_point_2 was parsed from valid hex. + uint32_t low_bit = code_point_2 - 0xdc00; + if (low_bit >> 10) { + if(!allow_replacement) { return false; } + code_point = substitution_code_point; + } else { + code_point = (((code_point - 0xd800) << 10) | low_bit) + 0x10000; + *src_ptr += 6; + } - code_point = - (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; + } } else if (code_point >= 0xdc00 && code_point <= 0xdfff) { // If we encounter a low surrogate (not preceded by a high surrogate) // then we have an error. - return false; + if(!allow_replacement) { return false; } + code_point = substitution_code_point; } size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); *dst_ptr += offset; return offset > 0; } + +// handle a unicode codepoint using the wobbly convention +// https://simonsapin.github.io/wtf-8/ +// write appropriate values into dest +// src will advance 6 bytes or 12 bytes +// dest will advance a variable amount (return via pointer) +// return true if the unicode codepoint was valid +// We work in little-endian then swap at write time +simdjson_warn_unused +simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t **src_ptr, + uint8_t **dst_ptr) { + // It is not ideal that this function is nearly identical to handle_unicode_codepoint. + // + // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the + // conversion isn't valid; we defer the check for this to inside the + // multilingual plane check + uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); + *src_ptr += 6; + // If we found a high surrogate, we must + // check for low surrogate for characters + // outside the Basic + // Multilingual Plane. + if (code_point >= 0xd800 && code_point < 0xdc00) { + const uint8_t *src_data = *src_ptr; + /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ + if (((src_data[0] << 8) | src_data[1]) == ((static_cast ('\\') << 8) | static_cast ('u'))) { + uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); + uint32_t low_bit = code_point_2 - 0xdc00; + if ((low_bit >> 10) == 0) { + code_point = + (((code_point - 0xd800) << 10) | low_bit) + 0x10000; + *src_ptr += 6; + } + } + } + + size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); + *dst_ptr += offset; + return offset > 0; +} + + /** * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There * must be an unescaped quote terminating the string. It returns the final output @@ -15079,7 +15622,7 @@ simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + * SIMDJSON_PADDING bytes. */ -simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst) { +simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) { while (1) { // Copy the next n bytes, and find the backslash and quote in them. auto bs_quote = backslash_and_quote::copy_and_find(src, dst); @@ -15098,7 +15641,54 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, u within the unicode codepoint handling code. */ src += bs_dist; dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst)) { + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } else { + /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and + * write bs_dist+1 characters to output + * note this may reach beyond the part of the buffer we've actually + * seen. I think this is ok */ + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; /* bogus escape value is an error */ + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } else { + /* they are the same. Since they can't co-occur, it means we + * encountered neither. */ + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } + /* can't be reached */ + return nullptr; +} + +simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { + // It is not ideal that this function is nearly identical to parse_string. + while (1) { + // Copy the next n bytes, and find the backslash and quote in them. + auto bs_quote = backslash_and_quote::copy_and_find(src, dst); + // If the next thing is the end quote, copy and return + if (bs_quote.has_quote_first()) { + // we encountered quotes first. Move dst to point to quotes and exit + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + /* find out where the backspace is */ + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + /* we encountered backslash first. Handle backslash */ + if (escape_char == 'u') { + /* move src/dst up to the start; they will be further adjusted + within the unicode codepoint handling code. */ + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint_wobbly(&src, &dst)) { return nullptr; } } else { @@ -15791,7 +16381,7 @@ simdjson_inline tape_builder::tape_builder(dom::document &doc) noexcept : tape{d simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { iter.log_value(key ? "key" : "string"); uint8_t *dst = on_start_string(iter); - dst = stringparsing::parse_string(value+1, dst); + dst = stringparsing::parse_string(value+1, dst, false); // We do not allow replacement when the escape characters are invalid. if (dst == nullptr) { iter.log_error("Invalid escape in string"); return STRING_ERROR; @@ -15968,8 +16558,12 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu return stage2::tape_builder::parse_document(*this, _doc); } -simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst) const noexcept { - return westmere::stringparsing::parse_string(src, dst); +simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool replacement_char) const noexcept { + return westmere::stringparsing::parse_string(src, dst, replacement_char); +} + +simdjson_warn_unused uint8_t *dom_parser_implementation::parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept { + return westmere::stringparsing::parse_wobbly_string(src, dst); } simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { diff --git a/singleheader/simdjson.h b/singleheader/simdjson.h index 6f999cc3f..7c957468b 100644 --- a/singleheader/simdjson.h +++ b/singleheader/simdjson.h @@ -1,4 +1,4 @@ -/* auto-generated on 2023-02-27 23:41:52 -0500. Do not edit! */ +/* auto-generated on 2023-03-02 09:39:35 -0500. Do not edit! */ /* begin file include/simdjson.h */ #ifndef SIMDJSON_H #define SIMDJSON_H @@ -43,7 +43,7 @@ #define SIMDJSON_SIMDJSON_VERSION_H /** The version of simdjson being used (major.minor.revision) */ -#define SIMDJSON_VERSION "3.1.2" +#define SIMDJSON_VERSION "3.1.3" namespace simdjson { enum { @@ -58,7 +58,7 @@ enum { /** * The revision (major.minor.REVISION) of simdjson being used. */ - SIMDJSON_VERSION_REVISION = 2 + SIMDJSON_VERSION_REVISION = 3 }; } // namespace simdjson @@ -2977,9 +2977,26 @@ public: * * @param str pointer to the beginning of a valid UTF-8 JSON string, must end with an unescaped quote. * @param dst pointer to a destination buffer, it must point a region in memory of sufficient size. + * @param allow_replacement whether we allow a replacement character when the UTF-8 contains unmatched surrogate pairs. * @return end of the of the written region (exclusive) or nullptr in case of error. */ - simdjson_warn_unused virtual uint8_t *parse_string(const uint8_t *src, uint8_t *dst) const noexcept = 0; + simdjson_warn_unused virtual uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept = 0; + + /** + * Unescape a NON-valid UTF-8 string from src to dst, stopping at a final unescaped quote. There + * must be an unescaped quote terminating the string. It returns the final output + * position as pointer. In case of error (e.g., the string has bad escaped codes), + * then null_nullptrptr is returned. It is assumed that the output buffer is large + * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + + * SIMDJSON_PADDING bytes. + * + * Overridden by each implementation. + * + * @param str pointer to the beginning of a possibly invalid UTF-8 JSON string, must end with an unescaped quote. + * @param dst pointer to a destination buffer, it must point a region in memory of sufficient size. + * @return end of the of the written region (exclusive) or nullptr in case of error. + */ + simdjson_warn_unused virtual uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept = 0; /** * Change the capacity of this parser. @@ -9754,7 +9771,8 @@ public: simdjson_warn_unused error_code stage1(const uint8_t *buf, size_t len, stage1_mode partial) noexcept final; simdjson_warn_unused error_code stage2(dom::document &doc) noexcept final; simdjson_warn_unused error_code stage2_next(dom::document &doc) noexcept final; - simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst) const noexcept final; + simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept final; + simdjson_warn_unused uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept final; inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final; inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final; private: @@ -12101,7 +12119,8 @@ public: simdjson_warn_unused error_code stage1(const uint8_t *buf, size_t len, stage1_mode partial) noexcept final; simdjson_warn_unused error_code stage2(dom::document &doc) noexcept final; simdjson_warn_unused error_code stage2_next(dom::document &doc) noexcept final; - simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst) const noexcept final; + simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept final; + simdjson_warn_unused uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept final; inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final; inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final; private: @@ -13875,7 +13894,8 @@ public: simdjson_warn_unused error_code stage1(const uint8_t *buf, size_t len, stage1_mode partial) noexcept final; simdjson_warn_unused error_code stage2(dom::document &doc) noexcept final; simdjson_warn_unused error_code stage2_next(dom::document &doc) noexcept final; - simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst) const noexcept final; + simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept final; + simdjson_warn_unused uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept final; inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final; inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final; private: @@ -16064,7 +16084,8 @@ public: simdjson_warn_unused error_code stage1(const uint8_t *buf, size_t len, stage1_mode partial) noexcept final; simdjson_warn_unused error_code stage2(dom::document &doc) noexcept final; simdjson_warn_unused error_code stage2_next(dom::document &doc) noexcept final; - simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst) const noexcept final; + simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept final; + simdjson_warn_unused uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept final; inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final; inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final; private: @@ -18187,7 +18208,8 @@ public: simdjson_warn_unused error_code stage1(const uint8_t *buf, size_t len, stage1_mode partial) noexcept final; simdjson_warn_unused error_code stage2(dom::document &doc) noexcept final; simdjson_warn_unused error_code stage2_next(dom::document &doc) noexcept final; - simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst) const noexcept final; + simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept final; + simdjson_warn_unused uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept final; inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final; inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final; private: @@ -20525,7 +20547,8 @@ public: simdjson_warn_unused error_code stage1(const uint8_t *buf, size_t len, stage1_mode partial) noexcept final; simdjson_warn_unused error_code stage2(dom::document &doc) noexcept final; simdjson_warn_unused error_code stage2_next(dom::document &doc) noexcept final; - simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst) const noexcept final; + simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept final; + simdjson_warn_unused uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept final; inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final; inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final; private: @@ -23061,6 +23084,20 @@ private: /** * Unescape this JSON string, replacing \\ with \, \n with newline, etc. + * The result will be a valid UTF-8. + * + * ## IMPORTANT: string_view lifetime + * + * The string_view is only valid until the next parse() call on the parser. + * + * @param iter A json_iterator, which contains a buffer where the string will be written. + * @param allow_replacement Whether we allow replacement of invalid surrogate pairs. + */ + simdjson_inline simdjson_warn_unused simdjson_result unescape(json_iterator &iter, bool allow_replacement) const noexcept; + + /** + * Unescape this JSON string, replacing \\ with \, \n with newline, etc. + * The result may not be a valid UTF-8. https://simonsapin.github.io/wtf-8/ * * ## IMPORTANT: string_view lifetime * @@ -23068,8 +23105,7 @@ private: * * @param iter A json_iterator, which contains a buffer where the string will be written. */ - simdjson_inline simdjson_warn_unused simdjson_result unescape(json_iterator &iter) const noexcept; - + simdjson_inline simdjson_warn_unused simdjson_result unescape_wobbly(json_iterator &iter) const noexcept; const uint8_t * buf{}; friend class object; friend class field; @@ -23104,7 +23140,8 @@ public: simdjson_inline ~simdjson_result() noexcept = default; ///< @private simdjson_inline simdjson_result raw() const noexcept; - simdjson_inline simdjson_warn_unused simdjson_result unescape(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::json_iterator &iter) const noexcept; + simdjson_inline simdjson_warn_unused simdjson_result unescape(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::json_iterator &iter, bool allow_replacement) const noexcept; + simdjson_inline simdjson_warn_unused simdjson_result unescape_wobbly(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::json_iterator &iter) const noexcept; }; } // namespace simdjson @@ -23498,7 +23535,8 @@ public: * Each raw_json_string should be unescaped once, or else the string buffer might * overflow. */ - simdjson_inline simdjson_result unescape(raw_json_string in) noexcept; + simdjson_inline simdjson_result unescape(raw_json_string in, bool allow_replacement) noexcept; + simdjson_inline simdjson_result unescape_wobbly(raw_json_string in) noexcept; simdjson_inline void reenter_child(token_position position, depth_t child_depth) noexcept; #if SIMDJSON_DEVELOPMENT_CHECKS @@ -23847,7 +23885,8 @@ public: * @{ */ - simdjson_warn_unused simdjson_inline simdjson_result get_string() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_string(bool allow_replacement) noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_wobbly_string() noexcept; simdjson_warn_unused simdjson_inline simdjson_result get_raw_json_string() noexcept; simdjson_warn_unused simdjson_inline simdjson_result get_uint64() noexcept; simdjson_warn_unused simdjson_inline simdjson_result get_uint64_in_string() noexcept; @@ -23862,7 +23901,8 @@ public: simdjson_warn_unused simdjson_inline simdjson_result get_number_type() noexcept; simdjson_warn_unused simdjson_inline simdjson_result get_number() noexcept; - simdjson_warn_unused simdjson_inline simdjson_result get_root_string(bool check_trailing) noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_root_string(bool check_trailing, bool allow_replacement) noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_root_wobbly_string(bool check_trailing) noexcept; simdjson_warn_unused simdjson_inline simdjson_result get_root_raw_json_string(bool check_trailing) noexcept; simdjson_warn_unused simdjson_inline simdjson_result get_root_uint64(bool check_trailing) noexcept; simdjson_warn_unused simdjson_inline simdjson_result get_root_uint64_in_string(bool check_trailing) noexcept; @@ -24488,11 +24528,24 @@ public: * * Important: Calling get_string() twice on the same document is an error. * + * @param Whether to allow a replacement character for unmatched surrogate pairs. * @returns An UTF-8 string. The string is stored in the parser and will be invalidated the next * time it parses a document or when it is destroyed. * @returns INCORRECT_TYPE if the JSON value is not a string. */ - simdjson_inline simdjson_result get_string() noexcept; + simdjson_inline simdjson_result get_string(bool allow_replacement = false) noexcept; + /** + * Cast this JSON value to a string. + * + * The string is not guaranteed to be valid UTF-8. See https://simonsapin.github.io/wtf-8/ + * + * Important: Calling get_wobbly_string() twice on the same document is an error. + * + * @returns An UTF-8 string. The string is stored in the parser and will be invalidated the next + * time it parses a document or when it is destroyed. + * @returns INCORRECT_TYPE if the JSON value is not a string. + */ + simdjson_inline simdjson_result get_wobbly_string() noexcept; /** * Cast this JSON value to a raw_json_string. * @@ -24991,7 +25044,8 @@ public: simdjson_inline simdjson_result get_int64_in_string() noexcept; simdjson_inline simdjson_result get_double() noexcept; simdjson_inline simdjson_result get_double_in_string() noexcept; - simdjson_inline simdjson_result get_string() noexcept; + simdjson_inline simdjson_result get_string(bool allow_replacement = false) noexcept; + simdjson_inline simdjson_result get_wobbly_string() noexcept; simdjson_inline simdjson_result get_raw_json_string() noexcept; simdjson_inline simdjson_result get_bool() noexcept; simdjson_inline simdjson_result get_value() noexcept; @@ -25059,7 +25113,8 @@ public: simdjson_inline simdjson_result get_int64_in_string() noexcept; simdjson_inline simdjson_result get_double() noexcept; simdjson_inline simdjson_result get_double_in_string() noexcept; - simdjson_inline simdjson_result get_string() noexcept; + simdjson_inline simdjson_result get_string(bool allow_replacement = false) noexcept; + simdjson_inline simdjson_result get_wobbly_string() noexcept; simdjson_inline simdjson_result get_raw_json_string() noexcept; simdjson_inline simdjson_result get_bool() noexcept; simdjson_inline simdjson_result get_value() noexcept; @@ -25129,7 +25184,8 @@ public: simdjson_inline simdjson_result get_int64_in_string() noexcept; simdjson_inline simdjson_result get_double() noexcept; simdjson_inline simdjson_result get_double_in_string() noexcept; - simdjson_inline simdjson_result get_string() noexcept; + simdjson_inline simdjson_result get_string(bool allow_replacement = false) noexcept; + simdjson_inline simdjson_result get_wobbly_string() noexcept; simdjson_inline simdjson_result get_raw_json_string() noexcept; simdjson_inline simdjson_result get_bool() noexcept; simdjson_inline simdjson_result get_value() noexcept; @@ -25305,8 +25361,23 @@ public: * time it parses a document or when it is destroyed. * @returns INCORRECT_TYPE if the JSON value is not a string. */ - simdjson_inline simdjson_result get_string() noexcept; + simdjson_inline simdjson_result get_string(bool allow_replacement = false) noexcept; + + /** + * Cast this JSON value to a "wobbly" string. + * + * The string is may not be a valid UTF-8 string. + * See https://simonsapin.github.io/wtf-8/ + * + * Important: a value should be consumed once. Calling get_wobbly_string() twice on the same value + * is an error. + * + * @returns An UTF-8 string. The string is stored in the parser and will be invalidated the next + * time it parses a document or when it is destroyed. + * @returns INCORRECT_TYPE if the JSON value is not a string. + */ + simdjson_inline simdjson_result get_wobbly_string() noexcept; /** * Cast this JSON value to a raw_json_string. * @@ -25761,7 +25832,8 @@ public: simdjson_inline simdjson_result get_int64_in_string() noexcept; simdjson_inline simdjson_result get_double() noexcept; simdjson_inline simdjson_result get_double_in_string() noexcept; - simdjson_inline simdjson_result get_string() noexcept; + simdjson_inline simdjson_result get_string(bool allow_replacement = false) noexcept; + simdjson_inline simdjson_result get_wobbly_string() noexcept; simdjson_inline simdjson_result get_raw_json_string() noexcept; simdjson_inline simdjson_result get_bool() noexcept; simdjson_inline simdjson_result is_null() noexcept; @@ -25893,7 +25965,7 @@ public: * This consumes the key: once you have called unescaped_key(), you cannot * call it again nor can you call key(). */ - simdjson_inline simdjson_warn_unused simdjson_result unescaped_key() noexcept; + simdjson_inline simdjson_warn_unused simdjson_result unescaped_key(bool allow_replacement) noexcept; /** * Get the key as a raw_json_string. Can be used for direct comparison with * an unescaped C string: e.g., key() == "test". @@ -25929,7 +26001,7 @@ public: simdjson_inline simdjson_result(error_code error) noexcept; ///< @private simdjson_inline simdjson_result() noexcept = default; - simdjson_inline simdjson_result unescaped_key() noexcept; + simdjson_inline simdjson_result unescaped_key(bool allow_replacement = false) noexcept; simdjson_inline simdjson_result key() noexcept; simdjson_inline simdjson_result value() noexcept; }; @@ -26431,6 +26503,31 @@ public: /** * Unescape this JSON string, replacing \\ with \, \n with newline, etc. to a user-provided buffer. + * The result must be valid UTF-8. + * The provided pointer is advanced to the end of the string by reference, and a string_view instance + * is returned. You can ensure that your buffer is large enough by allocating a block of memory at least + * as large as the input JSON plus SIMDJSON_PADDING and then unescape all strings to this one buffer. + * + * This unescape function is a low-level function. If you want a more user-friendly approach, you should + * avoid raw_json_string instances (e.g., by calling unescaped_key() instead of key() or get_string() + * instead of get_raw_json_string()). + * + * ## IMPORTANT: string_view lifetime + * + * The string_view is only valid as long as the bytes in dst. + * + * @param raw_json_string input + * @param dst A pointer to a buffer at least large enough to write this string as well as + * an additional SIMDJSON_PADDING bytes. + * @param allow_replacement Whether we allow a replacement if the input string contains unmatched surrogate pairs. + * @return A string_view pointing at the unescaped string in dst + * @error STRING_ERROR if escapes are incorrect. + */ + simdjson_inline simdjson_result unescape(raw_json_string in, uint8_t *&dst, bool allow_replacement = false) const noexcept; + + /** + * Unescape this JSON string, replacing \\ with \, \n with newline, etc. to a user-provided buffer. + * The result may not be valid UTF-8. See https://simonsapin.github.io/wtf-8/ * The provided pointer is advanced to the end of the string by reference, and a string_view instance * is returned. You can ensure that your buffer is large enough by allocating a block of memory at least * as large as the input JSON plus SIMDJSON_PADDING and then unescape all strings to this one buffer. @@ -26449,7 +26546,8 @@ public: * @return A string_view pointing at the unescaped string in dst * @error STRING_ERROR if escapes are incorrect. */ - simdjson_inline simdjson_result unescape(raw_json_string in, uint8_t *&dst) const noexcept; + simdjson_inline simdjson_result unescape_wobbly(raw_json_string in, uint8_t *&dst) const noexcept; + private: /** @private [for benchmarking access] The implementation to use */ std::unique_ptr implementation{}; @@ -27432,10 +27530,13 @@ simdjson_unused simdjson_inline bool operator!=(std::string_view c, const raw_js } -simdjson_inline simdjson_warn_unused simdjson_result raw_json_string::unescape(json_iterator &iter) const noexcept { - return iter.unescape(*this); +simdjson_inline simdjson_warn_unused simdjson_result raw_json_string::unescape(json_iterator &iter, bool allow_replacement) const noexcept { + return iter.unescape(*this, allow_replacement); } +simdjson_inline simdjson_warn_unused simdjson_result raw_json_string::unescape_wobbly(json_iterator &iter) const noexcept { + return iter.unescape_wobbly(*this); +} simdjson_unused simdjson_inline std::ostream &operator<<(std::ostream &out, const raw_json_string &str) noexcept { bool in_escape = false; @@ -27466,11 +27567,14 @@ simdjson_inline simdjson_result simdjson_result simdjson_result::unescape(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::json_iterator &iter) const noexcept { +simdjson_inline simdjson_warn_unused simdjson_result simdjson_result::unescape(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::json_iterator &iter, bool allow_replacement) const noexcept { if (error()) { return error(); } - return first.unescape(iter); + return first.unescape(iter, allow_replacement); +} +simdjson_inline simdjson_warn_unused simdjson_result simdjson_result::unescape_wobbly(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::json_iterator &iter) const noexcept { + if (error()) { return error(); } + return first.unescape_wobbly(iter); } - } // namespace simdjson /* end file include/simdjson/generic/ondemand/raw_json_string-inl.h */ /* begin file include/simdjson/generic/ondemand/token_iterator-inl.h */ @@ -27872,8 +27976,12 @@ simdjson_inline token_position json_iterator::position() const noexcept { return token.position(); } -simdjson_inline simdjson_result json_iterator::unescape(raw_json_string in) noexcept { - return parser->unescape(in, _string_buf_loc); +simdjson_inline simdjson_result json_iterator::unescape(raw_json_string in, bool allow_replacement) noexcept { + return parser->unescape(in, _string_buf_loc, allow_replacement); +} + +simdjson_inline simdjson_result json_iterator::unescape_wobbly(raw_json_string in) noexcept { + return parser->unescape_wobbly(in, _string_buf_loc); } simdjson_inline void json_iterator::reenter_child(token_position position, depth_t child_depth) noexcept { @@ -28417,8 +28525,11 @@ simdjson_warn_unused simdjson_inline simdjson_result value_iterator::parse return is_null_string; } -simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_string() noexcept { - return get_raw_json_string().unescape(json_iter()); +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_string(bool allow_replacement) noexcept { + return get_raw_json_string().unescape(json_iter(), allow_replacement); +} +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_wobbly_string() noexcept { + return get_raw_json_string().unescape_wobbly(json_iter()); } simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_raw_json_string() noexcept { auto json = peek_scalar("string"); @@ -28534,8 +28645,11 @@ simdjson_inline simdjson_result value_iterator::get_root_number(bool che advance_root_scalar("number"); return num; } -simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_string(bool check_trailing) noexcept { - return get_root_raw_json_string(check_trailing).unescape(json_iter()); +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_string(bool check_trailing, bool allow_replacement) noexcept { + return get_root_raw_json_string(check_trailing).unescape(json_iter(), allow_replacement); +} +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_wobbly_string(bool check_trailing) noexcept { + return get_root_raw_json_string(check_trailing).unescape_wobbly(json_iter()); } simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_raw_json_string(bool check_trailing) noexcept { auto json = peek_scalar("string"); @@ -29450,8 +29564,11 @@ simdjson_inline simdjson_result document::get_double() noexcept { simdjson_inline simdjson_result document::get_double_in_string() noexcept { return get_root_value_iterator().get_root_double_in_string(true); } -simdjson_inline simdjson_result document::get_string() noexcept { - return get_root_value_iterator().get_root_string(true); +simdjson_inline simdjson_result document::get_string(bool allow_replacement) noexcept { + return get_root_value_iterator().get_root_string(true, allow_replacement); +} +simdjson_inline simdjson_result document::get_wobbly_string() noexcept { + return get_root_value_iterator().get_root_wobbly_string(true); } simdjson_inline simdjson_result document::get_raw_json_string() noexcept { return get_root_value_iterator().get_root_raw_json_string(true); @@ -29466,7 +29583,7 @@ simdjson_inline simdjson_result document::is_null() noexcept { template<> simdjson_inline simdjson_result document::get() & noexcept { return get_array(); } template<> simdjson_inline simdjson_result document::get() & noexcept { return get_object(); } template<> simdjson_inline simdjson_result document::get() & noexcept { return get_raw_json_string(); } -template<> simdjson_inline simdjson_result document::get() & noexcept { return get_string(); } +template<> simdjson_inline simdjson_result document::get() & noexcept { return get_string(false); } template<> simdjson_inline simdjson_result document::get() & noexcept { return get_double(); } template<> simdjson_inline simdjson_result document::get() & noexcept { return get_uint64(); } template<> simdjson_inline simdjson_result document::get() & noexcept { return get_int64(); } @@ -29474,7 +29591,7 @@ template<> simdjson_inline simdjson_result document::get() & noexcept { re template<> simdjson_inline simdjson_result document::get() & noexcept { return get_value(); } template<> simdjson_inline simdjson_result document::get() && noexcept { return get_raw_json_string(); } -template<> simdjson_inline simdjson_result document::get() && noexcept { return get_string(); } +template<> simdjson_inline simdjson_result document::get() && noexcept { return get_string(false); } template<> simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_double(); } template<> simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_uint64(); } template<> simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_int64(); } @@ -29494,7 +29611,7 @@ simdjson_inline document::operator object() & noexcept(false) { return get_objec simdjson_inline document::operator uint64_t() noexcept(false) { return get_uint64(); } simdjson_inline document::operator int64_t() noexcept(false) { return get_int64(); } simdjson_inline document::operator double() noexcept(false) { return get_double(); } -simdjson_inline document::operator std::string_view() noexcept(false) { return get_string(); } +simdjson_inline document::operator std::string_view() noexcept(false) { return get_string(false); } simdjson_inline document::operator raw_json_string() noexcept(false) { return get_raw_json_string(); } simdjson_inline document::operator bool() noexcept(false) { return get_bool(); } simdjson_inline document::operator value() noexcept(false) { return get_value(); } @@ -29715,9 +29832,13 @@ simdjson_inline simdjson_result simdjson_result simdjson_result::get_string() noexcept { +simdjson_inline simdjson_result simdjson_result::get_string(bool allow_replacement) noexcept { if (error()) { return error(); } - return first.get_string(); + return first.get_string(allow_replacement); +} +simdjson_inline simdjson_result simdjson_result::get_wobbly_string() noexcept { + if (error()) { return error(); } + return first.get_wobbly_string(); } simdjson_inline simdjson_result simdjson_result::get_raw_json_string() noexcept { if (error()) { return error(); } @@ -29894,7 +30015,8 @@ simdjson_inline simdjson_result document_reference::get_int64() noexcep simdjson_inline simdjson_result document_reference::get_int64_in_string() noexcept { return doc->get_root_value_iterator().get_root_int64_in_string(false); } simdjson_inline simdjson_result document_reference::get_double() noexcept { return doc->get_root_value_iterator().get_root_double(false); } simdjson_inline simdjson_result document_reference::get_double_in_string() noexcept { return doc->get_root_value_iterator().get_root_double(false); } -simdjson_inline simdjson_result document_reference::get_string() noexcept { return doc->get_root_value_iterator().get_root_string(false); } +simdjson_inline simdjson_result document_reference::get_string(bool allow_replacement) noexcept { return doc->get_root_value_iterator().get_root_string(false, allow_replacement); } +simdjson_inline simdjson_result document_reference::get_wobbly_string() noexcept { return doc->get_root_value_iterator().get_root_wobbly_string(false); } simdjson_inline simdjson_result document_reference::get_raw_json_string() noexcept { return doc->get_root_value_iterator().get_root_raw_json_string(false); } simdjson_inline simdjson_result document_reference::get_bool() noexcept { return doc->get_root_value_iterator().get_root_bool(false); } simdjson_inline simdjson_result document_reference::get_value() noexcept { return doc->get_value(); } @@ -30026,9 +30148,13 @@ simdjson_inline simdjson_result simdjson_result simdjson_result::get_string() noexcept { +simdjson_inline simdjson_result simdjson_result::get_string(bool allow_replacement) noexcept { if (error()) { return error(); } - return first.get_string(); + return first.get_string(allow_replacement); +} +simdjson_inline simdjson_result simdjson_result::get_wobbly_string() noexcept { + if (error()) { return error(); } + return first.get_wobbly_string(); } simdjson_inline simdjson_result simdjson_result::get_raw_json_string() noexcept { if (error()) { return error(); } @@ -30160,8 +30286,11 @@ simdjson_inline simdjson_result value::start_or_resume_object() noexcept simdjson_inline simdjson_result value::get_raw_json_string() noexcept { return iter.get_raw_json_string(); } -simdjson_inline simdjson_result value::get_string() noexcept { - return iter.get_string(); +simdjson_inline simdjson_result value::get_string(bool allow_replacement) noexcept { + return iter.get_string(allow_replacement); +} +simdjson_inline simdjson_result value::get_wobbly_string() noexcept { + return iter.get_wobbly_string(); } simdjson_inline simdjson_result value::get_double() noexcept { return iter.get_double(); @@ -30190,7 +30319,7 @@ simdjson_inline simdjson_result value::is_null() noexcept { template<> simdjson_inline simdjson_result value::get() noexcept { return get_array(); } template<> simdjson_inline simdjson_result value::get() noexcept { return get_object(); } template<> simdjson_inline simdjson_result value::get() noexcept { return get_raw_json_string(); } -template<> simdjson_inline simdjson_result value::get() noexcept { return get_string(); } +template<> simdjson_inline simdjson_result value::get() noexcept { return get_string(false); } template<> simdjson_inline simdjson_result value::get() noexcept { return get_number(); } template<> simdjson_inline simdjson_result value::get() noexcept { return get_double(); } template<> simdjson_inline simdjson_result value::get() noexcept { return get_uint64(); } @@ -30218,7 +30347,7 @@ simdjson_inline value::operator double() noexcept(false) { return get_double(); } simdjson_inline value::operator std::string_view() noexcept(false) { - return get_string(); + return get_string(false); } simdjson_inline value::operator raw_json_string() noexcept(false) { return get_raw_json_string(); @@ -30428,9 +30557,13 @@ simdjson_inline simdjson_result simdjson_result simdjson_result::get_string() noexcept { +simdjson_inline simdjson_result simdjson_result::get_string(bool allow_replacement) noexcept { if (error()) { return error(); } - return first.get_string(); + return first.get_string(allow_replacement); +} +simdjson_inline simdjson_result simdjson_result::get_wobbly_string() noexcept { + if (error()) { return error(); } + return first.get_wobbly_string(); } simdjson_inline simdjson_result simdjson_result::get_raw_json_string() noexcept { if (error()) { return error(); } @@ -30569,9 +30702,9 @@ simdjson_inline simdjson_result field::start(const value_iterator &parent return field(key, parent_iter.child()); } -simdjson_inline simdjson_warn_unused simdjson_result field::unescaped_key() noexcept { +simdjson_inline simdjson_warn_unused simdjson_result field::unescaped_key(bool allow_replacement) noexcept { SIMDJSON_ASSUME(first.buf != nullptr); // We would like to call .alive() but Visual Studio won't let us. - simdjson_result answer = first.unescape(second.iter.json_iter()); + simdjson_result answer = first.unescape(second.iter.json_iter(), allow_replacement); first.consume(); return answer; } @@ -30614,9 +30747,9 @@ simdjson_inline simdjson_result simdjson_result::unescaped_key() noexcept { +simdjson_inline simdjson_result simdjson_result::unescaped_key(bool allow_replacement) noexcept { if (error()) { return error(); } - return first.unescaped_key(); + return first.unescaped_key(allow_replacement); } simdjson_inline simdjson_result simdjson_result::value() noexcept { if (error()) { return error(); } @@ -30975,8 +31108,16 @@ simdjson_inline void parser::set_max_capacity(size_t max_capacity) noexcept { } } -simdjson_inline simdjson_warn_unused simdjson_result parser::unescape(raw_json_string in, uint8_t *&dst) const noexcept { - uint8_t *end = implementation->parse_string(in.buf, dst); +simdjson_inline simdjson_warn_unused simdjson_result parser::unescape(raw_json_string in, uint8_t *&dst, bool allow_replacement) const noexcept { + uint8_t *end = implementation->parse_string(in.buf, dst, allow_replacement); + if (!end) { return STRING_ERROR; } + std::string_view result(reinterpret_cast(dst), end-dst); + dst = end; + return result; +} + +simdjson_inline simdjson_warn_unused simdjson_result parser::unescape_wobbly(raw_json_string in, uint8_t *&dst) const noexcept { + uint8_t *end = implementation->parse_wobbly_string(in.buf, dst); if (!end) { return STRING_ERROR; } std::string_view result(reinterpret_cast(dst), end-dst); dst = end;