From 06afe9ecb374c4e8e106e48207b6607a65a26c8b Mon Sep 17 00:00:00 2001 From: John Keiser Date: Tue, 1 Aug 2023 00:07:58 -0700 Subject: [PATCH] New escape algorithm with significantly less data dependency --- include/simdjson/arm64/begin.h | 2 + include/simdjson/arm64/end.h | 1 + include/simdjson/ppc64/begin.h | 2 + include/simdjson/ppc64/end.h | 1 + singleheader/simdjson.cpp | 2386 +++++++++++++--------- singleheader/simdjson.h | 56 +- src/arm64.cpp | 12 - src/generic/stage1/amalgamated.h | 3 +- src/generic/stage1/buf_block_reader.h | 11 + src/generic/stage1/json_escape_scanner.h | 151 ++ src/generic/stage1/json_string_scanner.h | 79 +- src/haswell.cpp | 10 - src/icelake.cpp | 10 - src/ppc64.cpp | 12 - src/westmere.cpp | 10 - tests/dom/basictests.cpp | 3 +- 16 files changed, 1639 insertions(+), 1110 deletions(-) create mode 100644 src/generic/stage1/json_escape_scanner.h diff --git a/include/simdjson/arm64/begin.h b/include/simdjson/arm64/begin.h index d085622bf..ee48cec05 100644 --- a/include/simdjson/arm64/begin.h +++ b/include/simdjson/arm64/begin.h @@ -6,3 +6,5 @@ #include "simdjson/arm64/numberparsing_defs.h" #include "simdjson/arm64/simd.h" #include "simdjson/arm64/stringparsing_defs.h" + +#define SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT 1 \ No newline at end of file diff --git a/include/simdjson/arm64/end.h b/include/simdjson/arm64/end.h index d797a3f5f..b92378df9 100644 --- a/include/simdjson/arm64/end.h +++ b/include/simdjson/arm64/end.h @@ -2,4 +2,5 @@ #include "simdjson/arm64/base.h" #endif // SIMDJSON_CONDITIONAL_INCLUDE +#undef SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT #undef SIMDJSON_IMPLEMENTATION diff --git a/include/simdjson/ppc64/begin.h b/include/simdjson/ppc64/begin.h index b8fd8c605..e6fc425ef 100644 --- a/include/simdjson/ppc64/begin.h +++ b/include/simdjson/ppc64/begin.h @@ -6,3 +6,5 @@ #include "simdjson/ppc64/numberparsing_defs.h" #include "simdjson/ppc64/simd.h" #include "simdjson/ppc64/stringparsing_defs.h" + +#define SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT 1 \ No newline at end of file diff --git a/include/simdjson/ppc64/end.h b/include/simdjson/ppc64/end.h index d6ad657e0..48ae0d6fe 100644 --- a/include/simdjson/ppc64/end.h +++ b/include/simdjson/ppc64/end.h @@ -2,4 +2,5 @@ #include "simdjson/ppc64/base.h" #endif // SIMDJSON_CONDITIONAL_INCLUDE +#undef SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT #undef SIMDJSON_IMPLEMENTATION diff --git a/singleheader/simdjson.cpp b/singleheader/simdjson.cpp index 0656194d4..064872ef1 100644 --- a/singleheader/simdjson.cpp +++ b/singleheader/simdjson.cpp @@ -1,4 +1,4 @@ -/* auto-generated on 2023-07-20 14:14:44 -0700. Do not edit! */ +/* auto-generated on 2023-07-21 12:09:57 -0700. Do not edit! */ /* including simdjson.cpp: */ /* begin file simdjson.cpp */ #define SIMDJSON_SRC_SIMDJSON_CPP @@ -108,8 +108,6 @@ #else #define SIMDJSON_IS_32BITS 1 -// We do not support 32-bit platforms, but it can be -// handy to identify them. #if defined(_M_IX86) || defined(__i386__) #define SIMDJSON_IS_X86_32BITS 1 #elif defined(__arm__) || defined(_M_ARM) @@ -8125,6 +8123,8 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin #endif // SIMDJSON_ARM64_STRINGPARSING_DEFS_H /* end file simdjson/arm64/stringparsing_defs.h */ + +#define SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT 1 /* end file simdjson/arm64/begin.h */ /* including simdjson/generic/amalgamated.h for arm64: #include "simdjson/generic/amalgamated.h" */ /* begin file simdjson/generic/amalgamated.h for arm64 */ @@ -9969,6 +9969,7 @@ simdjson_inline implementation_simdjson_result_base::implementation_simdjson_ /* amalgamation skipped (editor-only): #include "simdjson/arm64/base.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ +#undef SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT /* undefining SIMDJSON_IMPLEMENTATION from "arm64" */ #undef SIMDJSON_IMPLEMENTATION /* end file simdjson/arm64/end.h */ @@ -10867,6 +10868,8 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin #endif // SIMDJSON_ARM64_STRINGPARSING_DEFS_H /* end file simdjson/arm64/stringparsing_defs.h */ + +#define SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT 1 /* end file simdjson/arm64/begin.h */ /* including generic/amalgamated.h for arm64: #include */ /* begin file generic/amalgamated.h for arm64 */ @@ -10992,6 +10995,279 @@ using utf8_validation::utf8_checker; #endif // SIMDJSON_SRC_GENERIC_STAGE1_BASE_H /* end file generic/stage1/base.h for arm64 */ +/* including generic/stage1/buf_block_reader.h for arm64: #include */ +/* begin file generic/stage1/buf_block_reader.h for arm64 */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include + +namespace simdjson { +namespace arm64 { +namespace { +namespace stage1 { + +// Walks through a buffer in block-sized increments, loading the last part with spaces +template +struct buf_block_reader { +public: + simdjson_inline buf_block_reader(const uint8_t *_buf, size_t _len); + simdjson_inline size_t block_index(); + simdjson_inline bool has_full_block() const; + simdjson_inline const uint8_t *full_block() const; + /** + * Get the last block, padded with spaces. + * + * There will always be a last block, with at least 1 byte, unless len == 0 (in which case this + * function fills the buffer with spaces and returns 0. In particular, if len == STEP_SIZE there + * will be 0 full_blocks and 1 remainder block with STEP_SIZE bytes and no spaces for padding. + * + * @return the number of effective characters in the last block. + */ + simdjson_inline size_t get_remainder(uint8_t *dst) const; + simdjson_inline void advance(); +private: + const uint8_t *buf; + const size_t len; + const size_t lenminusstep; + size_t idx; +}; + +// Routines to print masks and text for debugging bitmask operations +simdjson_unused static char * format_input_text_64(const uint8_t *text) { + static char buf[sizeof(simd8x64) + 1]; + for (size_t i=0; i); i++) { + buf[i] = int8_t(text[i]) < ' ' ? '_' : int8_t(text[i]); + } + buf[sizeof(simd8x64)] = '\0'; + return buf; +} + +// Routines to print masks and text for debugging bitmask operations +simdjson_unused static char * format_input_text(const simd8x64& in) { + static char buf[sizeof(simd8x64) + 1]; + in.store(reinterpret_cast(buf)); + for (size_t i=0; i); i++) { + if (buf[i] < ' ') { buf[i] = '_'; } + } + buf[sizeof(simd8x64)] = '\0'; + return buf; +} + +simdjson_unused static char * format_input_text(const simd8x64& in, uint64_t mask) { + static char buf[sizeof(simd8x64) + 1]; + in.store(reinterpret_cast(buf)); + for (size_t i=0; i); i++) { + if (buf[i] <= ' ') { buf[i] = '_'; } + if (!(mask & (size_t(1) << i))) { buf[i] = ' '; } + } + buf[sizeof(simd8x64)] = '\0'; + return buf; +} + +simdjson_unused static char * format_mask(uint64_t mask) { + static char buf[sizeof(simd8x64) + 1]; + for (size_t i=0; i<64; i++) { + buf[i] = (mask & (size_t(1) << i)) ? 'X' : ' '; + } + buf[64] = '\0'; + return buf; +} + +template +simdjson_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) : buf{_buf}, len{_len}, lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {} + +template +simdjson_inline size_t buf_block_reader::block_index() { return idx; } + +template +simdjson_inline bool buf_block_reader::has_full_block() const { + return idx < lenminusstep; +} + +template +simdjson_inline const uint8_t *buf_block_reader::full_block() const { + return &buf[idx]; +} + +template +simdjson_inline size_t buf_block_reader::get_remainder(uint8_t *dst) const { + if(len == idx) { return 0; } // memcpy(dst, null, 0) will trigger an error with some sanitizers + std::memset(dst, 0x20, STEP_SIZE); // std::memset STEP_SIZE because it's more efficient to write out 8 or 16 bytes at once. + std::memcpy(dst, buf + idx, len - idx); + return len - idx; +} + +template +simdjson_inline void buf_block_reader::advance() { + idx += STEP_SIZE; +} + +} // namespace stage1 +} // unnamed namespace +} // namespace arm64 +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H +/* end file generic/stage1/buf_block_reader.h for arm64 */ +/* including generic/stage1/json_escape_scanner.h for arm64: #include */ +/* begin file generic/stage1/json_escape_scanner.h for arm64 */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_ESCAPE_SCANNER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_ESCAPE_SCANNER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { +namespace arm64 { +namespace { +namespace stage1 { + +/** + * Scans for escape characters in JSON, taking care with multiple backslashes (\\n vs. \n). + */ +struct json_escape_scanner { + /** The actual escape characters (the backslashes themselves). */ + uint64_t next_is_escaped; + + struct escaped_and_escape { + /** + * Mask of escaped characters. + * + * ``` + * \n \\n \\\n \\\\n \ + * 0100100010100101000 + * n \ \ n \ \ + * ``` + */ + uint64_t escaped; + /** + * Mask of escape characters. + * + * ``` + * \n \\n \\\n \\\\n \ + * 1001000101001010001 + * \ \ \ \ \ \ \ + * ``` + */ + uint64_t escape; + }; + + /** + * Get a mask of both escape and escaped characters (the characters following a backslash). + * + * @param potential_escape A mask of the character that can escape others (but could be + * escaped itself). e.g. block.eq('\\') + */ + simdjson_really_inline escaped_and_escape next(uint64_t backslash) noexcept { + +#if !SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT + if (!backslash) { return {next_escaped_without_backslashes(), 0}; } +#endif + + // | | Mask (shows characters instead of 1's) | Depth | Instructions | + // |--------------------------------|----------------------------------------|-------|---------------------| + // | string | `\\n_\\\n___\\\n___\\\\___\\\\__\\\` | + // | | ` even odd even odd odd` | + // | potential_escape | ` \ \\\ \\\ \\\\ \\\\ \\\` | 1 | 1 (backslash & ~first_is_escaped) + // | escape_and_terminal_code | ` \n \ \n \ \n \ \ \ \ \ \` | 5 | 5 (next_escape_and_terminal_code()) + // | escaped | `\ \ n \ n \ \ \ \ \ ` X | 6 | 7 (escape_and_terminal_code ^ (potential_escape | first_is_escaped)) + // | escape | ` \ \ \ \ \ \ \ \ \ \` | 6 | 8 (escape_and_terminal_code & backslash) + // | first_is_escaped | `\ ` | 7 (*) | 9 (escape >> 63) () + // (*) this is not needed until the next iteration + uint64_t escape_and_terminal_code = next_escape_and_terminal_code(backslash & ~this->next_is_escaped); + uint64_t escaped = escape_and_terminal_code ^ (backslash | this->next_is_escaped); + uint64_t escape = escape_and_terminal_code & backslash; + this->next_is_escaped = escape >> 63; + return {.escaped=escaped, .escape=escape}; + } + +private: + static constexpr const uint64_t ODD_BITS = 0xAAAAAAAAAAAAAAAAULL; + + uint64_t next_escaped_without_backslashes() noexcept { + uint64_t escaped = this->next_is_escaped; + this->next_is_escaped = 0; + return escaped; + } + + /** + * Returns a mask of the next escape characters (masking out escaped backslashes), along with + * any non-backslash escape codes. + * + * \n \\n \\\n \\\\n returns: + * \n \ \ \n \ \ + * 11 100 1011 10100 + * + * You are expected to mask out the first bit yourself if the previous block had a trailing + * escape. + * + * & the result with potential_escape to get just the escape characters. + * ^ the result with (potential_escape | first_is_escaped) to get escaped characters. + */ + static simdjson_inline uint64_t next_escape_and_terminal_code(uint64_t potential_escape) noexcept { + // If we were to just shift and mask out any odd bits, we'd actually get a *half* right answer: + // any even-aligned backslash runs would be correct! Odd-aligned backslash runs would be + // inverted (\\\ would be 010 instead of 101). + // + // ``` + // string: | ____\\\\_\\\\_____ | + // maybe_escaped | ODD | \ \ \ \ | + // even-aligned ^^^ ^^^^ odd-aligned + // ``` + // + // Taking that into account, our basic strategy is: + // + // 1. Use subtraction to produce a mask with 1's for even-aligned runs and 0's for + // odd-aligned runs. + // 2. XOR all odd bits, which masks out the odd bits in even-aligned runs, and brings IN the + // odd bits in odd-aligned runs. + // 3. & with backslash to clean up any stray bits. + // runs are set to 0, and then XORing with "odd": + // + // | | Mask (shows characters instead of 1's) | Instructions | + // |--------------------------------|----------------------------------------|---------------------| + // | string | `\\n_\\\n___\\\n___\\\\___\\\\__\\\` | + // | | ` even odd even odd odd` | + // | maybe_escaped | ` n \\n \\n \\\_ \\\_ \\` X | 1 (potential_escape << 1) + // | maybe_escaped_and_odd | ` \n_ \\n _ \\\n_ _ \\\__ _\\\_ \\\` | 1 (maybe_escaped | odd) + // | even_series_codes_and_odd | ` n_\\\ _ n_ _\\\\ _ _ ` | 1 (maybe_escaped_and_odd - potential_escape) + // | escape_and_terminal_code | ` \n \ \n \ \n \ \ \ \ \ \` | 1 (^ odd) + // + + // Escaped characters are characters following an escape. + uint64_t maybe_escaped = potential_escape << 1; + + // To distinguish odd from even escape sequences, therefore, we turn on any *starting* + // escapes that are on an odd byte. (We actually bring in all odd bits, for speed.) + // - Odd runs of backslashes are 0000, and the code at the end ("n" in \n or \\n) is 1. + // - Odd runs of backslashes are 1111, and the code at the end ("n" in \n or \\n) is 0. + // - All other odd bytes are 1, and even bytes are 0. + uint64_t maybe_escaped_and_odd_bits = maybe_escaped | ODD_BITS; + uint64_t even_series_codes_and_odd_bits = maybe_escaped_and_odd_bits - potential_escape; + + // Now we flip all odd bytes back with xor. This: + // - Makes odd runs of backslashes go from 0000 to 1010 + // - Makes even runs of backslashes go from 1111 to 1010 + // - Sets actually-escaped codes to 1 (the n in \n and \\n: \n = 11, \\n = 100) + // - Resets all other bytes to 0 + return even_series_codes_and_odd_bits ^ ODD_BITS; + } +}; + +} // namespace stage1 +} // unnamed namespace +} // namespace arm64 +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H +/* end file generic/stage1/json_escape_scanner.h for arm64 */ /* including generic/stage1/json_string_scanner.h for arm64: #include */ /* begin file generic/stage1/json_string_scanner.h for arm64 */ #ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H @@ -10999,6 +11275,7 @@ using utf8_validation::utf8_checker; /* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ /* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H */ /* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ namespace simdjson { @@ -11008,19 +11285,13 @@ namespace stage1 { struct json_string_block { // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 - simdjson_inline json_string_block(uint64_t backslash, uint64_t escaped, uint64_t quote, uint64_t in_string) : - _backslash(backslash), _escaped(escaped), _quote(quote), _in_string(in_string) {} + simdjson_inline json_string_block(uint64_t escaped, uint64_t quote, uint64_t in_string) : + _escaped(escaped), _quote(quote), _in_string(in_string) {} // Escaped characters (characters following an escape() character) simdjson_inline uint64_t escaped() const { return _escaped; } - // Escape characters (backslashes that are not escaped--i.e. in \\, includes only the first \) - simdjson_inline uint64_t escape() const { return _backslash & ~_escaped; } // Real (non-backslashed) quotes simdjson_inline uint64_t quote() const { return _quote; } - // Start quotes of strings - simdjson_inline uint64_t string_start() const { return _quote & _in_string; } - // End quotes of strings - simdjson_inline uint64_t string_end() const { return _quote & ~_in_string; } // Only characters inside the string (not including the quotes) simdjson_inline uint64_t string_content() const { return _in_string & ~_quote; } // Return a mask of whether the given characters are inside a string (only works on non-quotes) @@ -11030,11 +11301,9 @@ struct json_string_block { // Tail of string (everything except the start quote) simdjson_inline uint64_t string_tail() const { return _in_string ^ _quote; } - // backslash characters - uint64_t _backslash; // escaped characters (backslashed--does not include the hex characters after \u) uint64_t _escaped; - // real quotes (non-backslashed ones) + // real quotes (non-escaped ones) uint64_t _quote; // string characters (includes start quote but not end quote) uint64_t _in_string; @@ -11048,60 +11317,12 @@ public: simdjson_inline error_code finish(); private: - // Intended to be defined by the implementation - simdjson_inline uint64_t find_escaped(uint64_t escape); - simdjson_inline uint64_t find_escaped_branchless(uint64_t escape); - + // Scans for escape characters + json_escape_scanner escape_scanner; // Whether the last iteration was still inside a string (all 1's = true, all 0's = false). uint64_t prev_in_string = 0ULL; - // Whether the first character of the next iteration is escaped. - uint64_t prev_escaped = 0ULL; }; -// -// Finds escaped characters (characters following \). -// -// Handles runs of backslashes like \\\" and \\\\" correctly (yielding 0101 and 01010, respectively). -// -// Does this by: -// - Shift the escape mask to get potentially escaped characters (characters after backslashes). -// - Mask escaped sequences that start on *even* bits with 1010101010 (odd bits are escaped, even bits are not) -// - Mask escaped sequences that start on *odd* bits with 0101010101 (even bits are escaped, odd bits are not) -// -// To distinguish between escaped sequences starting on even/odd bits, it finds the start of all -// escape sequences, filters out the ones that start on even bits, and adds that to the mask of -// escape sequences. This causes the addition to clear out the sequences starting on odd bits (since -// the start bit causes a carry), and leaves even-bit sequences alone. -// -// Example: -// -// text | \\\ | \\\"\\\" \\\" \\"\\" | -// escape | xxx | xx xxx xxx xx xx | Removed overflow backslash; will | it into follows_escape -// odd_starts | x | x x x | escape & ~even_bits & ~follows_escape -// even_seq | c| cxxx c xx c | c = carry bit -- will be masked out later -// invert_mask | | cxxx c xx c| even_seq << 1 -// follows_escape | xx | x xx xxx xxx xx xx | Includes overflow bit -// escaped | x | x x x x x x x x | -// desired | x | x x x x x x x x | -// text | \\\ | \\\"\\\" \\\" \\"\\" | -// -simdjson_inline uint64_t json_string_scanner::find_escaped_branchless(uint64_t backslash) { - // If there was overflow, pretend the first character isn't a backslash - backslash &= ~prev_escaped; - uint64_t follows_escape = backslash << 1 | prev_escaped; - - // Get sequences starting on even bits by clearing out the odd series using + - const uint64_t even_bits = 0x5555555555555555ULL; - uint64_t odd_sequence_starts = backslash & ~even_bits & ~follows_escape; - uint64_t sequences_starting_on_even_bits; - prev_escaped = add_overflow(odd_sequence_starts, backslash, &sequences_starting_on_even_bits); - uint64_t invert_mask = sequences_starting_on_even_bits << 1; // The mask we want to return is the *escaped* bits, not escapes. - - // Mask every other backslashed character as an escaped character - // Flip the mask for sequences that start on even bits, to correct them - return (even_bits ^ invert_mask) & follows_escape; -} - // // Return a mask of all string characters plus end quotes. // @@ -11112,7 +11333,7 @@ simdjson_inline uint64_t json_string_scanner::find_escaped_branchless(uint64_t b // simdjson_inline json_string_block json_string_scanner::next(const simd::simd8x64& in) { const uint64_t backslash = in.eq('\\'); - const uint64_t escaped = find_escaped(backslash); + const uint64_t escaped = escape_scanner.next(backslash).escaped; const uint64_t quote = in.eq('"') & ~escaped; // @@ -11126,21 +11347,13 @@ simdjson_inline json_string_block json_string_scanner::next(const simd::simd8x64 // // Check if we're still in a string at the end of the box so the next block will know // - // right shift of a signed value expected to be well-defined and standard - // compliant as of C++20, John Regher from Utah U. says this is fine code - // prev_in_string = uint64_t(static_cast(in_string) >> 63); // Use ^ to turn the beginning quote off, and the end quote on. // We are returning a function-local object so either we get a move constructor // or we get copy elision. - return json_string_block( - backslash, - escaped, - quote, - in_string - ); + return json_string_block(escaped, quote, in_string); } simdjson_inline error_code json_string_scanner::finish() { @@ -11550,114 +11763,6 @@ simdjson_inline error_code json_scanner::finish() { /* end file generic/stage1/json_scanner.h for arm64 */ // All other declarations -/* including generic/stage1/buf_block_reader.h for arm64: #include */ -/* begin file generic/stage1/buf_block_reader.h for arm64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace arm64 { -namespace { -namespace stage1 { - -// Walks through a buffer in block-sized increments, loading the last part with spaces -template -struct buf_block_reader { -public: - simdjson_inline buf_block_reader(const uint8_t *_buf, size_t _len); - simdjson_inline size_t block_index(); - simdjson_inline bool has_full_block() const; - simdjson_inline const uint8_t *full_block() const; - /** - * Get the last block, padded with spaces. - * - * There will always be a last block, with at least 1 byte, unless len == 0 (in which case this - * function fills the buffer with spaces and returns 0. In particular, if len == STEP_SIZE there - * will be 0 full_blocks and 1 remainder block with STEP_SIZE bytes and no spaces for padding. - * - * @return the number of effective characters in the last block. - */ - simdjson_inline size_t get_remainder(uint8_t *dst) const; - simdjson_inline void advance(); -private: - const uint8_t *buf; - const size_t len; - const size_t lenminusstep; - size_t idx; -}; - -// Routines to print masks and text for debugging bitmask operations -simdjson_unused static char * format_input_text_64(const uint8_t *text) { - static char buf[sizeof(simd8x64) + 1]; - for (size_t i=0; i); i++) { - buf[i] = int8_t(text[i]) < ' ' ? '_' : int8_t(text[i]); - } - buf[sizeof(simd8x64)] = '\0'; - return buf; -} - -// Routines to print masks and text for debugging bitmask operations -simdjson_unused static char * format_input_text(const simd8x64& in) { - static char buf[sizeof(simd8x64) + 1]; - in.store(reinterpret_cast(buf)); - for (size_t i=0; i); i++) { - if (buf[i] < ' ') { buf[i] = '_'; } - } - buf[sizeof(simd8x64)] = '\0'; - return buf; -} - -simdjson_unused static char * format_mask(uint64_t mask) { - static char buf[sizeof(simd8x64) + 1]; - for (size_t i=0; i<64; i++) { - buf[i] = (mask & (size_t(1) << i)) ? 'X' : ' '; - } - buf[64] = '\0'; - return buf; -} - -template -simdjson_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) : buf{_buf}, len{_len}, lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {} - -template -simdjson_inline size_t buf_block_reader::block_index() { return idx; } - -template -simdjson_inline bool buf_block_reader::has_full_block() const { - return idx < lenminusstep; -} - -template -simdjson_inline const uint8_t *buf_block_reader::full_block() const { - return &buf[idx]; -} - -template -simdjson_inline size_t buf_block_reader::get_remainder(uint8_t *dst) const { - if(len == idx) { return 0; } // memcpy(dst, null, 0) will trigger an error with some sanitizers - std::memset(dst, 0x20, STEP_SIZE); // std::memset STEP_SIZE because it's more efficient to write out 8 or 16 bytes at once. - std::memcpy(dst, buf + idx, len - idx); - return len - idx; -} - -template -simdjson_inline void buf_block_reader::advance() { - idx += STEP_SIZE; -} - -} // namespace stage1 -} // unnamed namespace -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H -/* end file generic/stage1/buf_block_reader.h for arm64 */ /* including generic/stage1/find_next_document_index.h for arm64: #include */ /* begin file generic/stage1/find_next_document_index.h for arm64 */ #ifndef SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H @@ -13610,18 +13715,6 @@ simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, // namespace simdjson { namespace arm64 { -namespace { -namespace stage1 { - -simdjson_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { - // On ARM, we don't short-circuit this if there are no backslashes, because the branch gives us no - // benefit and therefore makes things worse. - // if (!backslash) { uint64_t escaped = prev_escaped; prev_escaped = 0; return escaped; } - return find_escaped_branchless(backslash); -} - -} // namespace stage1 -} // unnamed namespace simdjson_warn_unused error_code implementation::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept { return arm64::stage1::json_minifier::minify<64>(buf, len, dst, dst_len); @@ -13668,6 +13761,7 @@ simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t * /* amalgamation skipped (editor-only): #include "simdjson/arm64/base.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ +#undef SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT /* undefining SIMDJSON_IMPLEMENTATION from "arm64" */ #undef SIMDJSON_IMPLEMENTATION /* end file simdjson/arm64/end.h */ @@ -17749,7 +17843,7 @@ simdjson_inline int leading_zeroes(uint64_t input_num) { #if SIMDJSON_REGULAR_VISUAL_STUDIO simdjson_inline unsigned __int64 count_ones(uint64_t input_num) { - // note: we do not support legacy 32-bit Windows + // note: we do not support legacy 32-bit Windows in this kernel return __popcnt64(input_num);// Visual Studio wants two underscores } #else @@ -20335,7 +20429,7 @@ simdjson_inline int leading_zeroes(uint64_t input_num) { #if SIMDJSON_REGULAR_VISUAL_STUDIO simdjson_inline unsigned __int64 count_ones(uint64_t input_num) { - // note: we do not support legacy 32-bit Windows + // note: we do not support legacy 32-bit Windows in this kernel return __popcnt64(input_num);// Visual Studio wants two underscores } #else @@ -21009,6 +21103,279 @@ using utf8_validation::utf8_checker; #endif // SIMDJSON_SRC_GENERIC_STAGE1_BASE_H /* end file generic/stage1/base.h for haswell */ +/* including generic/stage1/buf_block_reader.h for haswell: #include */ +/* begin file generic/stage1/buf_block_reader.h for haswell */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include + +namespace simdjson { +namespace haswell { +namespace { +namespace stage1 { + +// Walks through a buffer in block-sized increments, loading the last part with spaces +template +struct buf_block_reader { +public: + simdjson_inline buf_block_reader(const uint8_t *_buf, size_t _len); + simdjson_inline size_t block_index(); + simdjson_inline bool has_full_block() const; + simdjson_inline const uint8_t *full_block() const; + /** + * Get the last block, padded with spaces. + * + * There will always be a last block, with at least 1 byte, unless len == 0 (in which case this + * function fills the buffer with spaces and returns 0. In particular, if len == STEP_SIZE there + * will be 0 full_blocks and 1 remainder block with STEP_SIZE bytes and no spaces for padding. + * + * @return the number of effective characters in the last block. + */ + simdjson_inline size_t get_remainder(uint8_t *dst) const; + simdjson_inline void advance(); +private: + const uint8_t *buf; + const size_t len; + const size_t lenminusstep; + size_t idx; +}; + +// Routines to print masks and text for debugging bitmask operations +simdjson_unused static char * format_input_text_64(const uint8_t *text) { + static char buf[sizeof(simd8x64) + 1]; + for (size_t i=0; i); i++) { + buf[i] = int8_t(text[i]) < ' ' ? '_' : int8_t(text[i]); + } + buf[sizeof(simd8x64)] = '\0'; + return buf; +} + +// Routines to print masks and text for debugging bitmask operations +simdjson_unused static char * format_input_text(const simd8x64& in) { + static char buf[sizeof(simd8x64) + 1]; + in.store(reinterpret_cast(buf)); + for (size_t i=0; i); i++) { + if (buf[i] < ' ') { buf[i] = '_'; } + } + buf[sizeof(simd8x64)] = '\0'; + return buf; +} + +simdjson_unused static char * format_input_text(const simd8x64& in, uint64_t mask) { + static char buf[sizeof(simd8x64) + 1]; + in.store(reinterpret_cast(buf)); + for (size_t i=0; i); i++) { + if (buf[i] <= ' ') { buf[i] = '_'; } + if (!(mask & (size_t(1) << i))) { buf[i] = ' '; } + } + buf[sizeof(simd8x64)] = '\0'; + return buf; +} + +simdjson_unused static char * format_mask(uint64_t mask) { + static char buf[sizeof(simd8x64) + 1]; + for (size_t i=0; i<64; i++) { + buf[i] = (mask & (size_t(1) << i)) ? 'X' : ' '; + } + buf[64] = '\0'; + return buf; +} + +template +simdjson_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) : buf{_buf}, len{_len}, lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {} + +template +simdjson_inline size_t buf_block_reader::block_index() { return idx; } + +template +simdjson_inline bool buf_block_reader::has_full_block() const { + return idx < lenminusstep; +} + +template +simdjson_inline const uint8_t *buf_block_reader::full_block() const { + return &buf[idx]; +} + +template +simdjson_inline size_t buf_block_reader::get_remainder(uint8_t *dst) const { + if(len == idx) { return 0; } // memcpy(dst, null, 0) will trigger an error with some sanitizers + std::memset(dst, 0x20, STEP_SIZE); // std::memset STEP_SIZE because it's more efficient to write out 8 or 16 bytes at once. + std::memcpy(dst, buf + idx, len - idx); + return len - idx; +} + +template +simdjson_inline void buf_block_reader::advance() { + idx += STEP_SIZE; +} + +} // namespace stage1 +} // unnamed namespace +} // namespace haswell +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H +/* end file generic/stage1/buf_block_reader.h for haswell */ +/* including generic/stage1/json_escape_scanner.h for haswell: #include */ +/* begin file generic/stage1/json_escape_scanner.h for haswell */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_ESCAPE_SCANNER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_ESCAPE_SCANNER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { +namespace haswell { +namespace { +namespace stage1 { + +/** + * Scans for escape characters in JSON, taking care with multiple backslashes (\\n vs. \n). + */ +struct json_escape_scanner { + /** The actual escape characters (the backslashes themselves). */ + uint64_t next_is_escaped; + + struct escaped_and_escape { + /** + * Mask of escaped characters. + * + * ``` + * \n \\n \\\n \\\\n \ + * 0100100010100101000 + * n \ \ n \ \ + * ``` + */ + uint64_t escaped; + /** + * Mask of escape characters. + * + * ``` + * \n \\n \\\n \\\\n \ + * 1001000101001010001 + * \ \ \ \ \ \ \ + * ``` + */ + uint64_t escape; + }; + + /** + * Get a mask of both escape and escaped characters (the characters following a backslash). + * + * @param potential_escape A mask of the character that can escape others (but could be + * escaped itself). e.g. block.eq('\\') + */ + simdjson_really_inline escaped_and_escape next(uint64_t backslash) noexcept { + +#if !SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT + if (!backslash) { return {next_escaped_without_backslashes(), 0}; } +#endif + + // | | Mask (shows characters instead of 1's) | Depth | Instructions | + // |--------------------------------|----------------------------------------|-------|---------------------| + // | string | `\\n_\\\n___\\\n___\\\\___\\\\__\\\` | + // | | ` even odd even odd odd` | + // | potential_escape | ` \ \\\ \\\ \\\\ \\\\ \\\` | 1 | 1 (backslash & ~first_is_escaped) + // | escape_and_terminal_code | ` \n \ \n \ \n \ \ \ \ \ \` | 5 | 5 (next_escape_and_terminal_code()) + // | escaped | `\ \ n \ n \ \ \ \ \ ` X | 6 | 7 (escape_and_terminal_code ^ (potential_escape | first_is_escaped)) + // | escape | ` \ \ \ \ \ \ \ \ \ \` | 6 | 8 (escape_and_terminal_code & backslash) + // | first_is_escaped | `\ ` | 7 (*) | 9 (escape >> 63) () + // (*) this is not needed until the next iteration + uint64_t escape_and_terminal_code = next_escape_and_terminal_code(backslash & ~this->next_is_escaped); + uint64_t escaped = escape_and_terminal_code ^ (backslash | this->next_is_escaped); + uint64_t escape = escape_and_terminal_code & backslash; + this->next_is_escaped = escape >> 63; + return {.escaped=escaped, .escape=escape}; + } + +private: + static constexpr const uint64_t ODD_BITS = 0xAAAAAAAAAAAAAAAAULL; + + uint64_t next_escaped_without_backslashes() noexcept { + uint64_t escaped = this->next_is_escaped; + this->next_is_escaped = 0; + return escaped; + } + + /** + * Returns a mask of the next escape characters (masking out escaped backslashes), along with + * any non-backslash escape codes. + * + * \n \\n \\\n \\\\n returns: + * \n \ \ \n \ \ + * 11 100 1011 10100 + * + * You are expected to mask out the first bit yourself if the previous block had a trailing + * escape. + * + * & the result with potential_escape to get just the escape characters. + * ^ the result with (potential_escape | first_is_escaped) to get escaped characters. + */ + static simdjson_inline uint64_t next_escape_and_terminal_code(uint64_t potential_escape) noexcept { + // If we were to just shift and mask out any odd bits, we'd actually get a *half* right answer: + // any even-aligned backslash runs would be correct! Odd-aligned backslash runs would be + // inverted (\\\ would be 010 instead of 101). + // + // ``` + // string: | ____\\\\_\\\\_____ | + // maybe_escaped | ODD | \ \ \ \ | + // even-aligned ^^^ ^^^^ odd-aligned + // ``` + // + // Taking that into account, our basic strategy is: + // + // 1. Use subtraction to produce a mask with 1's for even-aligned runs and 0's for + // odd-aligned runs. + // 2. XOR all odd bits, which masks out the odd bits in even-aligned runs, and brings IN the + // odd bits in odd-aligned runs. + // 3. & with backslash to clean up any stray bits. + // runs are set to 0, and then XORing with "odd": + // + // | | Mask (shows characters instead of 1's) | Instructions | + // |--------------------------------|----------------------------------------|---------------------| + // | string | `\\n_\\\n___\\\n___\\\\___\\\\__\\\` | + // | | ` even odd even odd odd` | + // | maybe_escaped | ` n \\n \\n \\\_ \\\_ \\` X | 1 (potential_escape << 1) + // | maybe_escaped_and_odd | ` \n_ \\n _ \\\n_ _ \\\__ _\\\_ \\\` | 1 (maybe_escaped | odd) + // | even_series_codes_and_odd | ` n_\\\ _ n_ _\\\\ _ _ ` | 1 (maybe_escaped_and_odd - potential_escape) + // | escape_and_terminal_code | ` \n \ \n \ \n \ \ \ \ \ \` | 1 (^ odd) + // + + // Escaped characters are characters following an escape. + uint64_t maybe_escaped = potential_escape << 1; + + // To distinguish odd from even escape sequences, therefore, we turn on any *starting* + // escapes that are on an odd byte. (We actually bring in all odd bits, for speed.) + // - Odd runs of backslashes are 0000, and the code at the end ("n" in \n or \\n) is 1. + // - Odd runs of backslashes are 1111, and the code at the end ("n" in \n or \\n) is 0. + // - All other odd bytes are 1, and even bytes are 0. + uint64_t maybe_escaped_and_odd_bits = maybe_escaped | ODD_BITS; + uint64_t even_series_codes_and_odd_bits = maybe_escaped_and_odd_bits - potential_escape; + + // Now we flip all odd bytes back with xor. This: + // - Makes odd runs of backslashes go from 0000 to 1010 + // - Makes even runs of backslashes go from 1111 to 1010 + // - Sets actually-escaped codes to 1 (the n in \n and \\n: \n = 11, \\n = 100) + // - Resets all other bytes to 0 + return even_series_codes_and_odd_bits ^ ODD_BITS; + } +}; + +} // namespace stage1 +} // unnamed namespace +} // namespace haswell +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H +/* end file generic/stage1/json_escape_scanner.h for haswell */ /* including generic/stage1/json_string_scanner.h for haswell: #include */ /* begin file generic/stage1/json_string_scanner.h for haswell */ #ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H @@ -21016,6 +21383,7 @@ using utf8_validation::utf8_checker; /* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ /* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H */ /* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ namespace simdjson { @@ -21025,19 +21393,13 @@ namespace stage1 { struct json_string_block { // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 - simdjson_inline json_string_block(uint64_t backslash, uint64_t escaped, uint64_t quote, uint64_t in_string) : - _backslash(backslash), _escaped(escaped), _quote(quote), _in_string(in_string) {} + simdjson_inline json_string_block(uint64_t escaped, uint64_t quote, uint64_t in_string) : + _escaped(escaped), _quote(quote), _in_string(in_string) {} // Escaped characters (characters following an escape() character) simdjson_inline uint64_t escaped() const { return _escaped; } - // Escape characters (backslashes that are not escaped--i.e. in \\, includes only the first \) - simdjson_inline uint64_t escape() const { return _backslash & ~_escaped; } // Real (non-backslashed) quotes simdjson_inline uint64_t quote() const { return _quote; } - // Start quotes of strings - simdjson_inline uint64_t string_start() const { return _quote & _in_string; } - // End quotes of strings - simdjson_inline uint64_t string_end() const { return _quote & ~_in_string; } // Only characters inside the string (not including the quotes) simdjson_inline uint64_t string_content() const { return _in_string & ~_quote; } // Return a mask of whether the given characters are inside a string (only works on non-quotes) @@ -21047,11 +21409,9 @@ struct json_string_block { // Tail of string (everything except the start quote) simdjson_inline uint64_t string_tail() const { return _in_string ^ _quote; } - // backslash characters - uint64_t _backslash; // escaped characters (backslashed--does not include the hex characters after \u) uint64_t _escaped; - // real quotes (non-backslashed ones) + // real quotes (non-escaped ones) uint64_t _quote; // string characters (includes start quote but not end quote) uint64_t _in_string; @@ -21065,60 +21425,12 @@ public: simdjson_inline error_code finish(); private: - // Intended to be defined by the implementation - simdjson_inline uint64_t find_escaped(uint64_t escape); - simdjson_inline uint64_t find_escaped_branchless(uint64_t escape); - + // Scans for escape characters + json_escape_scanner escape_scanner; // Whether the last iteration was still inside a string (all 1's = true, all 0's = false). uint64_t prev_in_string = 0ULL; - // Whether the first character of the next iteration is escaped. - uint64_t prev_escaped = 0ULL; }; -// -// Finds escaped characters (characters following \). -// -// Handles runs of backslashes like \\\" and \\\\" correctly (yielding 0101 and 01010, respectively). -// -// Does this by: -// - Shift the escape mask to get potentially escaped characters (characters after backslashes). -// - Mask escaped sequences that start on *even* bits with 1010101010 (odd bits are escaped, even bits are not) -// - Mask escaped sequences that start on *odd* bits with 0101010101 (even bits are escaped, odd bits are not) -// -// To distinguish between escaped sequences starting on even/odd bits, it finds the start of all -// escape sequences, filters out the ones that start on even bits, and adds that to the mask of -// escape sequences. This causes the addition to clear out the sequences starting on odd bits (since -// the start bit causes a carry), and leaves even-bit sequences alone. -// -// Example: -// -// text | \\\ | \\\"\\\" \\\" \\"\\" | -// escape | xxx | xx xxx xxx xx xx | Removed overflow backslash; will | it into follows_escape -// odd_starts | x | x x x | escape & ~even_bits & ~follows_escape -// even_seq | c| cxxx c xx c | c = carry bit -- will be masked out later -// invert_mask | | cxxx c xx c| even_seq << 1 -// follows_escape | xx | x xx xxx xxx xx xx | Includes overflow bit -// escaped | x | x x x x x x x x | -// desired | x | x x x x x x x x | -// text | \\\ | \\\"\\\" \\\" \\"\\" | -// -simdjson_inline uint64_t json_string_scanner::find_escaped_branchless(uint64_t backslash) { - // If there was overflow, pretend the first character isn't a backslash - backslash &= ~prev_escaped; - uint64_t follows_escape = backslash << 1 | prev_escaped; - - // Get sequences starting on even bits by clearing out the odd series using + - const uint64_t even_bits = 0x5555555555555555ULL; - uint64_t odd_sequence_starts = backslash & ~even_bits & ~follows_escape; - uint64_t sequences_starting_on_even_bits; - prev_escaped = add_overflow(odd_sequence_starts, backslash, &sequences_starting_on_even_bits); - uint64_t invert_mask = sequences_starting_on_even_bits << 1; // The mask we want to return is the *escaped* bits, not escapes. - - // Mask every other backslashed character as an escaped character - // Flip the mask for sequences that start on even bits, to correct them - return (even_bits ^ invert_mask) & follows_escape; -} - // // Return a mask of all string characters plus end quotes. // @@ -21129,7 +21441,7 @@ simdjson_inline uint64_t json_string_scanner::find_escaped_branchless(uint64_t b // simdjson_inline json_string_block json_string_scanner::next(const simd::simd8x64& in) { const uint64_t backslash = in.eq('\\'); - const uint64_t escaped = find_escaped(backslash); + const uint64_t escaped = escape_scanner.next(backslash).escaped; const uint64_t quote = in.eq('"') & ~escaped; // @@ -21143,21 +21455,13 @@ simdjson_inline json_string_block json_string_scanner::next(const simd::simd8x64 // // Check if we're still in a string at the end of the box so the next block will know // - // right shift of a signed value expected to be well-defined and standard - // compliant as of C++20, John Regher from Utah U. says this is fine code - // prev_in_string = uint64_t(static_cast(in_string) >> 63); // Use ^ to turn the beginning quote off, and the end quote on. // We are returning a function-local object so either we get a move constructor // or we get copy elision. - return json_string_block( - backslash, - escaped, - quote, - in_string - ); + return json_string_block(escaped, quote, in_string); } simdjson_inline error_code json_string_scanner::finish() { @@ -21567,114 +21871,6 @@ simdjson_inline error_code json_scanner::finish() { /* end file generic/stage1/json_scanner.h for haswell */ // All other declarations -/* including generic/stage1/buf_block_reader.h for haswell: #include */ -/* begin file generic/stage1/buf_block_reader.h for haswell */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace haswell { -namespace { -namespace stage1 { - -// Walks through a buffer in block-sized increments, loading the last part with spaces -template -struct buf_block_reader { -public: - simdjson_inline buf_block_reader(const uint8_t *_buf, size_t _len); - simdjson_inline size_t block_index(); - simdjson_inline bool has_full_block() const; - simdjson_inline const uint8_t *full_block() const; - /** - * Get the last block, padded with spaces. - * - * There will always be a last block, with at least 1 byte, unless len == 0 (in which case this - * function fills the buffer with spaces and returns 0. In particular, if len == STEP_SIZE there - * will be 0 full_blocks and 1 remainder block with STEP_SIZE bytes and no spaces for padding. - * - * @return the number of effective characters in the last block. - */ - simdjson_inline size_t get_remainder(uint8_t *dst) const; - simdjson_inline void advance(); -private: - const uint8_t *buf; - const size_t len; - const size_t lenminusstep; - size_t idx; -}; - -// Routines to print masks and text for debugging bitmask operations -simdjson_unused static char * format_input_text_64(const uint8_t *text) { - static char buf[sizeof(simd8x64) + 1]; - for (size_t i=0; i); i++) { - buf[i] = int8_t(text[i]) < ' ' ? '_' : int8_t(text[i]); - } - buf[sizeof(simd8x64)] = '\0'; - return buf; -} - -// Routines to print masks and text for debugging bitmask operations -simdjson_unused static char * format_input_text(const simd8x64& in) { - static char buf[sizeof(simd8x64) + 1]; - in.store(reinterpret_cast(buf)); - for (size_t i=0; i); i++) { - if (buf[i] < ' ') { buf[i] = '_'; } - } - buf[sizeof(simd8x64)] = '\0'; - return buf; -} - -simdjson_unused static char * format_mask(uint64_t mask) { - static char buf[sizeof(simd8x64) + 1]; - for (size_t i=0; i<64; i++) { - buf[i] = (mask & (size_t(1) << i)) ? 'X' : ' '; - } - buf[64] = '\0'; - return buf; -} - -template -simdjson_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) : buf{_buf}, len{_len}, lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {} - -template -simdjson_inline size_t buf_block_reader::block_index() { return idx; } - -template -simdjson_inline bool buf_block_reader::has_full_block() const { - return idx < lenminusstep; -} - -template -simdjson_inline const uint8_t *buf_block_reader::full_block() const { - return &buf[idx]; -} - -template -simdjson_inline size_t buf_block_reader::get_remainder(uint8_t *dst) const { - if(len == idx) { return 0; } // memcpy(dst, null, 0) will trigger an error with some sanitizers - std::memset(dst, 0x20, STEP_SIZE); // std::memset STEP_SIZE because it's more efficient to write out 8 or 16 bytes at once. - std::memcpy(dst, buf + idx, len - idx); - return len - idx; -} - -template -simdjson_inline void buf_block_reader::advance() { - idx += STEP_SIZE; -} - -} // namespace stage1 -} // unnamed namespace -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H -/* end file generic/stage1/buf_block_reader.h for haswell */ /* including generic/stage1/find_next_document_index.h for haswell: #include */ /* begin file generic/stage1/find_next_document_index.h for haswell */ #ifndef SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H @@ -23625,16 +23821,6 @@ simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, // namespace simdjson { namespace haswell { -namespace { -namespace stage1 { - -simdjson_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { - if (!backslash) { uint64_t escaped = prev_escaped; prev_escaped = 0; return escaped; } - return find_escaped_branchless(backslash); -} - -} // namespace stage1 -} // unnamed namespace simdjson_warn_unused error_code implementation::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept { return haswell::stage1::json_minifier::minify<128>(buf, len, dst, dst_len); @@ -27103,6 +27289,279 @@ using utf8_validation::utf8_checker; #endif // SIMDJSON_SRC_GENERIC_STAGE1_BASE_H /* end file generic/stage1/base.h for icelake */ +/* including generic/stage1/buf_block_reader.h for icelake: #include */ +/* begin file generic/stage1/buf_block_reader.h for icelake */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include + +namespace simdjson { +namespace icelake { +namespace { +namespace stage1 { + +// Walks through a buffer in block-sized increments, loading the last part with spaces +template +struct buf_block_reader { +public: + simdjson_inline buf_block_reader(const uint8_t *_buf, size_t _len); + simdjson_inline size_t block_index(); + simdjson_inline bool has_full_block() const; + simdjson_inline const uint8_t *full_block() const; + /** + * Get the last block, padded with spaces. + * + * There will always be a last block, with at least 1 byte, unless len == 0 (in which case this + * function fills the buffer with spaces and returns 0. In particular, if len == STEP_SIZE there + * will be 0 full_blocks and 1 remainder block with STEP_SIZE bytes and no spaces for padding. + * + * @return the number of effective characters in the last block. + */ + simdjson_inline size_t get_remainder(uint8_t *dst) const; + simdjson_inline void advance(); +private: + const uint8_t *buf; + const size_t len; + const size_t lenminusstep; + size_t idx; +}; + +// Routines to print masks and text for debugging bitmask operations +simdjson_unused static char * format_input_text_64(const uint8_t *text) { + static char buf[sizeof(simd8x64) + 1]; + for (size_t i=0; i); i++) { + buf[i] = int8_t(text[i]) < ' ' ? '_' : int8_t(text[i]); + } + buf[sizeof(simd8x64)] = '\0'; + return buf; +} + +// Routines to print masks and text for debugging bitmask operations +simdjson_unused static char * format_input_text(const simd8x64& in) { + static char buf[sizeof(simd8x64) + 1]; + in.store(reinterpret_cast(buf)); + for (size_t i=0; i); i++) { + if (buf[i] < ' ') { buf[i] = '_'; } + } + buf[sizeof(simd8x64)] = '\0'; + return buf; +} + +simdjson_unused static char * format_input_text(const simd8x64& in, uint64_t mask) { + static char buf[sizeof(simd8x64) + 1]; + in.store(reinterpret_cast(buf)); + for (size_t i=0; i); i++) { + if (buf[i] <= ' ') { buf[i] = '_'; } + if (!(mask & (size_t(1) << i))) { buf[i] = ' '; } + } + buf[sizeof(simd8x64)] = '\0'; + return buf; +} + +simdjson_unused static char * format_mask(uint64_t mask) { + static char buf[sizeof(simd8x64) + 1]; + for (size_t i=0; i<64; i++) { + buf[i] = (mask & (size_t(1) << i)) ? 'X' : ' '; + } + buf[64] = '\0'; + return buf; +} + +template +simdjson_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) : buf{_buf}, len{_len}, lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {} + +template +simdjson_inline size_t buf_block_reader::block_index() { return idx; } + +template +simdjson_inline bool buf_block_reader::has_full_block() const { + return idx < lenminusstep; +} + +template +simdjson_inline const uint8_t *buf_block_reader::full_block() const { + return &buf[idx]; +} + +template +simdjson_inline size_t buf_block_reader::get_remainder(uint8_t *dst) const { + if(len == idx) { return 0; } // memcpy(dst, null, 0) will trigger an error with some sanitizers + std::memset(dst, 0x20, STEP_SIZE); // std::memset STEP_SIZE because it's more efficient to write out 8 or 16 bytes at once. + std::memcpy(dst, buf + idx, len - idx); + return len - idx; +} + +template +simdjson_inline void buf_block_reader::advance() { + idx += STEP_SIZE; +} + +} // namespace stage1 +} // unnamed namespace +} // namespace icelake +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H +/* end file generic/stage1/buf_block_reader.h for icelake */ +/* including generic/stage1/json_escape_scanner.h for icelake: #include */ +/* begin file generic/stage1/json_escape_scanner.h for icelake */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_ESCAPE_SCANNER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_ESCAPE_SCANNER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { +namespace icelake { +namespace { +namespace stage1 { + +/** + * Scans for escape characters in JSON, taking care with multiple backslashes (\\n vs. \n). + */ +struct json_escape_scanner { + /** The actual escape characters (the backslashes themselves). */ + uint64_t next_is_escaped; + + struct escaped_and_escape { + /** + * Mask of escaped characters. + * + * ``` + * \n \\n \\\n \\\\n \ + * 0100100010100101000 + * n \ \ n \ \ + * ``` + */ + uint64_t escaped; + /** + * Mask of escape characters. + * + * ``` + * \n \\n \\\n \\\\n \ + * 1001000101001010001 + * \ \ \ \ \ \ \ + * ``` + */ + uint64_t escape; + }; + + /** + * Get a mask of both escape and escaped characters (the characters following a backslash). + * + * @param potential_escape A mask of the character that can escape others (but could be + * escaped itself). e.g. block.eq('\\') + */ + simdjson_really_inline escaped_and_escape next(uint64_t backslash) noexcept { + +#if !SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT + if (!backslash) { return {next_escaped_without_backslashes(), 0}; } +#endif + + // | | Mask (shows characters instead of 1's) | Depth | Instructions | + // |--------------------------------|----------------------------------------|-------|---------------------| + // | string | `\\n_\\\n___\\\n___\\\\___\\\\__\\\` | + // | | ` even odd even odd odd` | + // | potential_escape | ` \ \\\ \\\ \\\\ \\\\ \\\` | 1 | 1 (backslash & ~first_is_escaped) + // | escape_and_terminal_code | ` \n \ \n \ \n \ \ \ \ \ \` | 5 | 5 (next_escape_and_terminal_code()) + // | escaped | `\ \ n \ n \ \ \ \ \ ` X | 6 | 7 (escape_and_terminal_code ^ (potential_escape | first_is_escaped)) + // | escape | ` \ \ \ \ \ \ \ \ \ \` | 6 | 8 (escape_and_terminal_code & backslash) + // | first_is_escaped | `\ ` | 7 (*) | 9 (escape >> 63) () + // (*) this is not needed until the next iteration + uint64_t escape_and_terminal_code = next_escape_and_terminal_code(backslash & ~this->next_is_escaped); + uint64_t escaped = escape_and_terminal_code ^ (backslash | this->next_is_escaped); + uint64_t escape = escape_and_terminal_code & backslash; + this->next_is_escaped = escape >> 63; + return {.escaped=escaped, .escape=escape}; + } + +private: + static constexpr const uint64_t ODD_BITS = 0xAAAAAAAAAAAAAAAAULL; + + uint64_t next_escaped_without_backslashes() noexcept { + uint64_t escaped = this->next_is_escaped; + this->next_is_escaped = 0; + return escaped; + } + + /** + * Returns a mask of the next escape characters (masking out escaped backslashes), along with + * any non-backslash escape codes. + * + * \n \\n \\\n \\\\n returns: + * \n \ \ \n \ \ + * 11 100 1011 10100 + * + * You are expected to mask out the first bit yourself if the previous block had a trailing + * escape. + * + * & the result with potential_escape to get just the escape characters. + * ^ the result with (potential_escape | first_is_escaped) to get escaped characters. + */ + static simdjson_inline uint64_t next_escape_and_terminal_code(uint64_t potential_escape) noexcept { + // If we were to just shift and mask out any odd bits, we'd actually get a *half* right answer: + // any even-aligned backslash runs would be correct! Odd-aligned backslash runs would be + // inverted (\\\ would be 010 instead of 101). + // + // ``` + // string: | ____\\\\_\\\\_____ | + // maybe_escaped | ODD | \ \ \ \ | + // even-aligned ^^^ ^^^^ odd-aligned + // ``` + // + // Taking that into account, our basic strategy is: + // + // 1. Use subtraction to produce a mask with 1's for even-aligned runs and 0's for + // odd-aligned runs. + // 2. XOR all odd bits, which masks out the odd bits in even-aligned runs, and brings IN the + // odd bits in odd-aligned runs. + // 3. & with backslash to clean up any stray bits. + // runs are set to 0, and then XORing with "odd": + // + // | | Mask (shows characters instead of 1's) | Instructions | + // |--------------------------------|----------------------------------------|---------------------| + // | string | `\\n_\\\n___\\\n___\\\\___\\\\__\\\` | + // | | ` even odd even odd odd` | + // | maybe_escaped | ` n \\n \\n \\\_ \\\_ \\` X | 1 (potential_escape << 1) + // | maybe_escaped_and_odd | ` \n_ \\n _ \\\n_ _ \\\__ _\\\_ \\\` | 1 (maybe_escaped | odd) + // | even_series_codes_and_odd | ` n_\\\ _ n_ _\\\\ _ _ ` | 1 (maybe_escaped_and_odd - potential_escape) + // | escape_and_terminal_code | ` \n \ \n \ \n \ \ \ \ \ \` | 1 (^ odd) + // + + // Escaped characters are characters following an escape. + uint64_t maybe_escaped = potential_escape << 1; + + // To distinguish odd from even escape sequences, therefore, we turn on any *starting* + // escapes that are on an odd byte. (We actually bring in all odd bits, for speed.) + // - Odd runs of backslashes are 0000, and the code at the end ("n" in \n or \\n) is 1. + // - Odd runs of backslashes are 1111, and the code at the end ("n" in \n or \\n) is 0. + // - All other odd bytes are 1, and even bytes are 0. + uint64_t maybe_escaped_and_odd_bits = maybe_escaped | ODD_BITS; + uint64_t even_series_codes_and_odd_bits = maybe_escaped_and_odd_bits - potential_escape; + + // Now we flip all odd bytes back with xor. This: + // - Makes odd runs of backslashes go from 0000 to 1010 + // - Makes even runs of backslashes go from 1111 to 1010 + // - Sets actually-escaped codes to 1 (the n in \n and \\n: \n = 11, \\n = 100) + // - Resets all other bytes to 0 + return even_series_codes_and_odd_bits ^ ODD_BITS; + } +}; + +} // namespace stage1 +} // unnamed namespace +} // namespace icelake +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H +/* end file generic/stage1/json_escape_scanner.h for icelake */ /* including generic/stage1/json_string_scanner.h for icelake: #include */ /* begin file generic/stage1/json_string_scanner.h for icelake */ #ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H @@ -27110,6 +27569,7 @@ using utf8_validation::utf8_checker; /* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ /* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H */ /* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ namespace simdjson { @@ -27119,19 +27579,13 @@ namespace stage1 { struct json_string_block { // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 - simdjson_inline json_string_block(uint64_t backslash, uint64_t escaped, uint64_t quote, uint64_t in_string) : - _backslash(backslash), _escaped(escaped), _quote(quote), _in_string(in_string) {} + simdjson_inline json_string_block(uint64_t escaped, uint64_t quote, uint64_t in_string) : + _escaped(escaped), _quote(quote), _in_string(in_string) {} // Escaped characters (characters following an escape() character) simdjson_inline uint64_t escaped() const { return _escaped; } - // Escape characters (backslashes that are not escaped--i.e. in \\, includes only the first \) - simdjson_inline uint64_t escape() const { return _backslash & ~_escaped; } // Real (non-backslashed) quotes simdjson_inline uint64_t quote() const { return _quote; } - // Start quotes of strings - simdjson_inline uint64_t string_start() const { return _quote & _in_string; } - // End quotes of strings - simdjson_inline uint64_t string_end() const { return _quote & ~_in_string; } // Only characters inside the string (not including the quotes) simdjson_inline uint64_t string_content() const { return _in_string & ~_quote; } // Return a mask of whether the given characters are inside a string (only works on non-quotes) @@ -27141,11 +27595,9 @@ struct json_string_block { // Tail of string (everything except the start quote) simdjson_inline uint64_t string_tail() const { return _in_string ^ _quote; } - // backslash characters - uint64_t _backslash; // escaped characters (backslashed--does not include the hex characters after \u) uint64_t _escaped; - // real quotes (non-backslashed ones) + // real quotes (non-escaped ones) uint64_t _quote; // string characters (includes start quote but not end quote) uint64_t _in_string; @@ -27159,60 +27611,12 @@ public: simdjson_inline error_code finish(); private: - // Intended to be defined by the implementation - simdjson_inline uint64_t find_escaped(uint64_t escape); - simdjson_inline uint64_t find_escaped_branchless(uint64_t escape); - + // Scans for escape characters + json_escape_scanner escape_scanner; // Whether the last iteration was still inside a string (all 1's = true, all 0's = false). uint64_t prev_in_string = 0ULL; - // Whether the first character of the next iteration is escaped. - uint64_t prev_escaped = 0ULL; }; -// -// Finds escaped characters (characters following \). -// -// Handles runs of backslashes like \\\" and \\\\" correctly (yielding 0101 and 01010, respectively). -// -// Does this by: -// - Shift the escape mask to get potentially escaped characters (characters after backslashes). -// - Mask escaped sequences that start on *even* bits with 1010101010 (odd bits are escaped, even bits are not) -// - Mask escaped sequences that start on *odd* bits with 0101010101 (even bits are escaped, odd bits are not) -// -// To distinguish between escaped sequences starting on even/odd bits, it finds the start of all -// escape sequences, filters out the ones that start on even bits, and adds that to the mask of -// escape sequences. This causes the addition to clear out the sequences starting on odd bits (since -// the start bit causes a carry), and leaves even-bit sequences alone. -// -// Example: -// -// text | \\\ | \\\"\\\" \\\" \\"\\" | -// escape | xxx | xx xxx xxx xx xx | Removed overflow backslash; will | it into follows_escape -// odd_starts | x | x x x | escape & ~even_bits & ~follows_escape -// even_seq | c| cxxx c xx c | c = carry bit -- will be masked out later -// invert_mask | | cxxx c xx c| even_seq << 1 -// follows_escape | xx | x xx xxx xxx xx xx | Includes overflow bit -// escaped | x | x x x x x x x x | -// desired | x | x x x x x x x x | -// text | \\\ | \\\"\\\" \\\" \\"\\" | -// -simdjson_inline uint64_t json_string_scanner::find_escaped_branchless(uint64_t backslash) { - // If there was overflow, pretend the first character isn't a backslash - backslash &= ~prev_escaped; - uint64_t follows_escape = backslash << 1 | prev_escaped; - - // Get sequences starting on even bits by clearing out the odd series using + - const uint64_t even_bits = 0x5555555555555555ULL; - uint64_t odd_sequence_starts = backslash & ~even_bits & ~follows_escape; - uint64_t sequences_starting_on_even_bits; - prev_escaped = add_overflow(odd_sequence_starts, backslash, &sequences_starting_on_even_bits); - uint64_t invert_mask = sequences_starting_on_even_bits << 1; // The mask we want to return is the *escaped* bits, not escapes. - - // Mask every other backslashed character as an escaped character - // Flip the mask for sequences that start on even bits, to correct them - return (even_bits ^ invert_mask) & follows_escape; -} - // // Return a mask of all string characters plus end quotes. // @@ -27223,7 +27627,7 @@ simdjson_inline uint64_t json_string_scanner::find_escaped_branchless(uint64_t b // simdjson_inline json_string_block json_string_scanner::next(const simd::simd8x64& in) { const uint64_t backslash = in.eq('\\'); - const uint64_t escaped = find_escaped(backslash); + const uint64_t escaped = escape_scanner.next(backslash).escaped; const uint64_t quote = in.eq('"') & ~escaped; // @@ -27237,21 +27641,13 @@ simdjson_inline json_string_block json_string_scanner::next(const simd::simd8x64 // // Check if we're still in a string at the end of the box so the next block will know // - // right shift of a signed value expected to be well-defined and standard - // compliant as of C++20, John Regher from Utah U. says this is fine code - // prev_in_string = uint64_t(static_cast(in_string) >> 63); // Use ^ to turn the beginning quote off, and the end quote on. // We are returning a function-local object so either we get a move constructor // or we get copy elision. - return json_string_block( - backslash, - escaped, - quote, - in_string - ); + return json_string_block(escaped, quote, in_string); } simdjson_inline error_code json_string_scanner::finish() { @@ -27661,114 +28057,6 @@ simdjson_inline error_code json_scanner::finish() { /* end file generic/stage1/json_scanner.h for icelake */ // All other declarations -/* including generic/stage1/buf_block_reader.h for icelake: #include */ -/* begin file generic/stage1/buf_block_reader.h for icelake */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace icelake { -namespace { -namespace stage1 { - -// Walks through a buffer in block-sized increments, loading the last part with spaces -template -struct buf_block_reader { -public: - simdjson_inline buf_block_reader(const uint8_t *_buf, size_t _len); - simdjson_inline size_t block_index(); - simdjson_inline bool has_full_block() const; - simdjson_inline const uint8_t *full_block() const; - /** - * Get the last block, padded with spaces. - * - * There will always be a last block, with at least 1 byte, unless len == 0 (in which case this - * function fills the buffer with spaces and returns 0. In particular, if len == STEP_SIZE there - * will be 0 full_blocks and 1 remainder block with STEP_SIZE bytes and no spaces for padding. - * - * @return the number of effective characters in the last block. - */ - simdjson_inline size_t get_remainder(uint8_t *dst) const; - simdjson_inline void advance(); -private: - const uint8_t *buf; - const size_t len; - const size_t lenminusstep; - size_t idx; -}; - -// Routines to print masks and text for debugging bitmask operations -simdjson_unused static char * format_input_text_64(const uint8_t *text) { - static char buf[sizeof(simd8x64) + 1]; - for (size_t i=0; i); i++) { - buf[i] = int8_t(text[i]) < ' ' ? '_' : int8_t(text[i]); - } - buf[sizeof(simd8x64)] = '\0'; - return buf; -} - -// Routines to print masks and text for debugging bitmask operations -simdjson_unused static char * format_input_text(const simd8x64& in) { - static char buf[sizeof(simd8x64) + 1]; - in.store(reinterpret_cast(buf)); - for (size_t i=0; i); i++) { - if (buf[i] < ' ') { buf[i] = '_'; } - } - buf[sizeof(simd8x64)] = '\0'; - return buf; -} - -simdjson_unused static char * format_mask(uint64_t mask) { - static char buf[sizeof(simd8x64) + 1]; - for (size_t i=0; i<64; i++) { - buf[i] = (mask & (size_t(1) << i)) ? 'X' : ' '; - } - buf[64] = '\0'; - return buf; -} - -template -simdjson_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) : buf{_buf}, len{_len}, lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {} - -template -simdjson_inline size_t buf_block_reader::block_index() { return idx; } - -template -simdjson_inline bool buf_block_reader::has_full_block() const { - return idx < lenminusstep; -} - -template -simdjson_inline const uint8_t *buf_block_reader::full_block() const { - return &buf[idx]; -} - -template -simdjson_inline size_t buf_block_reader::get_remainder(uint8_t *dst) const { - if(len == idx) { return 0; } // memcpy(dst, null, 0) will trigger an error with some sanitizers - std::memset(dst, 0x20, STEP_SIZE); // std::memset STEP_SIZE because it's more efficient to write out 8 or 16 bytes at once. - std::memcpy(dst, buf + idx, len - idx); - return len - idx; -} - -template -simdjson_inline void buf_block_reader::advance() { - idx += STEP_SIZE; -} - -} // namespace stage1 -} // unnamed namespace -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H -/* end file generic/stage1/buf_block_reader.h for icelake */ /* including generic/stage1/find_next_document_index.h for icelake: #include */ /* begin file generic/stage1/find_next_document_index.h for icelake */ #ifndef SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H @@ -29762,16 +30050,6 @@ SIMDJSON_POP_DISABLE_WARNINGS // namespace simdjson { namespace icelake { -namespace { -namespace stage1 { - -simdjson_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { - if (!backslash) { uint64_t escaped = prev_escaped; prev_escaped = 0; return escaped; } - return find_escaped_branchless(backslash); -} - -} // namespace stage1 -} // unnamed namespace simdjson_warn_unused error_code implementation::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept { return icelake::stage1::json_minifier::minify<128>(buf, len, dst, dst_len); @@ -29958,7 +30236,7 @@ simdjson_inline int leading_zeroes(uint64_t input_num) { #if SIMDJSON_REGULAR_VISUAL_STUDIO simdjson_inline int count_ones(uint64_t input_num) { - // note: we do not support legacy 32-bit Windows + // note: we do not support legacy 32-bit Windows in this kernel return __popcnt64(input_num); // Visual Studio wants two underscores } #else @@ -30644,6 +30922,8 @@ backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { #endif // SIMDJSON_PPC64_STRINGPARSING_DEFS_H /* end file simdjson/ppc64/stringparsing_defs.h */ + +#define SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT 1 /* end file simdjson/ppc64/begin.h */ /* including simdjson/generic/amalgamated.h for ppc64: #include "simdjson/generic/amalgamated.h" */ /* begin file simdjson/generic/amalgamated.h for ppc64 */ @@ -32488,6 +32768,7 @@ simdjson_inline implementation_simdjson_result_base::implementation_simdjson_ /* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ +#undef SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT /* undefining SIMDJSON_IMPLEMENTATION from "ppc64" */ #undef SIMDJSON_IMPLEMENTATION /* end file simdjson/ppc64/end.h */ @@ -32652,7 +32933,7 @@ simdjson_inline int leading_zeroes(uint64_t input_num) { #if SIMDJSON_REGULAR_VISUAL_STUDIO simdjson_inline int count_ones(uint64_t input_num) { - // note: we do not support legacy 32-bit Windows + // note: we do not support legacy 32-bit Windows in this kernel return __popcnt64(input_num); // Visual Studio wants two underscores } #else @@ -33338,6 +33619,8 @@ backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { #endif // SIMDJSON_PPC64_STRINGPARSING_DEFS_H /* end file simdjson/ppc64/stringparsing_defs.h */ + +#define SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT 1 /* end file simdjson/ppc64/begin.h */ /* including generic/amalgamated.h for ppc64: #include */ /* begin file generic/amalgamated.h for ppc64 */ @@ -33463,6 +33746,279 @@ using utf8_validation::utf8_checker; #endif // SIMDJSON_SRC_GENERIC_STAGE1_BASE_H /* end file generic/stage1/base.h for ppc64 */ +/* including generic/stage1/buf_block_reader.h for ppc64: #include */ +/* begin file generic/stage1/buf_block_reader.h for ppc64 */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include + +namespace simdjson { +namespace ppc64 { +namespace { +namespace stage1 { + +// Walks through a buffer in block-sized increments, loading the last part with spaces +template +struct buf_block_reader { +public: + simdjson_inline buf_block_reader(const uint8_t *_buf, size_t _len); + simdjson_inline size_t block_index(); + simdjson_inline bool has_full_block() const; + simdjson_inline const uint8_t *full_block() const; + /** + * Get the last block, padded with spaces. + * + * There will always be a last block, with at least 1 byte, unless len == 0 (in which case this + * function fills the buffer with spaces and returns 0. In particular, if len == STEP_SIZE there + * will be 0 full_blocks and 1 remainder block with STEP_SIZE bytes and no spaces for padding. + * + * @return the number of effective characters in the last block. + */ + simdjson_inline size_t get_remainder(uint8_t *dst) const; + simdjson_inline void advance(); +private: + const uint8_t *buf; + const size_t len; + const size_t lenminusstep; + size_t idx; +}; + +// Routines to print masks and text for debugging bitmask operations +simdjson_unused static char * format_input_text_64(const uint8_t *text) { + static char buf[sizeof(simd8x64) + 1]; + for (size_t i=0; i); i++) { + buf[i] = int8_t(text[i]) < ' ' ? '_' : int8_t(text[i]); + } + buf[sizeof(simd8x64)] = '\0'; + return buf; +} + +// Routines to print masks and text for debugging bitmask operations +simdjson_unused static char * format_input_text(const simd8x64& in) { + static char buf[sizeof(simd8x64) + 1]; + in.store(reinterpret_cast(buf)); + for (size_t i=0; i); i++) { + if (buf[i] < ' ') { buf[i] = '_'; } + } + buf[sizeof(simd8x64)] = '\0'; + return buf; +} + +simdjson_unused static char * format_input_text(const simd8x64& in, uint64_t mask) { + static char buf[sizeof(simd8x64) + 1]; + in.store(reinterpret_cast(buf)); + for (size_t i=0; i); i++) { + if (buf[i] <= ' ') { buf[i] = '_'; } + if (!(mask & (size_t(1) << i))) { buf[i] = ' '; } + } + buf[sizeof(simd8x64)] = '\0'; + return buf; +} + +simdjson_unused static char * format_mask(uint64_t mask) { + static char buf[sizeof(simd8x64) + 1]; + for (size_t i=0; i<64; i++) { + buf[i] = (mask & (size_t(1) << i)) ? 'X' : ' '; + } + buf[64] = '\0'; + return buf; +} + +template +simdjson_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) : buf{_buf}, len{_len}, lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {} + +template +simdjson_inline size_t buf_block_reader::block_index() { return idx; } + +template +simdjson_inline bool buf_block_reader::has_full_block() const { + return idx < lenminusstep; +} + +template +simdjson_inline const uint8_t *buf_block_reader::full_block() const { + return &buf[idx]; +} + +template +simdjson_inline size_t buf_block_reader::get_remainder(uint8_t *dst) const { + if(len == idx) { return 0; } // memcpy(dst, null, 0) will trigger an error with some sanitizers + std::memset(dst, 0x20, STEP_SIZE); // std::memset STEP_SIZE because it's more efficient to write out 8 or 16 bytes at once. + std::memcpy(dst, buf + idx, len - idx); + return len - idx; +} + +template +simdjson_inline void buf_block_reader::advance() { + idx += STEP_SIZE; +} + +} // namespace stage1 +} // unnamed namespace +} // namespace ppc64 +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H +/* end file generic/stage1/buf_block_reader.h for ppc64 */ +/* including generic/stage1/json_escape_scanner.h for ppc64: #include */ +/* begin file generic/stage1/json_escape_scanner.h for ppc64 */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_ESCAPE_SCANNER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_ESCAPE_SCANNER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { +namespace ppc64 { +namespace { +namespace stage1 { + +/** + * Scans for escape characters in JSON, taking care with multiple backslashes (\\n vs. \n). + */ +struct json_escape_scanner { + /** The actual escape characters (the backslashes themselves). */ + uint64_t next_is_escaped; + + struct escaped_and_escape { + /** + * Mask of escaped characters. + * + * ``` + * \n \\n \\\n \\\\n \ + * 0100100010100101000 + * n \ \ n \ \ + * ``` + */ + uint64_t escaped; + /** + * Mask of escape characters. + * + * ``` + * \n \\n \\\n \\\\n \ + * 1001000101001010001 + * \ \ \ \ \ \ \ + * ``` + */ + uint64_t escape; + }; + + /** + * Get a mask of both escape and escaped characters (the characters following a backslash). + * + * @param potential_escape A mask of the character that can escape others (but could be + * escaped itself). e.g. block.eq('\\') + */ + simdjson_really_inline escaped_and_escape next(uint64_t backslash) noexcept { + +#if !SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT + if (!backslash) { return {next_escaped_without_backslashes(), 0}; } +#endif + + // | | Mask (shows characters instead of 1's) | Depth | Instructions | + // |--------------------------------|----------------------------------------|-------|---------------------| + // | string | `\\n_\\\n___\\\n___\\\\___\\\\__\\\` | + // | | ` even odd even odd odd` | + // | potential_escape | ` \ \\\ \\\ \\\\ \\\\ \\\` | 1 | 1 (backslash & ~first_is_escaped) + // | escape_and_terminal_code | ` \n \ \n \ \n \ \ \ \ \ \` | 5 | 5 (next_escape_and_terminal_code()) + // | escaped | `\ \ n \ n \ \ \ \ \ ` X | 6 | 7 (escape_and_terminal_code ^ (potential_escape | first_is_escaped)) + // | escape | ` \ \ \ \ \ \ \ \ \ \` | 6 | 8 (escape_and_terminal_code & backslash) + // | first_is_escaped | `\ ` | 7 (*) | 9 (escape >> 63) () + // (*) this is not needed until the next iteration + uint64_t escape_and_terminal_code = next_escape_and_terminal_code(backslash & ~this->next_is_escaped); + uint64_t escaped = escape_and_terminal_code ^ (backslash | this->next_is_escaped); + uint64_t escape = escape_and_terminal_code & backslash; + this->next_is_escaped = escape >> 63; + return {.escaped=escaped, .escape=escape}; + } + +private: + static constexpr const uint64_t ODD_BITS = 0xAAAAAAAAAAAAAAAAULL; + + uint64_t next_escaped_without_backslashes() noexcept { + uint64_t escaped = this->next_is_escaped; + this->next_is_escaped = 0; + return escaped; + } + + /** + * Returns a mask of the next escape characters (masking out escaped backslashes), along with + * any non-backslash escape codes. + * + * \n \\n \\\n \\\\n returns: + * \n \ \ \n \ \ + * 11 100 1011 10100 + * + * You are expected to mask out the first bit yourself if the previous block had a trailing + * escape. + * + * & the result with potential_escape to get just the escape characters. + * ^ the result with (potential_escape | first_is_escaped) to get escaped characters. + */ + static simdjson_inline uint64_t next_escape_and_terminal_code(uint64_t potential_escape) noexcept { + // If we were to just shift and mask out any odd bits, we'd actually get a *half* right answer: + // any even-aligned backslash runs would be correct! Odd-aligned backslash runs would be + // inverted (\\\ would be 010 instead of 101). + // + // ``` + // string: | ____\\\\_\\\\_____ | + // maybe_escaped | ODD | \ \ \ \ | + // even-aligned ^^^ ^^^^ odd-aligned + // ``` + // + // Taking that into account, our basic strategy is: + // + // 1. Use subtraction to produce a mask with 1's for even-aligned runs and 0's for + // odd-aligned runs. + // 2. XOR all odd bits, which masks out the odd bits in even-aligned runs, and brings IN the + // odd bits in odd-aligned runs. + // 3. & with backslash to clean up any stray bits. + // runs are set to 0, and then XORing with "odd": + // + // | | Mask (shows characters instead of 1's) | Instructions | + // |--------------------------------|----------------------------------------|---------------------| + // | string | `\\n_\\\n___\\\n___\\\\___\\\\__\\\` | + // | | ` even odd even odd odd` | + // | maybe_escaped | ` n \\n \\n \\\_ \\\_ \\` X | 1 (potential_escape << 1) + // | maybe_escaped_and_odd | ` \n_ \\n _ \\\n_ _ \\\__ _\\\_ \\\` | 1 (maybe_escaped | odd) + // | even_series_codes_and_odd | ` n_\\\ _ n_ _\\\\ _ _ ` | 1 (maybe_escaped_and_odd - potential_escape) + // | escape_and_terminal_code | ` \n \ \n \ \n \ \ \ \ \ \` | 1 (^ odd) + // + + // Escaped characters are characters following an escape. + uint64_t maybe_escaped = potential_escape << 1; + + // To distinguish odd from even escape sequences, therefore, we turn on any *starting* + // escapes that are on an odd byte. (We actually bring in all odd bits, for speed.) + // - Odd runs of backslashes are 0000, and the code at the end ("n" in \n or \\n) is 1. + // - Odd runs of backslashes are 1111, and the code at the end ("n" in \n or \\n) is 0. + // - All other odd bytes are 1, and even bytes are 0. + uint64_t maybe_escaped_and_odd_bits = maybe_escaped | ODD_BITS; + uint64_t even_series_codes_and_odd_bits = maybe_escaped_and_odd_bits - potential_escape; + + // Now we flip all odd bytes back with xor. This: + // - Makes odd runs of backslashes go from 0000 to 1010 + // - Makes even runs of backslashes go from 1111 to 1010 + // - Sets actually-escaped codes to 1 (the n in \n and \\n: \n = 11, \\n = 100) + // - Resets all other bytes to 0 + return even_series_codes_and_odd_bits ^ ODD_BITS; + } +}; + +} // namespace stage1 +} // unnamed namespace +} // namespace ppc64 +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H +/* end file generic/stage1/json_escape_scanner.h for ppc64 */ /* including generic/stage1/json_string_scanner.h for ppc64: #include */ /* begin file generic/stage1/json_string_scanner.h for ppc64 */ #ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H @@ -33470,6 +34026,7 @@ using utf8_validation::utf8_checker; /* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ /* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H */ /* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ namespace simdjson { @@ -33479,19 +34036,13 @@ namespace stage1 { struct json_string_block { // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 - simdjson_inline json_string_block(uint64_t backslash, uint64_t escaped, uint64_t quote, uint64_t in_string) : - _backslash(backslash), _escaped(escaped), _quote(quote), _in_string(in_string) {} + simdjson_inline json_string_block(uint64_t escaped, uint64_t quote, uint64_t in_string) : + _escaped(escaped), _quote(quote), _in_string(in_string) {} // Escaped characters (characters following an escape() character) simdjson_inline uint64_t escaped() const { return _escaped; } - // Escape characters (backslashes that are not escaped--i.e. in \\, includes only the first \) - simdjson_inline uint64_t escape() const { return _backslash & ~_escaped; } // Real (non-backslashed) quotes simdjson_inline uint64_t quote() const { return _quote; } - // Start quotes of strings - simdjson_inline uint64_t string_start() const { return _quote & _in_string; } - // End quotes of strings - simdjson_inline uint64_t string_end() const { return _quote & ~_in_string; } // Only characters inside the string (not including the quotes) simdjson_inline uint64_t string_content() const { return _in_string & ~_quote; } // Return a mask of whether the given characters are inside a string (only works on non-quotes) @@ -33501,11 +34052,9 @@ struct json_string_block { // Tail of string (everything except the start quote) simdjson_inline uint64_t string_tail() const { return _in_string ^ _quote; } - // backslash characters - uint64_t _backslash; // escaped characters (backslashed--does not include the hex characters after \u) uint64_t _escaped; - // real quotes (non-backslashed ones) + // real quotes (non-escaped ones) uint64_t _quote; // string characters (includes start quote but not end quote) uint64_t _in_string; @@ -33519,60 +34068,12 @@ public: simdjson_inline error_code finish(); private: - // Intended to be defined by the implementation - simdjson_inline uint64_t find_escaped(uint64_t escape); - simdjson_inline uint64_t find_escaped_branchless(uint64_t escape); - + // Scans for escape characters + json_escape_scanner escape_scanner; // Whether the last iteration was still inside a string (all 1's = true, all 0's = false). uint64_t prev_in_string = 0ULL; - // Whether the first character of the next iteration is escaped. - uint64_t prev_escaped = 0ULL; }; -// -// Finds escaped characters (characters following \). -// -// Handles runs of backslashes like \\\" and \\\\" correctly (yielding 0101 and 01010, respectively). -// -// Does this by: -// - Shift the escape mask to get potentially escaped characters (characters after backslashes). -// - Mask escaped sequences that start on *even* bits with 1010101010 (odd bits are escaped, even bits are not) -// - Mask escaped sequences that start on *odd* bits with 0101010101 (even bits are escaped, odd bits are not) -// -// To distinguish between escaped sequences starting on even/odd bits, it finds the start of all -// escape sequences, filters out the ones that start on even bits, and adds that to the mask of -// escape sequences. This causes the addition to clear out the sequences starting on odd bits (since -// the start bit causes a carry), and leaves even-bit sequences alone. -// -// Example: -// -// text | \\\ | \\\"\\\" \\\" \\"\\" | -// escape | xxx | xx xxx xxx xx xx | Removed overflow backslash; will | it into follows_escape -// odd_starts | x | x x x | escape & ~even_bits & ~follows_escape -// even_seq | c| cxxx c xx c | c = carry bit -- will be masked out later -// invert_mask | | cxxx c xx c| even_seq << 1 -// follows_escape | xx | x xx xxx xxx xx xx | Includes overflow bit -// escaped | x | x x x x x x x x | -// desired | x | x x x x x x x x | -// text | \\\ | \\\"\\\" \\\" \\"\\" | -// -simdjson_inline uint64_t json_string_scanner::find_escaped_branchless(uint64_t backslash) { - // If there was overflow, pretend the first character isn't a backslash - backslash &= ~prev_escaped; - uint64_t follows_escape = backslash << 1 | prev_escaped; - - // Get sequences starting on even bits by clearing out the odd series using + - const uint64_t even_bits = 0x5555555555555555ULL; - uint64_t odd_sequence_starts = backslash & ~even_bits & ~follows_escape; - uint64_t sequences_starting_on_even_bits; - prev_escaped = add_overflow(odd_sequence_starts, backslash, &sequences_starting_on_even_bits); - uint64_t invert_mask = sequences_starting_on_even_bits << 1; // The mask we want to return is the *escaped* bits, not escapes. - - // Mask every other backslashed character as an escaped character - // Flip the mask for sequences that start on even bits, to correct them - return (even_bits ^ invert_mask) & follows_escape; -} - // // Return a mask of all string characters plus end quotes. // @@ -33583,7 +34084,7 @@ simdjson_inline uint64_t json_string_scanner::find_escaped_branchless(uint64_t b // simdjson_inline json_string_block json_string_scanner::next(const simd::simd8x64& in) { const uint64_t backslash = in.eq('\\'); - const uint64_t escaped = find_escaped(backslash); + const uint64_t escaped = escape_scanner.next(backslash).escaped; const uint64_t quote = in.eq('"') & ~escaped; // @@ -33597,21 +34098,13 @@ simdjson_inline json_string_block json_string_scanner::next(const simd::simd8x64 // // Check if we're still in a string at the end of the box so the next block will know // - // right shift of a signed value expected to be well-defined and standard - // compliant as of C++20, John Regher from Utah U. says this is fine code - // prev_in_string = uint64_t(static_cast(in_string) >> 63); // Use ^ to turn the beginning quote off, and the end quote on. // We are returning a function-local object so either we get a move constructor // or we get copy elision. - return json_string_block( - backslash, - escaped, - quote, - in_string - ); + return json_string_block(escaped, quote, in_string); } simdjson_inline error_code json_string_scanner::finish() { @@ -34021,114 +34514,6 @@ simdjson_inline error_code json_scanner::finish() { /* end file generic/stage1/json_scanner.h for ppc64 */ // All other declarations -/* including generic/stage1/buf_block_reader.h for ppc64: #include */ -/* begin file generic/stage1/buf_block_reader.h for ppc64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace ppc64 { -namespace { -namespace stage1 { - -// Walks through a buffer in block-sized increments, loading the last part with spaces -template -struct buf_block_reader { -public: - simdjson_inline buf_block_reader(const uint8_t *_buf, size_t _len); - simdjson_inline size_t block_index(); - simdjson_inline bool has_full_block() const; - simdjson_inline const uint8_t *full_block() const; - /** - * Get the last block, padded with spaces. - * - * There will always be a last block, with at least 1 byte, unless len == 0 (in which case this - * function fills the buffer with spaces and returns 0. In particular, if len == STEP_SIZE there - * will be 0 full_blocks and 1 remainder block with STEP_SIZE bytes and no spaces for padding. - * - * @return the number of effective characters in the last block. - */ - simdjson_inline size_t get_remainder(uint8_t *dst) const; - simdjson_inline void advance(); -private: - const uint8_t *buf; - const size_t len; - const size_t lenminusstep; - size_t idx; -}; - -// Routines to print masks and text for debugging bitmask operations -simdjson_unused static char * format_input_text_64(const uint8_t *text) { - static char buf[sizeof(simd8x64) + 1]; - for (size_t i=0; i); i++) { - buf[i] = int8_t(text[i]) < ' ' ? '_' : int8_t(text[i]); - } - buf[sizeof(simd8x64)] = '\0'; - return buf; -} - -// Routines to print masks and text for debugging bitmask operations -simdjson_unused static char * format_input_text(const simd8x64& in) { - static char buf[sizeof(simd8x64) + 1]; - in.store(reinterpret_cast(buf)); - for (size_t i=0; i); i++) { - if (buf[i] < ' ') { buf[i] = '_'; } - } - buf[sizeof(simd8x64)] = '\0'; - return buf; -} - -simdjson_unused static char * format_mask(uint64_t mask) { - static char buf[sizeof(simd8x64) + 1]; - for (size_t i=0; i<64; i++) { - buf[i] = (mask & (size_t(1) << i)) ? 'X' : ' '; - } - buf[64] = '\0'; - return buf; -} - -template -simdjson_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) : buf{_buf}, len{_len}, lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {} - -template -simdjson_inline size_t buf_block_reader::block_index() { return idx; } - -template -simdjson_inline bool buf_block_reader::has_full_block() const { - return idx < lenminusstep; -} - -template -simdjson_inline const uint8_t *buf_block_reader::full_block() const { - return &buf[idx]; -} - -template -simdjson_inline size_t buf_block_reader::get_remainder(uint8_t *dst) const { - if(len == idx) { return 0; } // memcpy(dst, null, 0) will trigger an error with some sanitizers - std::memset(dst, 0x20, STEP_SIZE); // std::memset STEP_SIZE because it's more efficient to write out 8 or 16 bytes at once. - std::memcpy(dst, buf + idx, len - idx); - return len - idx; -} - -template -simdjson_inline void buf_block_reader::advance() { - idx += STEP_SIZE; -} - -} // namespace stage1 -} // unnamed namespace -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H -/* end file generic/stage1/buf_block_reader.h for ppc64 */ /* including generic/stage1/find_next_document_index.h for ppc64: #include */ /* begin file generic/stage1/find_next_document_index.h for ppc64 */ #ifndef SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H @@ -36052,18 +36437,6 @@ simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, // namespace simdjson { namespace ppc64 { -namespace { -namespace stage1 { - -simdjson_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { - // On PPC, we don't short-circuit this if there are no backslashes, because the branch gives us no - // benefit and therefore makes things worse. - // if (!backslash) { uint64_t escaped = prev_escaped; prev_escaped = 0; return escaped; } - return find_escaped_branchless(backslash); -} - -} // namespace stage1 -} // unnamed namespace simdjson_warn_unused error_code implementation::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept { return ppc64::stage1::json_minifier::minify<64>(buf, len, dst, dst_len); @@ -36110,6 +36483,7 @@ simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t * /* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ +#undef SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT /* undefining SIMDJSON_IMPLEMENTATION from "ppc64" */ #undef SIMDJSON_IMPLEMENTATION /* end file simdjson/ppc64/end.h */ @@ -36263,7 +36637,7 @@ simdjson_inline int leading_zeroes(uint64_t input_num) { #if SIMDJSON_REGULAR_VISUAL_STUDIO simdjson_inline unsigned __int64 count_ones(uint64_t input_num) { - // note: we do not support legacy 32-bit Windows + // note: we do not support legacy 32-bit Windows in this kernel return __popcnt64(input_num);// Visual Studio wants two underscores } #else @@ -36850,7 +37224,7 @@ simdjson_inline int leading_zeroes(uint64_t input_num) { #if SIMDJSON_REGULAR_VISUAL_STUDIO simdjson_inline unsigned __int64 count_ones(uint64_t input_num) { - // note: we do not support legacy 32-bit Windows + // note: we do not support legacy 32-bit Windows in this kernel return __popcnt64(input_num);// Visual Studio wants two underscores } #else @@ -39281,7 +39655,7 @@ simdjson_inline int leading_zeroes(uint64_t input_num) { #if SIMDJSON_REGULAR_VISUAL_STUDIO simdjson_inline unsigned __int64 count_ones(uint64_t input_num) { - // note: we do not support legacy 32-bit Windows + // note: we do not support legacy 32-bit Windows in this kernel return __popcnt64(input_num);// Visual Studio wants two underscores } #else @@ -39868,7 +40242,7 @@ simdjson_inline int leading_zeroes(uint64_t input_num) { #if SIMDJSON_REGULAR_VISUAL_STUDIO simdjson_inline unsigned __int64 count_ones(uint64_t input_num) { - // note: we do not support legacy 32-bit Windows + // note: we do not support legacy 32-bit Windows in this kernel return __popcnt64(input_num);// Visual Studio wants two underscores } #else @@ -40403,6 +40777,279 @@ using utf8_validation::utf8_checker; #endif // SIMDJSON_SRC_GENERIC_STAGE1_BASE_H /* end file generic/stage1/base.h for westmere */ +/* including generic/stage1/buf_block_reader.h for westmere: #include */ +/* begin file generic/stage1/buf_block_reader.h for westmere */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include + +namespace simdjson { +namespace westmere { +namespace { +namespace stage1 { + +// Walks through a buffer in block-sized increments, loading the last part with spaces +template +struct buf_block_reader { +public: + simdjson_inline buf_block_reader(const uint8_t *_buf, size_t _len); + simdjson_inline size_t block_index(); + simdjson_inline bool has_full_block() const; + simdjson_inline const uint8_t *full_block() const; + /** + * Get the last block, padded with spaces. + * + * There will always be a last block, with at least 1 byte, unless len == 0 (in which case this + * function fills the buffer with spaces and returns 0. In particular, if len == STEP_SIZE there + * will be 0 full_blocks and 1 remainder block with STEP_SIZE bytes and no spaces for padding. + * + * @return the number of effective characters in the last block. + */ + simdjson_inline size_t get_remainder(uint8_t *dst) const; + simdjson_inline void advance(); +private: + const uint8_t *buf; + const size_t len; + const size_t lenminusstep; + size_t idx; +}; + +// Routines to print masks and text for debugging bitmask operations +simdjson_unused static char * format_input_text_64(const uint8_t *text) { + static char buf[sizeof(simd8x64) + 1]; + for (size_t i=0; i); i++) { + buf[i] = int8_t(text[i]) < ' ' ? '_' : int8_t(text[i]); + } + buf[sizeof(simd8x64)] = '\0'; + return buf; +} + +// Routines to print masks and text for debugging bitmask operations +simdjson_unused static char * format_input_text(const simd8x64& in) { + static char buf[sizeof(simd8x64) + 1]; + in.store(reinterpret_cast(buf)); + for (size_t i=0; i); i++) { + if (buf[i] < ' ') { buf[i] = '_'; } + } + buf[sizeof(simd8x64)] = '\0'; + return buf; +} + +simdjson_unused static char * format_input_text(const simd8x64& in, uint64_t mask) { + static char buf[sizeof(simd8x64) + 1]; + in.store(reinterpret_cast(buf)); + for (size_t i=0; i); i++) { + if (buf[i] <= ' ') { buf[i] = '_'; } + if (!(mask & (size_t(1) << i))) { buf[i] = ' '; } + } + buf[sizeof(simd8x64)] = '\0'; + return buf; +} + +simdjson_unused static char * format_mask(uint64_t mask) { + static char buf[sizeof(simd8x64) + 1]; + for (size_t i=0; i<64; i++) { + buf[i] = (mask & (size_t(1) << i)) ? 'X' : ' '; + } + buf[64] = '\0'; + return buf; +} + +template +simdjson_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) : buf{_buf}, len{_len}, lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {} + +template +simdjson_inline size_t buf_block_reader::block_index() { return idx; } + +template +simdjson_inline bool buf_block_reader::has_full_block() const { + return idx < lenminusstep; +} + +template +simdjson_inline const uint8_t *buf_block_reader::full_block() const { + return &buf[idx]; +} + +template +simdjson_inline size_t buf_block_reader::get_remainder(uint8_t *dst) const { + if(len == idx) { return 0; } // memcpy(dst, null, 0) will trigger an error with some sanitizers + std::memset(dst, 0x20, STEP_SIZE); // std::memset STEP_SIZE because it's more efficient to write out 8 or 16 bytes at once. + std::memcpy(dst, buf + idx, len - idx); + return len - idx; +} + +template +simdjson_inline void buf_block_reader::advance() { + idx += STEP_SIZE; +} + +} // namespace stage1 +} // unnamed namespace +} // namespace westmere +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H +/* end file generic/stage1/buf_block_reader.h for westmere */ +/* including generic/stage1/json_escape_scanner.h for westmere: #include */ +/* begin file generic/stage1/json_escape_scanner.h for westmere */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_ESCAPE_SCANNER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_ESCAPE_SCANNER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { +namespace westmere { +namespace { +namespace stage1 { + +/** + * Scans for escape characters in JSON, taking care with multiple backslashes (\\n vs. \n). + */ +struct json_escape_scanner { + /** The actual escape characters (the backslashes themselves). */ + uint64_t next_is_escaped; + + struct escaped_and_escape { + /** + * Mask of escaped characters. + * + * ``` + * \n \\n \\\n \\\\n \ + * 0100100010100101000 + * n \ \ n \ \ + * ``` + */ + uint64_t escaped; + /** + * Mask of escape characters. + * + * ``` + * \n \\n \\\n \\\\n \ + * 1001000101001010001 + * \ \ \ \ \ \ \ + * ``` + */ + uint64_t escape; + }; + + /** + * Get a mask of both escape and escaped characters (the characters following a backslash). + * + * @param potential_escape A mask of the character that can escape others (but could be + * escaped itself). e.g. block.eq('\\') + */ + simdjson_really_inline escaped_and_escape next(uint64_t backslash) noexcept { + +#if !SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT + if (!backslash) { return {next_escaped_without_backslashes(), 0}; } +#endif + + // | | Mask (shows characters instead of 1's) | Depth | Instructions | + // |--------------------------------|----------------------------------------|-------|---------------------| + // | string | `\\n_\\\n___\\\n___\\\\___\\\\__\\\` | + // | | ` even odd even odd odd` | + // | potential_escape | ` \ \\\ \\\ \\\\ \\\\ \\\` | 1 | 1 (backslash & ~first_is_escaped) + // | escape_and_terminal_code | ` \n \ \n \ \n \ \ \ \ \ \` | 5 | 5 (next_escape_and_terminal_code()) + // | escaped | `\ \ n \ n \ \ \ \ \ ` X | 6 | 7 (escape_and_terminal_code ^ (potential_escape | first_is_escaped)) + // | escape | ` \ \ \ \ \ \ \ \ \ \` | 6 | 8 (escape_and_terminal_code & backslash) + // | first_is_escaped | `\ ` | 7 (*) | 9 (escape >> 63) () + // (*) this is not needed until the next iteration + uint64_t escape_and_terminal_code = next_escape_and_terminal_code(backslash & ~this->next_is_escaped); + uint64_t escaped = escape_and_terminal_code ^ (backslash | this->next_is_escaped); + uint64_t escape = escape_and_terminal_code & backslash; + this->next_is_escaped = escape >> 63; + return {.escaped=escaped, .escape=escape}; + } + +private: + static constexpr const uint64_t ODD_BITS = 0xAAAAAAAAAAAAAAAAULL; + + uint64_t next_escaped_without_backslashes() noexcept { + uint64_t escaped = this->next_is_escaped; + this->next_is_escaped = 0; + return escaped; + } + + /** + * Returns a mask of the next escape characters (masking out escaped backslashes), along with + * any non-backslash escape codes. + * + * \n \\n \\\n \\\\n returns: + * \n \ \ \n \ \ + * 11 100 1011 10100 + * + * You are expected to mask out the first bit yourself if the previous block had a trailing + * escape. + * + * & the result with potential_escape to get just the escape characters. + * ^ the result with (potential_escape | first_is_escaped) to get escaped characters. + */ + static simdjson_inline uint64_t next_escape_and_terminal_code(uint64_t potential_escape) noexcept { + // If we were to just shift and mask out any odd bits, we'd actually get a *half* right answer: + // any even-aligned backslash runs would be correct! Odd-aligned backslash runs would be + // inverted (\\\ would be 010 instead of 101). + // + // ``` + // string: | ____\\\\_\\\\_____ | + // maybe_escaped | ODD | \ \ \ \ | + // even-aligned ^^^ ^^^^ odd-aligned + // ``` + // + // Taking that into account, our basic strategy is: + // + // 1. Use subtraction to produce a mask with 1's for even-aligned runs and 0's for + // odd-aligned runs. + // 2. XOR all odd bits, which masks out the odd bits in even-aligned runs, and brings IN the + // odd bits in odd-aligned runs. + // 3. & with backslash to clean up any stray bits. + // runs are set to 0, and then XORing with "odd": + // + // | | Mask (shows characters instead of 1's) | Instructions | + // |--------------------------------|----------------------------------------|---------------------| + // | string | `\\n_\\\n___\\\n___\\\\___\\\\__\\\` | + // | | ` even odd even odd odd` | + // | maybe_escaped | ` n \\n \\n \\\_ \\\_ \\` X | 1 (potential_escape << 1) + // | maybe_escaped_and_odd | ` \n_ \\n _ \\\n_ _ \\\__ _\\\_ \\\` | 1 (maybe_escaped | odd) + // | even_series_codes_and_odd | ` n_\\\ _ n_ _\\\\ _ _ ` | 1 (maybe_escaped_and_odd - potential_escape) + // | escape_and_terminal_code | ` \n \ \n \ \n \ \ \ \ \ \` | 1 (^ odd) + // + + // Escaped characters are characters following an escape. + uint64_t maybe_escaped = potential_escape << 1; + + // To distinguish odd from even escape sequences, therefore, we turn on any *starting* + // escapes that are on an odd byte. (We actually bring in all odd bits, for speed.) + // - Odd runs of backslashes are 0000, and the code at the end ("n" in \n or \\n) is 1. + // - Odd runs of backslashes are 1111, and the code at the end ("n" in \n or \\n) is 0. + // - All other odd bytes are 1, and even bytes are 0. + uint64_t maybe_escaped_and_odd_bits = maybe_escaped | ODD_BITS; + uint64_t even_series_codes_and_odd_bits = maybe_escaped_and_odd_bits - potential_escape; + + // Now we flip all odd bytes back with xor. This: + // - Makes odd runs of backslashes go from 0000 to 1010 + // - Makes even runs of backslashes go from 1111 to 1010 + // - Sets actually-escaped codes to 1 (the n in \n and \\n: \n = 11, \\n = 100) + // - Resets all other bytes to 0 + return even_series_codes_and_odd_bits ^ ODD_BITS; + } +}; + +} // namespace stage1 +} // unnamed namespace +} // namespace westmere +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H +/* end file generic/stage1/json_escape_scanner.h for westmere */ /* including generic/stage1/json_string_scanner.h for westmere: #include */ /* begin file generic/stage1/json_string_scanner.h for westmere */ #ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H @@ -40410,6 +41057,7 @@ using utf8_validation::utf8_checker; /* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ /* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H */ /* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ namespace simdjson { @@ -40419,19 +41067,13 @@ namespace stage1 { struct json_string_block { // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 - simdjson_inline json_string_block(uint64_t backslash, uint64_t escaped, uint64_t quote, uint64_t in_string) : - _backslash(backslash), _escaped(escaped), _quote(quote), _in_string(in_string) {} + simdjson_inline json_string_block(uint64_t escaped, uint64_t quote, uint64_t in_string) : + _escaped(escaped), _quote(quote), _in_string(in_string) {} // Escaped characters (characters following an escape() character) simdjson_inline uint64_t escaped() const { return _escaped; } - // Escape characters (backslashes that are not escaped--i.e. in \\, includes only the first \) - simdjson_inline uint64_t escape() const { return _backslash & ~_escaped; } // Real (non-backslashed) quotes simdjson_inline uint64_t quote() const { return _quote; } - // Start quotes of strings - simdjson_inline uint64_t string_start() const { return _quote & _in_string; } - // End quotes of strings - simdjson_inline uint64_t string_end() const { return _quote & ~_in_string; } // Only characters inside the string (not including the quotes) simdjson_inline uint64_t string_content() const { return _in_string & ~_quote; } // Return a mask of whether the given characters are inside a string (only works on non-quotes) @@ -40441,11 +41083,9 @@ struct json_string_block { // Tail of string (everything except the start quote) simdjson_inline uint64_t string_tail() const { return _in_string ^ _quote; } - // backslash characters - uint64_t _backslash; // escaped characters (backslashed--does not include the hex characters after \u) uint64_t _escaped; - // real quotes (non-backslashed ones) + // real quotes (non-escaped ones) uint64_t _quote; // string characters (includes start quote but not end quote) uint64_t _in_string; @@ -40459,60 +41099,12 @@ public: simdjson_inline error_code finish(); private: - // Intended to be defined by the implementation - simdjson_inline uint64_t find_escaped(uint64_t escape); - simdjson_inline uint64_t find_escaped_branchless(uint64_t escape); - + // Scans for escape characters + json_escape_scanner escape_scanner; // Whether the last iteration was still inside a string (all 1's = true, all 0's = false). uint64_t prev_in_string = 0ULL; - // Whether the first character of the next iteration is escaped. - uint64_t prev_escaped = 0ULL; }; -// -// Finds escaped characters (characters following \). -// -// Handles runs of backslashes like \\\" and \\\\" correctly (yielding 0101 and 01010, respectively). -// -// Does this by: -// - Shift the escape mask to get potentially escaped characters (characters after backslashes). -// - Mask escaped sequences that start on *even* bits with 1010101010 (odd bits are escaped, even bits are not) -// - Mask escaped sequences that start on *odd* bits with 0101010101 (even bits are escaped, odd bits are not) -// -// To distinguish between escaped sequences starting on even/odd bits, it finds the start of all -// escape sequences, filters out the ones that start on even bits, and adds that to the mask of -// escape sequences. This causes the addition to clear out the sequences starting on odd bits (since -// the start bit causes a carry), and leaves even-bit sequences alone. -// -// Example: -// -// text | \\\ | \\\"\\\" \\\" \\"\\" | -// escape | xxx | xx xxx xxx xx xx | Removed overflow backslash; will | it into follows_escape -// odd_starts | x | x x x | escape & ~even_bits & ~follows_escape -// even_seq | c| cxxx c xx c | c = carry bit -- will be masked out later -// invert_mask | | cxxx c xx c| even_seq << 1 -// follows_escape | xx | x xx xxx xxx xx xx | Includes overflow bit -// escaped | x | x x x x x x x x | -// desired | x | x x x x x x x x | -// text | \\\ | \\\"\\\" \\\" \\"\\" | -// -simdjson_inline uint64_t json_string_scanner::find_escaped_branchless(uint64_t backslash) { - // If there was overflow, pretend the first character isn't a backslash - backslash &= ~prev_escaped; - uint64_t follows_escape = backslash << 1 | prev_escaped; - - // Get sequences starting on even bits by clearing out the odd series using + - const uint64_t even_bits = 0x5555555555555555ULL; - uint64_t odd_sequence_starts = backslash & ~even_bits & ~follows_escape; - uint64_t sequences_starting_on_even_bits; - prev_escaped = add_overflow(odd_sequence_starts, backslash, &sequences_starting_on_even_bits); - uint64_t invert_mask = sequences_starting_on_even_bits << 1; // The mask we want to return is the *escaped* bits, not escapes. - - // Mask every other backslashed character as an escaped character - // Flip the mask for sequences that start on even bits, to correct them - return (even_bits ^ invert_mask) & follows_escape; -} - // // Return a mask of all string characters plus end quotes. // @@ -40523,7 +41115,7 @@ simdjson_inline uint64_t json_string_scanner::find_escaped_branchless(uint64_t b // simdjson_inline json_string_block json_string_scanner::next(const simd::simd8x64& in) { const uint64_t backslash = in.eq('\\'); - const uint64_t escaped = find_escaped(backslash); + const uint64_t escaped = escape_scanner.next(backslash).escaped; const uint64_t quote = in.eq('"') & ~escaped; // @@ -40537,21 +41129,13 @@ simdjson_inline json_string_block json_string_scanner::next(const simd::simd8x64 // // Check if we're still in a string at the end of the box so the next block will know // - // right shift of a signed value expected to be well-defined and standard - // compliant as of C++20, John Regher from Utah U. says this is fine code - // prev_in_string = uint64_t(static_cast(in_string) >> 63); // Use ^ to turn the beginning quote off, and the end quote on. // We are returning a function-local object so either we get a move constructor // or we get copy elision. - return json_string_block( - backslash, - escaped, - quote, - in_string - ); + return json_string_block(escaped, quote, in_string); } simdjson_inline error_code json_string_scanner::finish() { @@ -40961,114 +41545,6 @@ simdjson_inline error_code json_scanner::finish() { /* end file generic/stage1/json_scanner.h for westmere */ // All other declarations -/* including generic/stage1/buf_block_reader.h for westmere: #include */ -/* begin file generic/stage1/buf_block_reader.h for westmere */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace westmere { -namespace { -namespace stage1 { - -// Walks through a buffer in block-sized increments, loading the last part with spaces -template -struct buf_block_reader { -public: - simdjson_inline buf_block_reader(const uint8_t *_buf, size_t _len); - simdjson_inline size_t block_index(); - simdjson_inline bool has_full_block() const; - simdjson_inline const uint8_t *full_block() const; - /** - * Get the last block, padded with spaces. - * - * There will always be a last block, with at least 1 byte, unless len == 0 (in which case this - * function fills the buffer with spaces and returns 0. In particular, if len == STEP_SIZE there - * will be 0 full_blocks and 1 remainder block with STEP_SIZE bytes and no spaces for padding. - * - * @return the number of effective characters in the last block. - */ - simdjson_inline size_t get_remainder(uint8_t *dst) const; - simdjson_inline void advance(); -private: - const uint8_t *buf; - const size_t len; - const size_t lenminusstep; - size_t idx; -}; - -// Routines to print masks and text for debugging bitmask operations -simdjson_unused static char * format_input_text_64(const uint8_t *text) { - static char buf[sizeof(simd8x64) + 1]; - for (size_t i=0; i); i++) { - buf[i] = int8_t(text[i]) < ' ' ? '_' : int8_t(text[i]); - } - buf[sizeof(simd8x64)] = '\0'; - return buf; -} - -// Routines to print masks and text for debugging bitmask operations -simdjson_unused static char * format_input_text(const simd8x64& in) { - static char buf[sizeof(simd8x64) + 1]; - in.store(reinterpret_cast(buf)); - for (size_t i=0; i); i++) { - if (buf[i] < ' ') { buf[i] = '_'; } - } - buf[sizeof(simd8x64)] = '\0'; - return buf; -} - -simdjson_unused static char * format_mask(uint64_t mask) { - static char buf[sizeof(simd8x64) + 1]; - for (size_t i=0; i<64; i++) { - buf[i] = (mask & (size_t(1) << i)) ? 'X' : ' '; - } - buf[64] = '\0'; - return buf; -} - -template -simdjson_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) : buf{_buf}, len{_len}, lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {} - -template -simdjson_inline size_t buf_block_reader::block_index() { return idx; } - -template -simdjson_inline bool buf_block_reader::has_full_block() const { - return idx < lenminusstep; -} - -template -simdjson_inline const uint8_t *buf_block_reader::full_block() const { - return &buf[idx]; -} - -template -simdjson_inline size_t buf_block_reader::get_remainder(uint8_t *dst) const { - if(len == idx) { return 0; } // memcpy(dst, null, 0) will trigger an error with some sanitizers - std::memset(dst, 0x20, STEP_SIZE); // std::memset STEP_SIZE because it's more efficient to write out 8 or 16 bytes at once. - std::memcpy(dst, buf + idx, len - idx); - return len - idx; -} - -template -simdjson_inline void buf_block_reader::advance() { - idx += STEP_SIZE; -} - -} // namespace stage1 -} // unnamed namespace -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H -/* end file generic/stage1/buf_block_reader.h for westmere */ /* including generic/stage1/find_next_document_index.h for westmere: #include */ /* begin file generic/stage1/find_next_document_index.h for westmere */ #ifndef SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H @@ -43024,16 +43500,6 @@ simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, namespace simdjson { namespace westmere { -namespace { -namespace stage1 { - -simdjson_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { - if (!backslash) { uint64_t escaped = prev_escaped; prev_escaped = 0; return escaped; } - return find_escaped_branchless(backslash); -} - -} // namespace stage1 -} // unnamed namespace simdjson_warn_unused error_code implementation::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept { return westmere::stage1::json_minifier::minify<64>(buf, len, dst, dst_len); diff --git a/singleheader/simdjson.h b/singleheader/simdjson.h index f5b2d5fca..645f5e603 100644 --- a/singleheader/simdjson.h +++ b/singleheader/simdjson.h @@ -1,4 +1,4 @@ -/* auto-generated on 2023-07-20 14:14:44 -0700. Do not edit! */ +/* auto-generated on 2023-07-21 12:09:57 -0700. Do not edit! */ /* including simdjson.h: */ /* begin file simdjson.h */ #ifndef SIMDJSON_H @@ -128,8 +128,6 @@ #else #define SIMDJSON_IS_32BITS 1 -// We do not support 32-bit platforms, but it can be -// handy to identify them. #if defined(_M_IX86) || defined(__i386__) #define SIMDJSON_IS_X86_32BITS 1 #elif defined(__arm__) || defined(_M_ARM) @@ -11151,6 +11149,8 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin #endif // SIMDJSON_ARM64_STRINGPARSING_DEFS_H /* end file simdjson/arm64/stringparsing_defs.h */ + +#define SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT 1 /* end file simdjson/arm64/begin.h */ /* including simdjson/generic/amalgamated.h for arm64: #include "simdjson/generic/amalgamated.h" */ /* begin file simdjson/generic/amalgamated.h for arm64 */ @@ -12995,6 +12995,7 @@ simdjson_inline implementation_simdjson_result_base::implementation_simdjson_ /* amalgamation skipped (editor-only): #include "simdjson/arm64/base.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ +#undef SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT /* undefining SIMDJSON_IMPLEMENTATION from "arm64" */ #undef SIMDJSON_IMPLEMENTATION /* end file simdjson/arm64/end.h */ @@ -15205,7 +15206,7 @@ simdjson_inline int leading_zeroes(uint64_t input_num) { #if SIMDJSON_REGULAR_VISUAL_STUDIO simdjson_inline unsigned __int64 count_ones(uint64_t input_num) { - // note: we do not support legacy 32-bit Windows + // note: we do not support legacy 32-bit Windows in this kernel return __popcnt64(input_num);// Visual Studio wants two underscores } #else @@ -20275,7 +20276,7 @@ simdjson_inline int leading_zeroes(uint64_t input_num) { #if SIMDJSON_REGULAR_VISUAL_STUDIO simdjson_inline int count_ones(uint64_t input_num) { - // note: we do not support legacy 32-bit Windows + // note: we do not support legacy 32-bit Windows in this kernel return __popcnt64(input_num); // Visual Studio wants two underscores } #else @@ -20961,6 +20962,8 @@ backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { #endif // SIMDJSON_PPC64_STRINGPARSING_DEFS_H /* end file simdjson/ppc64/stringparsing_defs.h */ + +#define SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT 1 /* end file simdjson/ppc64/begin.h */ /* including simdjson/generic/amalgamated.h for ppc64: #include "simdjson/generic/amalgamated.h" */ /* begin file simdjson/generic/amalgamated.h for ppc64 */ @@ -22805,6 +22808,7 @@ simdjson_inline implementation_simdjson_result_base::implementation_simdjson_ /* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ +#undef SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT /* undefining SIMDJSON_IMPLEMENTATION from "ppc64" */ #undef SIMDJSON_IMPLEMENTATION /* end file simdjson/ppc64/end.h */ @@ -22948,7 +22952,7 @@ simdjson_inline int leading_zeroes(uint64_t input_num) { #if SIMDJSON_REGULAR_VISUAL_STUDIO simdjson_inline unsigned __int64 count_ones(uint64_t input_num) { - // note: we do not support legacy 32-bit Windows + // note: we do not support legacy 32-bit Windows in this kernel return __popcnt64(input_num);// Visual Studio wants two underscores } #else @@ -23535,7 +23539,7 @@ simdjson_inline int leading_zeroes(uint64_t input_num) { #if SIMDJSON_REGULAR_VISUAL_STUDIO simdjson_inline unsigned __int64 count_ones(uint64_t input_num) { - // note: we do not support legacy 32-bit Windows + // note: we do not support legacy 32-bit Windows in this kernel return __popcnt64(input_num);// Visual Studio wants two underscores } #else @@ -26697,6 +26701,8 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin #endif // SIMDJSON_ARM64_STRINGPARSING_DEFS_H /* end file simdjson/arm64/stringparsing_defs.h */ + +#define SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT 1 /* end file simdjson/arm64/begin.h */ /* including simdjson/generic/ondemand/amalgamated.h for arm64: #include "simdjson/generic/ondemand/amalgamated.h" */ /* begin file simdjson/generic/ondemand/amalgamated.h for arm64 */ @@ -33116,7 +33122,7 @@ simdjson_inline const uint8_t *json_iterator::return_current_and_advance() noexc simdjson_inline const uint8_t *json_iterator::unsafe_pointer() const noexcept { // deliberately done without safety guard: - return token.peek(0); + return token.peek(); } simdjson_inline const uint8_t *json_iterator::peek(int32_t delta) const noexcept { @@ -33719,7 +33725,7 @@ simdjson_inline simdjson_result object::raw_json() noexcept { const uint8_t * starting_point{iter.peek_start()}; auto error = consume(); if(error) { return error; } - const uint8_t * final_point{iter._json_iter->peek(0)}; + const uint8_t * final_point{iter._json_iter->peek()}; return std::string_view(reinterpret_cast(starting_point), size_t(final_point - starting_point)); } @@ -36227,6 +36233,7 @@ simdjson_inline simdjson_result::simdjson_resul /* amalgamation skipped (editor-only): #include "simdjson/arm64/base.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ +#undef SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT /* undefining SIMDJSON_IMPLEMENTATION from "arm64" */ #undef SIMDJSON_IMPLEMENTATION /* end file simdjson/arm64/end.h */ @@ -42857,7 +42864,7 @@ simdjson_inline const uint8_t *json_iterator::return_current_and_advance() noexc simdjson_inline const uint8_t *json_iterator::unsafe_pointer() const noexcept { // deliberately done without safety guard: - return token.peek(0); + return token.peek(); } simdjson_inline const uint8_t *json_iterator::peek(int32_t delta) const noexcept { @@ -43460,7 +43467,7 @@ simdjson_inline simdjson_result object::raw_json() noexcept { const uint8_t * starting_point{iter.peek_start()}; auto error = consume(); if(error) { return error; } - const uint8_t * final_point{iter._json_iter->peek(0)}; + const uint8_t * final_point{iter._json_iter->peek()}; return std::string_view(reinterpret_cast(starting_point), size_t(final_point - starting_point)); } @@ -46123,7 +46130,7 @@ simdjson_inline int leading_zeroes(uint64_t input_num) { #if SIMDJSON_REGULAR_VISUAL_STUDIO simdjson_inline unsigned __int64 count_ones(uint64_t input_num) { - // note: we do not support legacy 32-bit Windows + // note: we do not support legacy 32-bit Windows in this kernel return __popcnt64(input_num);// Visual Studio wants two underscores } #else @@ -53091,7 +53098,7 @@ simdjson_inline const uint8_t *json_iterator::return_current_and_advance() noexc simdjson_inline const uint8_t *json_iterator::unsafe_pointer() const noexcept { // deliberately done without safety guard: - return token.peek(0); + return token.peek(); } simdjson_inline const uint8_t *json_iterator::peek(int32_t delta) const noexcept { @@ -53694,7 +53701,7 @@ simdjson_inline simdjson_result object::raw_json() noexcept { const uint8_t * starting_point{iter.peek_start()}; auto error = consume(); if(error) { return error; } - const uint8_t * final_point{iter._json_iter->peek(0)}; + const uint8_t * final_point{iter._json_iter->peek()}; return std::string_view(reinterpret_cast(starting_point), size_t(final_point - starting_point)); } @@ -63324,7 +63331,7 @@ simdjson_inline const uint8_t *json_iterator::return_current_and_advance() noexc simdjson_inline const uint8_t *json_iterator::unsafe_pointer() const noexcept { // deliberately done without safety guard: - return token.peek(0); + return token.peek(); } simdjson_inline const uint8_t *json_iterator::peek(int32_t delta) const noexcept { @@ -63927,7 +63934,7 @@ simdjson_inline simdjson_result object::raw_json() noexcept { const uint8_t * starting_point{iter.peek_start()}; auto error = consume(); if(error) { return error; } - const uint8_t * final_point{iter._json_iter->peek(0)}; + const uint8_t * final_point{iter._json_iter->peek()}; return std::string_view(reinterpret_cast(starting_point), size_t(final_point - starting_point)); } @@ -66565,7 +66572,7 @@ simdjson_inline int leading_zeroes(uint64_t input_num) { #if SIMDJSON_REGULAR_VISUAL_STUDIO simdjson_inline int count_ones(uint64_t input_num) { - // note: we do not support legacy 32-bit Windows + // note: we do not support legacy 32-bit Windows in this kernel return __popcnt64(input_num); // Visual Studio wants two underscores } #else @@ -67251,6 +67258,8 @@ backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { #endif // SIMDJSON_PPC64_STRINGPARSING_DEFS_H /* end file simdjson/ppc64/stringparsing_defs.h */ + +#define SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT 1 /* end file simdjson/ppc64/begin.h */ /* including simdjson/generic/ondemand/amalgamated.h for ppc64: #include "simdjson/generic/ondemand/amalgamated.h" */ /* begin file simdjson/generic/ondemand/amalgamated.h for ppc64 */ @@ -73670,7 +73679,7 @@ simdjson_inline const uint8_t *json_iterator::return_current_and_advance() noexc simdjson_inline const uint8_t *json_iterator::unsafe_pointer() const noexcept { // deliberately done without safety guard: - return token.peek(0); + return token.peek(); } simdjson_inline const uint8_t *json_iterator::peek(int32_t delta) const noexcept { @@ -74273,7 +74282,7 @@ simdjson_inline simdjson_result object::raw_json() noexcept { const uint8_t * starting_point{iter.peek_start()}; auto error = consume(); if(error) { return error; } - const uint8_t * final_point{iter._json_iter->peek(0)}; + const uint8_t * final_point{iter._json_iter->peek()}; return std::string_view(reinterpret_cast(starting_point), size_t(final_point - starting_point)); } @@ -76781,6 +76790,7 @@ simdjson_inline simdjson_result::simdjson_resul /* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ +#undef SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT /* undefining SIMDJSON_IMPLEMENTATION from "ppc64" */ #undef SIMDJSON_IMPLEMENTATION /* end file simdjson/ppc64/end.h */ @@ -76924,7 +76934,7 @@ simdjson_inline int leading_zeroes(uint64_t input_num) { #if SIMDJSON_REGULAR_VISUAL_STUDIO simdjson_inline unsigned __int64 count_ones(uint64_t input_num) { - // note: we do not support legacy 32-bit Windows + // note: we do not support legacy 32-bit Windows in this kernel return __popcnt64(input_num);// Visual Studio wants two underscores } #else @@ -77511,7 +77521,7 @@ simdjson_inline int leading_zeroes(uint64_t input_num) { #if SIMDJSON_REGULAR_VISUAL_STUDIO simdjson_inline unsigned __int64 count_ones(uint64_t input_num) { - // note: we do not support legacy 32-bit Windows + // note: we do not support legacy 32-bit Windows in this kernel return __popcnt64(input_num);// Visual Studio wants two underscores } #else @@ -84340,7 +84350,7 @@ simdjson_inline const uint8_t *json_iterator::return_current_and_advance() noexc simdjson_inline const uint8_t *json_iterator::unsafe_pointer() const noexcept { // deliberately done without safety guard: - return token.peek(0); + return token.peek(); } simdjson_inline const uint8_t *json_iterator::peek(int32_t delta) const noexcept { @@ -84943,7 +84953,7 @@ simdjson_inline simdjson_result object::raw_json() noexcept { const uint8_t * starting_point{iter.peek_start()}; auto error = consume(); if(error) { return error; } - const uint8_t * final_point{iter._json_iter->peek(0)}; + const uint8_t * final_point{iter._json_iter->peek()}; return std::string_view(reinterpret_cast(starting_point), size_t(final_point - starting_point)); } diff --git a/src/arm64.cpp b/src/arm64.cpp index 5ec7d0cd8..2ece3784e 100644 --- a/src/arm64.cpp +++ b/src/arm64.cpp @@ -127,18 +127,6 @@ simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, // namespace simdjson { namespace arm64 { -namespace { -namespace stage1 { - -simdjson_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { - // On ARM, we don't short-circuit this if there are no backslashes, because the branch gives us no - // benefit and therefore makes things worse. - // if (!backslash) { uint64_t escaped = prev_escaped; prev_escaped = 0; return escaped; } - return find_escaped_branchless(backslash); -} - -} // namespace stage1 -} // unnamed namespace simdjson_warn_unused error_code implementation::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept { return arm64::stage1::json_minifier::minify<64>(buf, len, dst, dst_len); diff --git a/src/generic/stage1/amalgamated.h b/src/generic/stage1/amalgamated.h index e3ac67dce..ed083677f 100644 --- a/src/generic/stage1/amalgamated.h +++ b/src/generic/stage1/amalgamated.h @@ -1,11 +1,12 @@ // Stuff other things depend on #include +#include +#include #include #include #include // All other declarations -#include #include #include #include diff --git a/src/generic/stage1/buf_block_reader.h b/src/generic/stage1/buf_block_reader.h index 4efbae1c8..b3e4ec7d5 100644 --- a/src/generic/stage1/buf_block_reader.h +++ b/src/generic/stage1/buf_block_reader.h @@ -59,6 +59,17 @@ simdjson_unused static char * format_input_text(const simd8x64& in) { return buf; } +simdjson_unused static char * format_input_text(const simd8x64& in, uint64_t mask) { + static char buf[sizeof(simd8x64) + 1]; + in.store(reinterpret_cast(buf)); + for (size_t i=0; i); i++) { + if (buf[i] <= ' ') { buf[i] = '_'; } + if (!(mask & (size_t(1) << i))) { buf[i] = ' '; } + } + buf[sizeof(simd8x64)] = '\0'; + return buf; +} + simdjson_unused static char * format_mask(uint64_t mask) { static char buf[sizeof(simd8x64) + 1]; for (size_t i=0; i<64; i++) { diff --git a/src/generic/stage1/json_escape_scanner.h b/src/generic/stage1/json_escape_scanner.h new file mode 100644 index 000000000..2d9524912 --- /dev/null +++ b/src/generic/stage1/json_escape_scanner.h @@ -0,0 +1,151 @@ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_ESCAPE_SCANNER_H + +#ifndef SIMDJSON_CONDITIONAL_INCLUDE +#define SIMDJSON_SRC_GENERIC_STAGE1_JSON_ESCAPE_SCANNER_H +#include +#include +#endif // SIMDJSON_CONDITIONAL_INCLUDE + +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { +namespace { +namespace stage1 { + +/** + * Scans for escape characters in JSON, taking care with multiple backslashes (\\n vs. \n). + */ +struct json_escape_scanner { + /** The actual escape characters (the backslashes themselves). */ + uint64_t next_is_escaped; + + struct escaped_and_escape { + /** + * Mask of escaped characters. + * + * ``` + * \n \\n \\\n \\\\n \ + * 0100100010100101000 + * n \ \ n \ \ + * ``` + */ + uint64_t escaped; + /** + * Mask of escape characters. + * + * ``` + * \n \\n \\\n \\\\n \ + * 1001000101001010001 + * \ \ \ \ \ \ \ + * ``` + */ + uint64_t escape; + }; + + /** + * Get a mask of both escape and escaped characters (the characters following a backslash). + * + * @param potential_escape A mask of the character that can escape others (but could be + * escaped itself). e.g. block.eq('\\') + */ + simdjson_really_inline escaped_and_escape next(uint64_t backslash) noexcept { + +#if !SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT + if (!backslash) { return {next_escaped_without_backslashes(), 0}; } +#endif + + // | | Mask (shows characters instead of 1's) | Depth | Instructions | + // |--------------------------------|----------------------------------------|-------|---------------------| + // | string | `\\n_\\\n___\\\n___\\\\___\\\\__\\\` | + // | | ` even odd even odd odd` | + // | potential_escape | ` \ \\\ \\\ \\\\ \\\\ \\\` | 1 | 1 (backslash & ~first_is_escaped) + // | escape_and_terminal_code | ` \n \ \n \ \n \ \ \ \ \ \` | 5 | 5 (next_escape_and_terminal_code()) + // | escaped | `\ \ n \ n \ \ \ \ \ ` X | 6 | 7 (escape_and_terminal_code ^ (potential_escape | first_is_escaped)) + // | escape | ` \ \ \ \ \ \ \ \ \ \` | 6 | 8 (escape_and_terminal_code & backslash) + // | first_is_escaped | `\ ` | 7 (*) | 9 (escape >> 63) () + // (*) this is not needed until the next iteration + uint64_t escape_and_terminal_code = next_escape_and_terminal_code(backslash & ~this->next_is_escaped); + uint64_t escaped = escape_and_terminal_code ^ (backslash | this->next_is_escaped); + uint64_t escape = escape_and_terminal_code & backslash; + this->next_is_escaped = escape >> 63; + return {.escaped=escaped, .escape=escape}; + } + +private: + static constexpr const uint64_t ODD_BITS = 0xAAAAAAAAAAAAAAAAULL; + + uint64_t next_escaped_without_backslashes() noexcept { + uint64_t escaped = this->next_is_escaped; + this->next_is_escaped = 0; + return escaped; + } + + /** + * Returns a mask of the next escape characters (masking out escaped backslashes), along with + * any non-backslash escape codes. + * + * \n \\n \\\n \\\\n returns: + * \n \ \ \n \ \ + * 11 100 1011 10100 + * + * You are expected to mask out the first bit yourself if the previous block had a trailing + * escape. + * + * & the result with potential_escape to get just the escape characters. + * ^ the result with (potential_escape | first_is_escaped) to get escaped characters. + */ + static simdjson_inline uint64_t next_escape_and_terminal_code(uint64_t potential_escape) noexcept { + // If we were to just shift and mask out any odd bits, we'd actually get a *half* right answer: + // any even-aligned backslash runs would be correct! Odd-aligned backslash runs would be + // inverted (\\\ would be 010 instead of 101). + // + // ``` + // string: | ____\\\\_\\\\_____ | + // maybe_escaped | ODD | \ \ \ \ | + // even-aligned ^^^ ^^^^ odd-aligned + // ``` + // + // Taking that into account, our basic strategy is: + // + // 1. Use subtraction to produce a mask with 1's for even-aligned runs and 0's for + // odd-aligned runs. + // 2. XOR all odd bits, which masks out the odd bits in even-aligned runs, and brings IN the + // odd bits in odd-aligned runs. + // 3. & with backslash to clean up any stray bits. + // runs are set to 0, and then XORing with "odd": + // + // | | Mask (shows characters instead of 1's) | Instructions | + // |--------------------------------|----------------------------------------|---------------------| + // | string | `\\n_\\\n___\\\n___\\\\___\\\\__\\\` | + // | | ` even odd even odd odd` | + // | maybe_escaped | ` n \\n \\n \\\_ \\\_ \\` X | 1 (potential_escape << 1) + // | maybe_escaped_and_odd | ` \n_ \\n _ \\\n_ _ \\\__ _\\\_ \\\` | 1 (maybe_escaped | odd) + // | even_series_codes_and_odd | ` n_\\\ _ n_ _\\\\ _ _ ` | 1 (maybe_escaped_and_odd - potential_escape) + // | escape_and_terminal_code | ` \n \ \n \ \n \ \ \ \ \ \` | 1 (^ odd) + // + + // Escaped characters are characters following an escape. + uint64_t maybe_escaped = potential_escape << 1; + + // To distinguish odd from even escape sequences, therefore, we turn on any *starting* + // escapes that are on an odd byte. (We actually bring in all odd bits, for speed.) + // - Odd runs of backslashes are 0000, and the code at the end ("n" in \n or \\n) is 1. + // - Odd runs of backslashes are 1111, and the code at the end ("n" in \n or \\n) is 0. + // - All other odd bytes are 1, and even bytes are 0. + uint64_t maybe_escaped_and_odd_bits = maybe_escaped | ODD_BITS; + uint64_t even_series_codes_and_odd_bits = maybe_escaped_and_odd_bits - potential_escape; + + // Now we flip all odd bytes back with xor. This: + // - Makes odd runs of backslashes go from 0000 to 1010 + // - Makes even runs of backslashes go from 1111 to 1010 + // - Sets actually-escaped codes to 1 (the n in \n and \\n: \n = 11, \\n = 100) + // - Resets all other bytes to 0 + return even_series_codes_and_odd_bits ^ ODD_BITS; + } +}; + +} // namespace stage1 +} // unnamed namespace +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H \ No newline at end of file diff --git a/src/generic/stage1/json_string_scanner.h b/src/generic/stage1/json_string_scanner.h index 95d719cf3..82efba00a 100644 --- a/src/generic/stage1/json_string_scanner.h +++ b/src/generic/stage1/json_string_scanner.h @@ -3,6 +3,7 @@ #ifndef SIMDJSON_CONDITIONAL_INCLUDE #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H #include +#include #endif // SIMDJSON_CONDITIONAL_INCLUDE namespace simdjson { @@ -12,19 +13,13 @@ namespace stage1 { struct json_string_block { // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 - simdjson_inline json_string_block(uint64_t backslash, uint64_t escaped, uint64_t quote, uint64_t in_string) : - _backslash(backslash), _escaped(escaped), _quote(quote), _in_string(in_string) {} + simdjson_inline json_string_block(uint64_t escaped, uint64_t quote, uint64_t in_string) : + _escaped(escaped), _quote(quote), _in_string(in_string) {} // Escaped characters (characters following an escape() character) simdjson_inline uint64_t escaped() const { return _escaped; } - // Escape characters (backslashes that are not escaped--i.e. in \\, includes only the first \) - simdjson_inline uint64_t escape() const { return _backslash & ~_escaped; } // Real (non-backslashed) quotes simdjson_inline uint64_t quote() const { return _quote; } - // Start quotes of strings - simdjson_inline uint64_t string_start() const { return _quote & _in_string; } - // End quotes of strings - simdjson_inline uint64_t string_end() const { return _quote & ~_in_string; } // Only characters inside the string (not including the quotes) simdjson_inline uint64_t string_content() const { return _in_string & ~_quote; } // Return a mask of whether the given characters are inside a string (only works on non-quotes) @@ -34,11 +29,9 @@ struct json_string_block { // Tail of string (everything except the start quote) simdjson_inline uint64_t string_tail() const { return _in_string ^ _quote; } - // backslash characters - uint64_t _backslash; // escaped characters (backslashed--does not include the hex characters after \u) uint64_t _escaped; - // real quotes (non-backslashed ones) + // real quotes (non-escaped ones) uint64_t _quote; // string characters (includes start quote but not end quote) uint64_t _in_string; @@ -52,60 +45,12 @@ public: simdjson_inline error_code finish(); private: - // Intended to be defined by the implementation - simdjson_inline uint64_t find_escaped(uint64_t escape); - simdjson_inline uint64_t find_escaped_branchless(uint64_t escape); - + // Scans for escape characters + json_escape_scanner escape_scanner; // Whether the last iteration was still inside a string (all 1's = true, all 0's = false). uint64_t prev_in_string = 0ULL; - // Whether the first character of the next iteration is escaped. - uint64_t prev_escaped = 0ULL; }; -// -// Finds escaped characters (characters following \). -// -// Handles runs of backslashes like \\\" and \\\\" correctly (yielding 0101 and 01010, respectively). -// -// Does this by: -// - Shift the escape mask to get potentially escaped characters (characters after backslashes). -// - Mask escaped sequences that start on *even* bits with 1010101010 (odd bits are escaped, even bits are not) -// - Mask escaped sequences that start on *odd* bits with 0101010101 (even bits are escaped, odd bits are not) -// -// To distinguish between escaped sequences starting on even/odd bits, it finds the start of all -// escape sequences, filters out the ones that start on even bits, and adds that to the mask of -// escape sequences. This causes the addition to clear out the sequences starting on odd bits (since -// the start bit causes a carry), and leaves even-bit sequences alone. -// -// Example: -// -// text | \\\ | \\\"\\\" \\\" \\"\\" | -// escape | xxx | xx xxx xxx xx xx | Removed overflow backslash; will | it into follows_escape -// odd_starts | x | x x x | escape & ~even_bits & ~follows_escape -// even_seq | c| cxxx c xx c | c = carry bit -- will be masked out later -// invert_mask | | cxxx c xx c| even_seq << 1 -// follows_escape | xx | x xx xxx xxx xx xx | Includes overflow bit -// escaped | x | x x x x x x x x | -// desired | x | x x x x x x x x | -// text | \\\ | \\\"\\\" \\\" \\"\\" | -// -simdjson_inline uint64_t json_string_scanner::find_escaped_branchless(uint64_t backslash) { - // If there was overflow, pretend the first character isn't a backslash - backslash &= ~prev_escaped; - uint64_t follows_escape = backslash << 1 | prev_escaped; - - // Get sequences starting on even bits by clearing out the odd series using + - const uint64_t even_bits = 0x5555555555555555ULL; - uint64_t odd_sequence_starts = backslash & ~even_bits & ~follows_escape; - uint64_t sequences_starting_on_even_bits; - prev_escaped = add_overflow(odd_sequence_starts, backslash, &sequences_starting_on_even_bits); - uint64_t invert_mask = sequences_starting_on_even_bits << 1; // The mask we want to return is the *escaped* bits, not escapes. - - // Mask every other backslashed character as an escaped character - // Flip the mask for sequences that start on even bits, to correct them - return (even_bits ^ invert_mask) & follows_escape; -} - // // Return a mask of all string characters plus end quotes. // @@ -116,7 +61,7 @@ simdjson_inline uint64_t json_string_scanner::find_escaped_branchless(uint64_t b // simdjson_inline json_string_block json_string_scanner::next(const simd::simd8x64& in) { const uint64_t backslash = in.eq('\\'); - const uint64_t escaped = find_escaped(backslash); + const uint64_t escaped = escape_scanner.next(backslash).escaped; const uint64_t quote = in.eq('"') & ~escaped; // @@ -130,21 +75,13 @@ simdjson_inline json_string_block json_string_scanner::next(const simd::simd8x64 // // Check if we're still in a string at the end of the box so the next block will know // - // right shift of a signed value expected to be well-defined and standard - // compliant as of C++20, John Regher from Utah U. says this is fine code - // prev_in_string = uint64_t(static_cast(in_string) >> 63); // Use ^ to turn the beginning quote off, and the end quote on. // We are returning a function-local object so either we get a move constructor // or we get copy elision. - return json_string_block( - backslash, - escaped, - quote, - in_string - ); + return json_string_block(escaped, quote, in_string); } simdjson_inline error_code json_string_scanner::finish() { diff --git a/src/haswell.cpp b/src/haswell.cpp index 1524e8655..5eec137aa 100644 --- a/src/haswell.cpp +++ b/src/haswell.cpp @@ -125,16 +125,6 @@ simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, // namespace simdjson { namespace SIMDJSON_IMPLEMENTATION { -namespace { -namespace stage1 { - -simdjson_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { - if (!backslash) { uint64_t escaped = prev_escaped; prev_escaped = 0; return escaped; } - return find_escaped_branchless(backslash); -} - -} // namespace stage1 -} // unnamed namespace simdjson_warn_unused error_code implementation::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept { return haswell::stage1::json_minifier::minify<128>(buf, len, dst, dst_len); diff --git a/src/icelake.cpp b/src/icelake.cpp index ca1adf057..92c7d2faf 100644 --- a/src/icelake.cpp +++ b/src/icelake.cpp @@ -171,16 +171,6 @@ SIMDJSON_POP_DISABLE_WARNINGS // namespace simdjson { namespace icelake { -namespace { -namespace stage1 { - -simdjson_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { - if (!backslash) { uint64_t escaped = prev_escaped; prev_escaped = 0; return escaped; } - return find_escaped_branchless(backslash); -} - -} // namespace stage1 -} // unnamed namespace simdjson_warn_unused error_code implementation::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept { return icelake::stage1::json_minifier::minify<128>(buf, len, dst, dst_len); diff --git a/src/ppc64.cpp b/src/ppc64.cpp index 0de9e42f2..769164c17 100644 --- a/src/ppc64.cpp +++ b/src/ppc64.cpp @@ -98,18 +98,6 @@ simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, // namespace simdjson { namespace ppc64 { -namespace { -namespace stage1 { - -simdjson_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { - // On PPC, we don't short-circuit this if there are no backslashes, because the branch gives us no - // benefit and therefore makes things worse. - // if (!backslash) { uint64_t escaped = prev_escaped; prev_escaped = 0; return escaped; } - return find_escaped_branchless(backslash); -} - -} // namespace stage1 -} // unnamed namespace simdjson_warn_unused error_code implementation::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept { return ppc64::stage1::json_minifier::minify<64>(buf, len, dst, dst_len); diff --git a/src/westmere.cpp b/src/westmere.cpp index 2b1dba47b..45d5e2c3f 100644 --- a/src/westmere.cpp +++ b/src/westmere.cpp @@ -130,16 +130,6 @@ simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, namespace simdjson { namespace westmere { -namespace { -namespace stage1 { - -simdjson_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { - if (!backslash) { uint64_t escaped = prev_escaped; prev_escaped = 0; return escaped; } - return find_escaped_branchless(backslash); -} - -} // namespace stage1 -} // unnamed namespace simdjson_warn_unused error_code implementation::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept { return westmere::stage1::json_minifier::minify<64>(buf, len, dst, dst_len); diff --git a/tests/dom/basictests.cpp b/tests/dom/basictests.cpp index ee2423f34..b1f4bf4e3 100644 --- a/tests/dom/basictests.cpp +++ b/tests/dom/basictests.cpp @@ -2261,7 +2261,8 @@ int main(int argc, char *argv[]) { dom_api_tests::run() && type_tests::run() && format_tests::run() && - number_tests::run() + number_tests::run() && + true ) { std::cout << "Basic tests are ok." << std::endl; return EXIT_SUCCESS;