diff --git a/include/simdjson/generic/lookup_table.h b/include/simdjson/generic/lookup_table.h index 604f4bc3a..516f1ed7d 100644 --- a/include/simdjson/generic/lookup_table.h +++ b/include/simdjson/generic/lookup_table.h @@ -218,7 +218,7 @@ struct low_nibble_lookup { simdjson_inline simd8 operator[](const simd8& keys) const noexcept { return lookup(keys); } /** Look up the value corresponding the lower 4 bits of each input byte, and return it. */ simdjson_inline simd8 lookup(const simd8& keys) const noexcept { - return keys.lookup_low_nibble_ascii(keys); + return keys.lookup_low_nibble_ascii(table); } /** * Look up the value in the table. Behavior is system-dependent for indexes greater than 16. @@ -227,15 +227,15 @@ struct low_nibble_lookup { * - On platforms like Intel, index bits 4-6 will be ignored, but if the high bit is set, it * will not match anything in the table. greater than 16 will be ignored, *except* if the high bit is 1, */ - simdjson_inline simd8 lookup_unsafe(const simd8& shifted_keys) const noexcept { - return shifted_keys.lookup_16(table); + simdjson_inline simd8 lookup_unsafe(const simd8& keys) const noexcept { + return keys.lookup_16(table); } /** Look up the value corresponding the lower 4 bits of each input byte, and return it. */ simdjson_inline simd8x64 operator[](const simd8x64& keys) const noexcept { return lookup(keys); } /** Look up the value corresponding the lower 4 bits of each input byte, and return it. */ simdjson_inline simd8x64 lookup(const simd8x64& keys) const noexcept { - return keys.lookup_low_nibble_ascii(keys); + return keys.lookup_low_nibble_ascii(table); } /** * Look up the value in the table. Behavior is system-dependent for indexes greater than 16. diff --git a/include/simdjson/westmere/begin.h b/include/simdjson/westmere/begin.h index 011d71c10..56d9158c2 100644 --- a/include/simdjson/westmere/begin.h +++ b/include/simdjson/westmere/begin.h @@ -6,7 +6,6 @@ SIMDJSON_TARGET_REGION("sse4.2,pclmul,popcnt") #endif -#include "simdjson/westmere/bitmask.h" #include "simdjson/westmere/bitmask.h" #include "simdjson/westmere/numberparsing_defs.h" #include "simdjson/westmere/simd.h" diff --git a/include/simdjson/westmere/bitmask.h b/include/simdjson/westmere/bitmask.h index f8d33ff1b..d2a6e845d 100644 --- a/include/simdjson/westmere/bitmask.h +++ b/include/simdjson/westmere/bitmask.h @@ -10,6 +10,11 @@ namespace simdjson { namespace westmere { namespace bitmask { +simdjson_constinit uint64_t ALL = 0xFFFFFFFFFFFFFFFF; +simdjson_constinit uint64_t NONE = 0xFFFFFFFFFFFFFFFF; +simdjson_constinit uint64_t EVEN = 0x5555555555555555; +simdjson_constinit uint64_t ODD = 0xAAAAAAAAAAAAAAAA; + // We sometimes call trailing_zero on inputs that are zero, // but the algorithms do not end up using the returned value. // Sadly, sanitizers are not smart enough to figure it out. diff --git a/src/generic/amalgamated.h b/src/generic/amalgamated.h index 32154f602..352f9803a 100644 --- a/src/generic/amalgamated.h +++ b/src/generic/amalgamated.h @@ -4,4 +4,3 @@ #include #include -#include diff --git a/src/generic/base.h b/src/generic/base.h index 77947651a..cae3921ae 100644 --- a/src/generic/base.h +++ b/src/generic/base.h @@ -10,8 +10,6 @@ namespace simdjson { namespace SIMDJSON_IMPLEMENTATION { namespace { -struct json_character_block; - } // unnamed namespace } // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson diff --git a/src/generic/json_character_block.h b/src/generic/json_character_block.h deleted file mode 100644 index 7cce34c83..000000000 --- a/src/generic/json_character_block.h +++ /dev/null @@ -1,27 +0,0 @@ -#ifndef SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H - -#ifndef SIMDJSON_CONDITIONAL_INCLUDE -#define SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H -#include -#endif // SIMDJSON_CONDITIONAL_INCLUDE - -namespace simdjson { -namespace SIMDJSON_IMPLEMENTATION { -namespace { - -struct json_character_block { - static simdjson_inline json_character_block classify(const simd::simd8x64& in); - - simdjson_inline uint64_t whitespace() const noexcept { return _whitespace; } - simdjson_inline uint64_t op() const noexcept { return _op; } - simdjson_inline uint64_t scalar() const noexcept { return ~(op() | whitespace()); } - - uint64_t _whitespace; - uint64_t _op; -}; - -} // unnamed namespace -} // namespace SIMDJSON_IMPLEMENTATION -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H \ No newline at end of file diff --git a/src/generic/stage1/base.h b/src/generic/stage1/base.h index a6413fadb..a82f499d5 100644 --- a/src/generic/stage1/base.h +++ b/src/generic/stage1/base.h @@ -13,10 +13,8 @@ namespace stage1 { class bit_indexer; template struct buf_block_reader; -struct json_block; class json_minifier; class json_scanner; -struct json_string_block; class json_string_scanner; class json_structural_indexer; diff --git a/src/generic/stage1/json_minifier.h b/src/generic/stage1/json_minifier.h index 22ddaf2dd..31c983602 100644 --- a/src/generic/stage1/json_minifier.h +++ b/src/generic/stage1/json_minifier.h @@ -28,15 +28,14 @@ private: {} template simdjson_inline void step(const uint8_t *block_buf, buf_block_reader &reader) noexcept; - simdjson_inline void next(const simd::simd8x64& in, const json_block& block); + simdjson_inline void next(const simd::simd8x64& in, uint64_t whitespace); simdjson_inline error_code finish(uint8_t *dst_start, size_t &dst_len); json_scanner scanner{}; uint8_t *dst; }; -simdjson_inline void json_minifier::next(const simd::simd8x64& in, const json_block& block) { - uint64_t mask = block.whitespace(); - dst += in.compress(mask, dst); +simdjson_inline void json_minifier::next(const simd::simd8x64& in, uint64_t ws) { + dst += in.compress(ws, dst); } simdjson_inline error_code json_minifier::finish(uint8_t *dst_start, size_t &dst_len) { @@ -50,18 +49,18 @@ template<> simdjson_inline void json_minifier::step<128>(const uint8_t *block_buf, buf_block_reader<128> &reader) noexcept { simd::simd8x64 in_1(block_buf); simd::simd8x64 in_2(block_buf+64); - json_block block_1 = scanner.next(in_1); - json_block block_2 = scanner.next(in_2); - this->next(in_1, block_1); - this->next(in_2, block_2); + uint64_t ws_1 = scanner.next_whitespace(in_1); + uint64_t ws_2 = scanner.next_whitespace(in_2); + this->next(in_1, ws_1); + this->next(in_2, ws_2); reader.advance(); } template<> simdjson_inline void json_minifier::step<64>(const uint8_t *block_buf, buf_block_reader<64> &reader) noexcept { simd::simd8x64 in_1(block_buf); - json_block block_1 = scanner.next(in_1); - this->next(block_buf, block_1); + uint64_t ws_1 = scanner.next(in_1); + this->next(block_buf, ws_1); reader.advance(); } diff --git a/src/generic/stage1/json_scanner.h b/src/generic/stage1/json_scanner.h index 6e103a953..714e9d579 100644 --- a/src/generic/stage1/json_scanner.h +++ b/src/generic/stage1/json_scanner.h @@ -4,7 +4,6 @@ #ifndef SIMDJSON_CONDITIONAL_INCLUDE #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H #include -#include #include #include #endif // SIMDJSON_CONDITIONAL_INCLUDE @@ -31,6 +30,7 @@ class json_scanner { public: json_scanner() = default; simdjson_inline uint64_t next(const simd::simd8x64& in) noexcept; + simdjson_inline uint64_t next_whitespace(const simd::simd8x64& in) noexcept; // Returns either UNCLOSED_STRING or SUCCESS simdjson_inline error_code finish() const noexcept; @@ -64,14 +64,14 @@ private: { '\\', BACKSLASH }, }; static simdjson_constinit low_nibble_lookup WHITESPACE_MATCH = { - { ' '-1, 0xFF }, - { '\t'-1, 0xFF }, - { '\r'-1, 0xFF }, - { '\n'-1, 0xFF }, + { ' ', ' ' }, + { '\t', '\t' }, + { '\r', '\r' }, + { '\n', '\n' }, }; simdjson_inline uint64_t next_separated_values(uint64_t sep_open, uint64_t scalar_close) noexcept; - simdjson_inline void check_errors(uint64_t scalar, uint64_t sep, uint64_t open, uint64_t raw_quote, uint64_t separated_values, uint64_t in_string) noexcept; + simdjson_inline void check_errors(uint64_t scalar, uint64_t ctrl, uint64_t sep, uint64_t open, uint64_t raw_quote, uint64_t separated_values, uint64_t in_string) noexcept; // Whether the last character of the previous iteration is part of a scalar token // (anything except whitespace or a structural character/'operator'). @@ -100,7 +100,7 @@ simdjson_inline uint64_t json_scanner::next( uint64_t backslash = in.eq('\\'); // 3+LN (LN+simd:N) uint64_t raw_quote = in.eq('"'); // 3+LN (LN+simd:N) uint64_t in_string = string_scanner.next(backslash, raw_quote, separated_values); // 10+LN (+6) or (14+LN or 18+LN (+8+simd:3)) - // critical path = 10+LN (+6+2LN+simd:2N) or (14+LN or 18+LN (+8+2LN+simd:2N+3)) + // critical path = 10+LN (+6+2LN+simd:2N) or (14+LN or 18+LN) (+8+2LN+simd:2N+3) uint64_t lead_value = scalar_close & separated_values; // 8+LN (+1) // uint64_t op_without_comma = colon | open | close; // 8+LN (+1) (ternary) @@ -111,14 +111,16 @@ simdjson_inline uint64_t json_scanner::next( // critical path = 11+LN or (15+LN or 19+LN) (+3) uint64_t scalar = scalar_close & ~close; // 8+LN (+1) - check_errors(scalar, sep, open, raw_quote, separated_values, in_string); // 14+LN (+9) - // critical path = 14+LN (+10) + uint64_t ws = in.eq(WHITESPACE_MATCH.lookup(in)); // 6+LN (+LN+simd:2N) + uint64_t ctrl = ws_ctrl & ~ws; // 7+LN (+1) + check_errors(scalar, ctrl, sep, open, raw_quote, separated_values, in_string); // 14+LN (+9) + // critical path = 14+LN (+11+LN+simd:2N) return structurals; - // structurals: critical path = 11+LN (+24+7LN+simd:8N) or (15+LN or 19+LN) (+26+7LN+simd:8N+3) - // = icelake: 11 (24+simd:8 ) or (15 or 19) (26+simd:11) - // = haswell: 12 (31+simd:16) or (16 or 21) (33+simd:19) - // = westmere: 13 (38+simd:32) or (17 or 22) (40+simd:35) + // structurals: critical path = 11+LN (+25+8LN+simd:10N) or (15+LN or 19+LN) (+27+8LN+simd:10N+3) + // = icelake: 11 (24+simd:10) or (15 or 19) (27+simd:13) + // = haswell: 12 (31+simd:20) or (16 or 21) (35+simd:23) + // = westmere: 13 (38+simd:40) or (17 or 22) (43+simd:43) } simdjson_inline uint64_t json_scanner::next_separated_values( @@ -138,6 +140,7 @@ simdjson_inline uint64_t json_scanner::next_separated_values( simdjson_inline void json_scanner::check_errors( uint64_t scalar, // 8+LN + uint64_t ctrl, // 7+LN uint64_t sep, // 4+LN uint64_t open, // 6+LN uint64_t raw_quote, // 3+LN @@ -164,7 +167,7 @@ simdjson_inline void json_scanner::check_errors( uint64_t raw_separator_error = missing_separator_error | extra_separator_error | missing_separator_before_open_error; // flip lead quote off and trail quote on: lead quote errors uint64_t separator_error = raw_separator_error & (in_string ^ raw_quote); // 13+LN (+1) (ternary) - this->error |= separator_error; // 14+LN (+1) + this->error |= separator_error | ctrl; // 14+LN (+1) (ternary) // critical path = 14+LN (+3) // NOT validated: @@ -178,10 +181,37 @@ simdjson_inline void json_scanner::check_errors( // critical path = 14+LN (+9) } +simdjson_inline uint64_t json_scanner::next_whitespace( + const simd::simd8x64& in +) noexcept { + simd8x64 curlified = in | ('{' - '['); // 3 (+simd:N) + uint64_t open = curlified.eq('{'); // 6+LN (+LN+simd:N) + uint64_t close = curlified.eq('}'); // 6+LN (+LN+simd:N) + uint64_t comma = in.eq(','); // 3+LN (+LN+simd:N) + uint64_t colon = in.eq(':'); // 3+LN (+LN+simd:N) + uint64_t sep = comma | colon; // 4+LN (+1) + uint64_t ws = in.eq(WHITESPACE_MATCH.lookup(in)); // 6+LN (+LN+simd:2N) + uint64_t sep_open = sep | open; // 7+LN (+1) + uint64_t scalar_close = ~sep & ~open & ~ws; // 7+LN (+1) (ternary) + // total 7+LN (+3+5LN+simd:7N) + + uint64_t separated_values = next_separated_values(sep_open, scalar_close); // 8+LN (+2) + + uint64_t backslash = in.eq('\\'); // 3+LN (LN+simd:N) + uint64_t raw_quote = in.eq('"'); // 3+LN (LN+simd:N) + uint64_t in_string = string_scanner.next(backslash, raw_quote, separated_values); // 10+LN (+6) or (14+LN or 18+LN (+8+simd:3)) + // total 10+LN (+6+2LN+simd:2N) or (14+LN or 18+LN) (+8+2LN+simd:2N+3) + + return ws & ~in_string; + // critical path = 10+LN (+11+7LN+simd:9N) or (14+LN or 18+LN) (+13+7LN+simd:9N+3) +} + + simdjson_inline error_code json_scanner::finish() const noexcept { if (this->error | this->string_scanner.finish()) { return TAPE_ERROR; } + return SUCCESS; } } // namespace stage1 diff --git a/src/generic/stage1/json_structural_indexer.h b/src/generic/stage1/json_structural_indexer.h index 881e8e6a5..45aaa7c70 100644 --- a/src/generic/stage1/json_structural_indexer.h +++ b/src/generic/stage1/json_structural_indexer.h @@ -140,14 +140,13 @@ private: simdjson_inline json_structural_indexer(uint32_t *structural_indexes); template simdjson_inline void step(const uint8_t *block, buf_block_reader &reader) noexcept; - simdjson_inline void next(const simd::simd8x64& in, const json_block& block, size_t idx); + simdjson_inline void next(const simd::simd8x64& in, uint64_t structurals, size_t idx); simdjson_inline error_code finish(dom_parser_implementation &parser, size_t idx, size_t len, stage1_mode partial); json_scanner scanner{}; utf8_checker checker{}; bit_indexer indexer; uint64_t prev_structurals = 0; - uint64_t unescaped_chars_error = 0; }; simdjson_inline json_structural_indexer::json_structural_indexer(uint32_t *structural_indexes) : indexer{structural_indexes} {} @@ -221,29 +220,28 @@ template<> simdjson_inline void json_structural_indexer::step<128>(const uint8_t *block, buf_block_reader<128> &reader) noexcept { simd::simd8x64 in_1(block); simd::simd8x64 in_2(block+64); - json_block block_1 = scanner.next(in_1); - json_block block_2 = scanner.next(in_2); - this->next(in_1, block_1, reader.block_index()); - this->next(in_2, block_2, reader.block_index()+64); + uint64_t structurals_1 = scanner.next(in_1); + uint64_t structurals_2 = scanner.next(in_2); + this->next(in_1, structurals_1, reader.block_index()); + this->next(in_2, structurals_2, reader.block_index()+64); reader.advance(); } template<> simdjson_inline void json_structural_indexer::step<64>(const uint8_t *block, buf_block_reader<64> &reader) noexcept { simd::simd8x64 in_1(block); - json_block block_1 = scanner.next(in_1); - this->next(in_1, block_1, reader.block_index()); + uint64_t structurals_1 = scanner.next(in_1); + this->next(in_1, structurals_1, reader.block_index()); reader.advance(); } -simdjson_inline void json_structural_indexer::next(const simd::simd8x64& in, const json_block& block, size_t idx) { +simdjson_inline void json_structural_indexer::next(const simd::simd8x64& in,uint64_t structurals, size_t idx) { uint64_t unescaped = in.lteq(0x1F); #if SIMDJSON_UTF8VALIDATION checker.check_next_input(in); #endif indexer.write(uint32_t(idx-64), prev_structurals); // Output *last* iteration's structurals to the parser - prev_structurals = block.structural_start(); - unescaped_chars_error |= block.non_quote_inside_string(unescaped); + prev_structurals = structurals; } simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementation &parser, size_t idx, size_t len, stage1_mode partial) { @@ -258,9 +256,6 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa const bool have_unclosed_string = (error == UNCLOSED_STRING); if (simdjson_unlikely(should_we_exit)) { return error; } - if (unescaped_chars_error) { - return UNESCAPED_CHARS; - } parser.n_structural_indexes = uint32_t(indexer.tail - parser.structural_indexes.get()); /*** * The On Demand API requires special padding. diff --git a/src/haswell.cpp b/src/haswell.cpp index 3cf9b1011..2d0cc3338 100644 --- a/src/haswell.cpp +++ b/src/haswell.cpp @@ -38,53 +38,6 @@ namespace { using namespace simd; -enum class ws_op_t { - COMMA = 1 << 0, - COLON = 1 << 1, - OPEN = 1 << 2, - CLOSE = 1 << 3, - QUOTE = 1 << 4, - BACKSLASH = 1 << 5, - SPACE = 1 << 6, - TAB_CR_LF = 1 << 7, - OP = COMMA | COLON | OPEN | CLOSE, - SEPARATOR = COMMA | COLON, - WHITESPACE = SPACE | TAB_CR_LF, -}; - -struct json_character_block { - simd8x64 classified; - - static simdjson_constinit byte_classifier WS_MATCH { - { ',', ws_op_t::COMMA }, - { ':', ws_op_t::COLON }, - { '[', ws_op_t::OPEN }, - { '{', ws_op_t::OPEN }, - { ']', ws_op_t::CLOSE }, - { '}', ws_op_t::CLOSE }, - { '\"', ws_op_t::QUOTE }, - { ' ', ws_op_t::SPACE }, - { '\t', ws_op_t::TAB_CR_LF }, - { '\r', ws_op_t::TAB_CR_LF }, - { '\n', ws_op_t::TAB_CR_LF }, - { '\\', ws_op_t::BACKSLASH }, - }; - - // These are all mutually exclusive, so we can use > to - simdjson_inline uint64_t separators_or_open() const noexcept { return ; } - simdjson_inline uint64_t separators_or_open() const noexcept { return classified.any_bits_set(ws_op_t::OP); } - simdjson_inline uint64_t operators() const noexcept { return classified.any_bits_set(ws_op_t::OP); } - simdjson_inline uint64_t whitespace() const noexcept { return classified.any_bits_set(ws_op_t::WHITESPACE); } -}; - -// This identifies structural characters (comma, colon, braces, brackets), -// and ASCII white-space ('\r','\n','\t',' '). -simdjson_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { - // Turn [ and ] into { and } - uint64_t sep_and_open = in. - return { in.eq(WS_MATCH[in]), curlified.eq(OP_MATCH[in]) }; -} - simdjson_inline bool is_ascii(const simd8x64& in) { return in.reduce_or().is_ascii(); } diff --git a/src/icelake.cpp b/src/icelake.cpp index 15a9a3ba4..b424f1222 100644 --- a/src/icelake.cpp +++ b/src/icelake.cpp @@ -43,29 +43,6 @@ namespace { using namespace simd; -static simdjson_constinit low_nibble_lookup WS_MATCH{ - {' ', ' '}, - {'\t', '\t'}, - {'\n', '\n'}, - {'\r', '\r'}, -}; -static simdjson_constinit low_nibble_lookup OP_MATCH{ - {':', ':'}, - {',', ','}, - {'{', '{'}, - {'}', '}'}, - {'[', '{'}, - {']', '}'}, -}; - -// This identifies structural characters (comma, colon, braces, brackets), -// and ASCII white-space ('\r','\n','\t',' '). -simdjson_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { - // Turn [ and ] into { and } - const simd8x64 curlified = in | 0x20; - return { in.eq(WS_MATCH[in]), curlified.eq(OP_MATCH[in]) }; -} - simdjson_inline bool is_ascii(const simd8x64& input) { return input.reduce_or().is_ascii(); } diff --git a/src/westmere.cpp b/src/westmere.cpp index c38e029b3..4a924283c 100644 --- a/src/westmere.cpp +++ b/src/westmere.cpp @@ -38,29 +38,6 @@ namespace { using namespace simd; -static simdjson_constinit low_nibble_lookup WS_MATCH{ - {' ', ' '}, - {'\t', '\t'}, - {'\n', '\n'}, - {'\r', '\r'}, -}; -static simdjson_constinit low_nibble_lookup OP_MATCH{ - {':', ':'}, - {',', ','}, - {'{', '{'}, - {'}', '}'}, - {'[', '{'}, - {']', '}'}, -}; - -// This identifies structural characters (comma, colon, braces, brackets), -// and ASCII white-space ('\r','\n','\t',' '). -simdjson_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { - // Turn [ and ] into { and } - const simd8x64 curlified = in | 0x20; - return { in.eq(WS_MATCH[in]), curlified.eq(OP_MATCH[in]) }; -} - simdjson_inline bool is_ascii(const simd8x64& input) { return input.reduce_or().is_ascii(); }