From 3fde8a4eac96c88c352ff7819bbd116291611155 Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Mon, 4 Jul 2022 21:54:19 -0400 Subject: [PATCH] New release candidate (#1856) * Patch for possible AVX-512 overflow. * Updating the test for new padding. * Preparing new version. * replace binary integer literals with hex literals for C++11 compatibility (#1855) Binary integer literals are a C++14 feature, so those are not supported in C++11 and should be replaced by hexadecimal literals instead. Fixes #1854. Co-authored-by: Dirk Stolle --- CMakeLists.txt | 6 +-- Doxyfile | 2 +- benchmark/json2msgpack/simdjson_ondemand.h | 4 +- include/simdjson/arm64/intrinsics.h | 2 + include/simdjson/common_defs.h | 4 +- include/simdjson/haswell/intrinsics.h | 2 + include/simdjson/icelake/intrinsics.h | 2 + include/simdjson/ppc64/intrinsics.h | 2 + include/simdjson/simdjson_version.h | 4 +- include/simdjson/westmere/intrinsics.h | 2 +- singleheader/simdjson.cpp | 2 +- singleheader/simdjson.h | 20 +++++--- src/arm64/dom_parser_implementation.cpp | 12 ++--- src/fallback/dom_parser_implementation.cpp | 54 ++++++++++---------- src/generic/stage1/json_structural_indexer.h | 12 ++--- src/generic/stage1/utf8_lookup4_algorithm.h | 4 +- src/haswell/dom_parser_implementation.cpp | 10 ++-- src/icelake/dom_parser_implementation.cpp | 10 ++-- src/ppc64/dom_parser_implementation.cpp | 12 ++--- src/westmere/dom_parser_implementation.cpp | 10 ++-- tests/dom/errortests.cpp | 7 +-- tests/ondemand/ondemand_parse_api_tests.cpp | 20 ++++---- 22 files changed, 110 insertions(+), 93 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d973ebe9a..d5e9beea8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.14) project( simdjson # The version number is modified by tools/release.py - VERSION 2.1.0 + VERSION 2.2.0 DESCRIPTION "Parsing gigabytes of JSON per second" HOMEPAGE_URL "https://simdjson.org/" LANGUAGES CXX C @@ -20,8 +20,8 @@ string( # ---- Options, variables ---- # These version numbers are modified by tools/release.py -set(SIMDJSON_LIB_VERSION "12.0.0" CACHE STRING "simdjson library version") -set(SIMDJSON_LIB_SOVERSION "12" CACHE STRING "simdjson library soversion") +set(SIMDJSON_LIB_VERSION "13.0.0" CACHE STRING "simdjson library version") +set(SIMDJSON_LIB_SOVERSION "13" CACHE STRING "simdjson library soversion") option(SIMDJSON_ENABLE_THREADS "Link with thread support" ON) diff --git a/Doxyfile b/Doxyfile index 637157bc9..7f6526939 100644 --- a/Doxyfile +++ b/Doxyfile @@ -38,7 +38,7 @@ PROJECT_NAME = simdjson # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = "2.1.0" +PROJECT_NUMBER = "2.2.0" # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/benchmark/json2msgpack/simdjson_ondemand.h b/benchmark/json2msgpack/simdjson_ondemand.h index 15d86c64b..e1a12571b 100644 --- a/benchmark/json2msgpack/simdjson_ondemand.h +++ b/benchmark/json2msgpack/simdjson_ondemand.h @@ -34,8 +34,8 @@ struct simdjson2msgpack { * @brief Converts the provided JSON into msgpack. * * @param json JSON input - * @param buf temporary buffer (must be large enough, with 32 bytes of - * padding) + * @param buf temporary buffer (must be large enough, with simdjson::SIMDJSON_PADDING bytes + * of padding) * @return std::string_view msgpack output, writting to the temporary buffer */ inline std::string_view to_msgpack(const simdjson::padded_string &json, diff --git a/include/simdjson/arm64/intrinsics.h b/include/simdjson/arm64/intrinsics.h index 6f6ac4e2f..900d8d0b2 100644 --- a/include/simdjson/arm64/intrinsics.h +++ b/include/simdjson/arm64/intrinsics.h @@ -5,4 +5,6 @@ // you use visual studio or other compilers. #include +static_assert(sizeof(uint8x16_t) <= simdjson::SIMDJSON_PADDING, "insufficient padding for arm64"); + #endif // SIMDJSON_ARM64_INTRINSICS_H diff --git a/include/simdjson/common_defs.h b/include/simdjson/common_defs.h index 58918a244..ed1e35f6c 100644 --- a/include/simdjson/common_defs.h +++ b/include/simdjson/common_defs.h @@ -37,12 +37,12 @@ constexpr size_t SIMDJSON_MAXSIZE_BYTES = 0xFFFFFFFF; /** * The amount of padding needed in a buffer to parse JSON. * - * the input buf should be readable up to buf + SIMDJSON_PADDING + * The input buf should be readable up to buf + SIMDJSON_PADDING * this is a stopgap; there should be a better description of the * main loop and its behavior that abstracts over this * See https://github.com/simdjson/simdjson/issues/174 */ -constexpr size_t SIMDJSON_PADDING = 32; +constexpr size_t SIMDJSON_PADDING = 64; /** * By default, simdjson supports this many nested objects and arrays. diff --git a/include/simdjson/haswell/intrinsics.h b/include/simdjson/haswell/intrinsics.h index da15dfa6c..8db2e2b64 100644 --- a/include/simdjson/haswell/intrinsics.h +++ b/include/simdjson/haswell/intrinsics.h @@ -45,4 +45,6 @@ #endif // _blsr_u64 #endif // SIMDJSON_CLANG_VISUAL_STUDIO +static_assert(sizeof(__m256i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for haswell kernel."); + #endif // SIMDJSON_HASWELL_INTRINSICS_H diff --git a/include/simdjson/icelake/intrinsics.h b/include/simdjson/icelake/intrinsics.h index e5715f112..2a09d1679 100644 --- a/include/simdjson/icelake/intrinsics.h +++ b/include/simdjson/icelake/intrinsics.h @@ -53,4 +53,6 @@ #endif // _blsr_u64 #endif // SIMDJSON_CLANG_VISUAL_STUDIO +static_assert(sizeof(__m512i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for icelake"); + #endif // SIMDJSON_ICELAKE_INTRINSICS_H diff --git a/include/simdjson/ppc64/intrinsics.h b/include/simdjson/ppc64/intrinsics.h index 00dc79c19..0dd045d1a 100644 --- a/include/simdjson/ppc64/intrinsics.h +++ b/include/simdjson/ppc64/intrinsics.h @@ -16,4 +16,6 @@ #undef vector #endif +static_assert(sizeof(__vector unsigned char) <= simdjson::SIMDJSON_PADDING, "insufficient padding for ppc64"); + #endif // SIMDJSON_PPC64_INTRINSICS_H diff --git a/include/simdjson/simdjson_version.h b/include/simdjson/simdjson_version.h index 071b16e2b..8b932fd84 100644 --- a/include/simdjson/simdjson_version.h +++ b/include/simdjson/simdjson_version.h @@ -4,7 +4,7 @@ #define SIMDJSON_SIMDJSON_VERSION_H /** The version of simdjson being used (major.minor.revision) */ -#define SIMDJSON_VERSION 2.1.0 +#define SIMDJSON_VERSION 2.2.0 namespace simdjson { enum { @@ -15,7 +15,7 @@ enum { /** * The minor version (major.MINOR.revision) of simdjson being used. */ - SIMDJSON_VERSION_MINOR = 1, + SIMDJSON_VERSION_MINOR = 2, /** * The revision (major.minor.REVISION) of simdjson being used. */ diff --git a/include/simdjson/westmere/intrinsics.h b/include/simdjson/westmere/intrinsics.h index 370474124..530e4c602 100644 --- a/include/simdjson/westmere/intrinsics.h +++ b/include/simdjson/westmere/intrinsics.h @@ -22,6 +22,6 @@ #include // for _mm_clmulepi64_si128 #endif - +static_assert(sizeof(__m128i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for westmere"); #endif // SIMDJSON_WESTMERE_INTRINSICS_H diff --git a/singleheader/simdjson.cpp b/singleheader/simdjson.cpp index 3458b2a5f..ca4e15ff2 100644 --- a/singleheader/simdjson.cpp +++ b/singleheader/simdjson.cpp @@ -1,4 +1,4 @@ -/* auto-generated on 2022-06-30 10:29:50 -0400. Do not edit! */ +/* auto-generated on 2022-07-04 20:04:25 -0400. Do not edit! */ /* begin file src/simdjson.cpp */ #include "simdjson.h" diff --git a/singleheader/simdjson.h b/singleheader/simdjson.h index 7029b12e5..0c8f83ddf 100644 --- a/singleheader/simdjson.h +++ b/singleheader/simdjson.h @@ -1,4 +1,4 @@ -/* auto-generated on 2022-06-30 10:29:50 -0400. Do not edit! */ +/* auto-generated on 2022-07-04 20:04:25 -0400. Do not edit! */ /* begin file include/simdjson.h */ #ifndef SIMDJSON_H #define SIMDJSON_H @@ -43,7 +43,7 @@ #define SIMDJSON_SIMDJSON_VERSION_H /** The version of simdjson being used (major.minor.revision) */ -#define SIMDJSON_VERSION 2.1.0 +#define SIMDJSON_VERSION 2.2.0 namespace simdjson { enum { @@ -54,7 +54,7 @@ enum { /** * The minor version (major.MINOR.revision) of simdjson being used. */ - SIMDJSON_VERSION_MINOR = 1, + SIMDJSON_VERSION_MINOR = 2, /** * The revision (major.minor.REVISION) of simdjson being used. */ @@ -329,12 +329,12 @@ constexpr size_t SIMDJSON_MAXSIZE_BYTES = 0xFFFFFFFF; /** * The amount of padding needed in a buffer to parse JSON. * - * the input buf should be readable up to buf + SIMDJSON_PADDING + * The input buf should be readable up to buf + SIMDJSON_PADDING * this is a stopgap; there should be a better description of the * main loop and its behavior that abstracts over this * See https://github.com/simdjson/simdjson/issues/174 */ -constexpr size_t SIMDJSON_PADDING = 32; +constexpr size_t SIMDJSON_PADDING = 64; /** * By default, simdjson supports this many nested objects and arrays. @@ -9711,6 +9711,8 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth( // you use visual studio or other compilers. #include +static_assert(sizeof(uint8x16_t) <= simdjson::SIMDJSON_PADDING, "insufficient padding for arm64"); + #endif // SIMDJSON_ARM64_INTRINSICS_H /* end file include/simdjson/arm64/intrinsics.h */ /* begin file include/simdjson/arm64/bitmanipulation.h */ @@ -13724,6 +13726,8 @@ public: #endif // _blsr_u64 #endif // SIMDJSON_CLANG_VISUAL_STUDIO +static_assert(sizeof(__m512i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for icelake"); + #endif // SIMDJSON_ICELAKE_INTRINSICS_H /* end file include/simdjson/icelake/intrinsics.h */ @@ -15914,6 +15918,8 @@ public: #endif // _blsr_u64 #endif // SIMDJSON_CLANG_VISUAL_STUDIO +static_assert(sizeof(__m256i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for haswell kernel."); + #endif // SIMDJSON_HASWELL_INTRINSICS_H /* end file include/simdjson/haswell/intrinsics.h */ @@ -18148,6 +18154,8 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth( #undef vector #endif +static_assert(sizeof(__vector unsigned char) <= simdjson::SIMDJSON_PADDING, "insufficient padding for ppc64"); + #endif // SIMDJSON_PPC64_INTRINSICS_H /* end file include/simdjson/ppc64/intrinsics.h */ /* begin file include/simdjson/ppc64/bitmanipulation.h */ @@ -20369,7 +20377,7 @@ public: #include // for _mm_clmulepi64_si128 #endif - +static_assert(sizeof(__m128i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for westmere"); #endif // SIMDJSON_WESTMERE_INTRINSICS_H /* end file include/simdjson/westmere/intrinsics.h */ diff --git a/src/arm64/dom_parser_implementation.cpp b/src/arm64/dom_parser_implementation.cpp index 13ab024b1..b7dffaabb 100644 --- a/src/arm64/dom_parser_implementation.cpp +++ b/src/arm64/dom_parser_implementation.cpp @@ -76,13 +76,13 @@ simdjson_really_inline json_character_block json_character_block::classify(const simdjson_really_inline bool is_ascii(const simd8x64& input) { simd8 bits = input.reduce_or(); - return bits.max_val() < 0b10000000u; + return bits.max_val() < 0x80u; } simdjson_unused simdjson_really_inline simd8 must_be_continuation(const simd8 prev1, const simd8 prev2, const simd8 prev3) { - simd8 is_second_byte = prev1 >= uint8_t(0b11000000u); - simd8 is_third_byte = prev2 >= uint8_t(0b11100000u); - simd8 is_fourth_byte = prev3 >= uint8_t(0b11110000u); + simd8 is_second_byte = prev1 >= uint8_t(0xc0u); + simd8 is_third_byte = prev2 >= uint8_t(0xe0u); + simd8 is_fourth_byte = prev3 >= uint8_t(0xf0u); // Use ^ instead of | for is_*_byte, because ^ is commutative, and the caller is using ^ as well. // This will work fine because we only have to report errors for cases with 0-1 lead bytes. // Multiple lead bytes implies 2 overlapping multibyte characters, and if that happens, there is @@ -92,8 +92,8 @@ simdjson_unused simdjson_really_inline simd8 must_be_continuation(const si } simdjson_really_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { - simd8 is_third_byte = prev2 >= uint8_t(0b11100000u); - simd8 is_fourth_byte = prev3 >= uint8_t(0b11110000u); + simd8 is_third_byte = prev2 >= uint8_t(0xe0u); + simd8 is_fourth_byte = prev3 >= uint8_t(0xf0u); return is_third_byte ^ is_fourth_byte; } diff --git a/src/fallback/dom_parser_implementation.cpp b/src/fallback/dom_parser_implementation.cpp index dc4a48fd8..11e2287c6 100644 --- a/src/fallback/dom_parser_implementation.cpp +++ b/src/fallback/dom_parser_implementation.cpp @@ -27,12 +27,12 @@ simdjson_really_inline void add_structural() { } simdjson_really_inline bool is_continuation(uint8_t c) { - return (c & 0b11000000) == 0b10000000; + return (c & 0xc0) == 0x80; } simdjson_really_inline void validate_utf8_character() { // Continuation - if (simdjson_unlikely((buf[idx] & 0b01000000) == 0)) { + if (simdjson_unlikely((buf[idx] & 0x40) == 0)) { // extra continuation error = UTF8_ERROR; idx++; @@ -40,7 +40,7 @@ simdjson_really_inline void validate_utf8_character() { } // 2-byte - if ((buf[idx] & 0b00100000) == 0) { + if ((buf[idx] & 0x20) == 0) { // missing continuation if (simdjson_unlikely(idx+1 > len || !is_continuation(buf[idx+1]))) { if (idx+1 > len && is_streaming(partial)) { idx = len; return; } @@ -49,13 +49,13 @@ simdjson_really_inline void validate_utf8_character() { return; } // overlong: 1100000_ 10______ - if (buf[idx] <= 0b11000001) { error = UTF8_ERROR; } + if (buf[idx] <= 0xc1) { error = UTF8_ERROR; } idx += 2; return; } // 3-byte - if ((buf[idx] & 0b00010000) == 0) { + if ((buf[idx] & 0x10) == 0) { // missing continuation if (simdjson_unlikely(idx+2 > len || !is_continuation(buf[idx+1]) || !is_continuation(buf[idx+2]))) { if (idx+2 > len && is_streaming(partial)) { idx = len; return; } @@ -64,9 +64,9 @@ simdjson_really_inline void validate_utf8_character() { return; } // overlong: 11100000 100_____ ________ - if (buf[idx] == 0b11100000 && buf[idx+1] <= 0b10011111) { error = UTF8_ERROR; } + if (buf[idx] == 0xe0 && buf[idx+1] <= 0x9f) { error = UTF8_ERROR; } // surrogates: U+D800-U+DFFF 11101101 101_____ - if (buf[idx] == 0b11101101 && buf[idx+1] >= 0b10100000) { error = UTF8_ERROR; } + if (buf[idx] == 0xed && buf[idx+1] >= 0xa0) { error = UTF8_ERROR; } idx += 3; return; } @@ -80,14 +80,14 @@ simdjson_really_inline void validate_utf8_character() { return; } // overlong: 11110000 1000____ ________ ________ - if (buf[idx] == 0b11110000 && buf[idx+1] <= 0b10001111) { error = UTF8_ERROR; } + if (buf[idx] == 0xf0 && buf[idx+1] <= 0x8f) { error = UTF8_ERROR; } // too large: > U+10FFFF: // 11110100 (1001|101_)____ // 1111(1___|011_|0101) 10______ // also includes 5, 6, 7 and 8 byte characters: // 11111___ - if (buf[idx] == 0b11110100 && buf[idx+1] >= 0b10010000) { error = UTF8_ERROR; } - if (buf[idx] >= 0b11110101) { error = UTF8_ERROR; } + if (buf[idx] == 0xf4 && buf[idx+1] >= 0x90) { error = UTF8_ERROR; } + if (buf[idx] >= 0xf5) { error = UTF8_ERROR; } idx += 4; } @@ -97,7 +97,7 @@ simdjson_really_inline bool validate_string() { while (idx < len && buf[idx] != '"') { if (buf[idx] == '\\') { idx += 2; - } else if (simdjson_unlikely(buf[idx] & 0b10000000)) { + } else if (simdjson_unlikely(buf[idx] & 0x80)) { validate_utf8_character(); } else { if (buf[idx] < 0x20) { error = UNESCAPED_CHARS; } @@ -297,39 +297,39 @@ simdjson_warn_unused bool implementation::validate_utf8(const char *buf, size_t } } unsigned char byte = data[pos]; - if (byte < 0b10000000) { + if (byte < 0x80) { pos++; continue; - } else if ((byte & 0b11100000) == 0b11000000) { + } else if ((byte & 0xe0) == 0xc0) { next_pos = pos + 2; if (next_pos > len) { return false; } - if ((data[pos + 1] & 0b11000000) != 0b10000000) { return false; } + if ((data[pos + 1] & 0xc0) != 0x80) { return false; } // range check - code_point = (byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111); + code_point = (byte & 0x1f) << 6 | (data[pos + 1] & 0x3f); if (code_point < 0x80 || 0x7ff < code_point) { return false; } - } else if ((byte & 0b11110000) == 0b11100000) { + } else if ((byte & 0xf0) == 0xe0) { next_pos = pos + 3; if (next_pos > len) { return false; } - if ((data[pos + 1] & 0b11000000) != 0b10000000) { return false; } - if ((data[pos + 2] & 0b11000000) != 0b10000000) { return false; } + if ((data[pos + 1] & 0xc0) != 0x80) { return false; } + if ((data[pos + 2] & 0xc0) != 0x80) { return false; } // range check - code_point = (byte & 0b00001111) << 12 | - (data[pos + 1] & 0b00111111) << 6 | - (data[pos + 2] & 0b00111111); + code_point = (byte & 0x0f) << 12 | + (data[pos + 1] & 0x3f) << 6 | + (data[pos + 2] & 0x3f); if (code_point < 0x800 || 0xffff < code_point || (0xd7ff < code_point && code_point < 0xe000)) { return false; } - } else if ((byte & 0b11111000) == 0b11110000) { // 0b11110000 + } else if ((byte & 0xf8) == 0xf0) { // 0b11110000 next_pos = pos + 4; if (next_pos > len) { return false; } - if ((data[pos + 1] & 0b11000000) != 0b10000000) { return false; } - if ((data[pos + 2] & 0b11000000) != 0b10000000) { return false; } - if ((data[pos + 3] & 0b11000000) != 0b10000000) { return false; } + if ((data[pos + 1] & 0xc0) != 0x80) { return false; } + if ((data[pos + 2] & 0xc0) != 0x80) { return false; } + if ((data[pos + 3] & 0xc0) != 0x80) { return false; } // range check code_point = - (byte & 0b00000111) << 18 | (data[pos + 1] & 0b00111111) << 12 | - (data[pos + 2] & 0b00111111) << 6 | (data[pos + 3] & 0b00111111); + (byte & 0x07) << 18 | (data[pos + 1] & 0x3f) << 12 | + (data[pos + 2] & 0x3f) << 6 | (data[pos + 3] & 0x3f); if (code_point <= 0xffff || 0x10ffff < code_point) { return false; } } else { // we may have a continuation diff --git a/src/generic/stage1/json_structural_indexer.h b/src/generic/stage1/json_structural_indexer.h index f3db24b97..960f7a9be 100644 --- a/src/generic/stage1/json_structural_indexer.h +++ b/src/generic/stage1/json_structural_indexer.h @@ -160,19 +160,19 @@ simdjson_really_inline size_t trim_partial_utf8(const uint8_t *buf, size_t len) if (simdjson_unlikely(len < 3)) { switch (len) { case 2: - if (buf[len-1] >= 0b11000000) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left - if (buf[len-2] >= 0b11100000) { return len-2; } // 3- and 4-byte characters with only 2 bytes left + if (buf[len-1] >= 0xc0) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left + if (buf[len-2] >= 0xe0) { return len-2; } // 3- and 4-byte characters with only 2 bytes left return len; case 1: - if (buf[len-1] >= 0b11000000) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left + if (buf[len-1] >= 0xc0) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left return len; case 0: return len; } } - if (buf[len-1] >= 0b11000000) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left - if (buf[len-2] >= 0b11100000) { return len-2; } // 3- and 4-byte characters with only 1 byte left - if (buf[len-3] >= 0b11110000) { return len-3; } // 4-byte characters with only 3 bytes left + if (buf[len-1] >= 0xc0) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left + if (buf[len-2] >= 0xe0) { return len-2; } // 3- and 4-byte characters with only 1 byte left + if (buf[len-3] >= 0xf0) { return len-3; } // 4-byte characters with only 3 bytes left return len; } diff --git a/src/generic/stage1/utf8_lookup4_algorithm.h b/src/generic/stage1/utf8_lookup4_algorithm.h index 7d44419aa..88ee0051e 100644 --- a/src/generic/stage1/utf8_lookup4_algorithm.h +++ b/src/generic/stage1/utf8_lookup4_algorithm.h @@ -120,14 +120,14 @@ using namespace simd; 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 0b11110000u-1, 0b11100000u-1, 0b11000000u-1 + 255, 255, 255, 255, 255, 0xf0u-1, 0xe0u-1, 0xc0u-1 }; #else static const uint8_t max_array[32] = { 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 0b11110000u-1, 0b11100000u-1, 0b11000000u-1 + 255, 255, 255, 255, 255, 0xf0u-1, 0xe0u-1, 0xc0u-1 }; #endif const simd8 max_value(&max_array[sizeof(max_array)-sizeof(simd8)]); diff --git a/src/haswell/dom_parser_implementation.cpp b/src/haswell/dom_parser_implementation.cpp index dee564204..9f30605f0 100644 --- a/src/haswell/dom_parser_implementation.cpp +++ b/src/haswell/dom_parser_implementation.cpp @@ -87,16 +87,16 @@ simdjson_really_inline bool is_ascii(const simd8x64& input) { } simdjson_unused simdjson_really_inline simd8 must_be_continuation(const simd8 prev1, const simd8 prev2, const simd8 prev3) { - simd8 is_second_byte = prev1.saturating_sub(0b11000000u-1); // Only 11______ will be > 0 - simd8 is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0 - simd8 is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0 + simd8 is_second_byte = prev1.saturating_sub(0xc0u-1); // Only 11______ will be > 0 + simd8 is_third_byte = prev2.saturating_sub(0xe0u-1); // Only 111_____ will be > 0 + simd8 is_fourth_byte = prev3.saturating_sub(0xf0u-1); // Only 1111____ will be > 0 // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine. return simd8(is_second_byte | is_third_byte | is_fourth_byte) > int8_t(0); } simdjson_really_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { - simd8 is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0 - simd8 is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0 + simd8 is_third_byte = prev2.saturating_sub(0xe0u-1); // Only 111_____ will be > 0 + simd8 is_fourth_byte = prev3.saturating_sub(0xf0u-1); // Only 1111____ will be > 0 // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine. return simd8(is_third_byte | is_fourth_byte) > int8_t(0); } diff --git a/src/icelake/dom_parser_implementation.cpp b/src/icelake/dom_parser_implementation.cpp index 5ba675648..30b5363fc 100644 --- a/src/icelake/dom_parser_implementation.cpp +++ b/src/icelake/dom_parser_implementation.cpp @@ -84,16 +84,16 @@ simdjson_really_inline bool is_ascii(const simd8x64& input) { } simdjson_unused simdjson_really_inline simd8 must_be_continuation(const simd8 prev1, const simd8 prev2, const simd8 prev3) { - simd8 is_second_byte = prev1.saturating_sub(0b11000000u-1); // Only 11______ will be > 0 - simd8 is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0 - simd8 is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0 + simd8 is_second_byte = prev1.saturating_sub(0xc0u-1); // Only 11______ will be > 0 + simd8 is_third_byte = prev2.saturating_sub(0xe0u-1); // Only 111_____ will be > 0 + simd8 is_fourth_byte = prev3.saturating_sub(0xf0u-1); // Only 1111____ will be > 0 // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine. return simd8(is_second_byte | is_third_byte | is_fourth_byte) > int8_t(0); } simdjson_really_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { - simd8 is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0 - simd8 is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0 + simd8 is_third_byte = prev2.saturating_sub(0xe0u-1); // Only 111_____ will be > 0 + simd8 is_fourth_byte = prev3.saturating_sub(0xf0u-1); // Only 1111____ will be > 0 // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine. return simd8(is_third_byte | is_fourth_byte) > int8_t(0); } diff --git a/src/ppc64/dom_parser_implementation.cpp b/src/ppc64/dom_parser_implementation.cpp index 6d650e9cb..58bbd1cd3 100644 --- a/src/ppc64/dom_parser_implementation.cpp +++ b/src/ppc64/dom_parser_implementation.cpp @@ -50,20 +50,20 @@ simdjson_really_inline json_character_block json_character_block::classify(const simdjson_really_inline bool is_ascii(const simd8x64& input) { // careful: 0x80 is not ascii. - return input.reduce_or().saturating_sub(0b01111111u).bits_not_set_anywhere(); + return input.reduce_or().saturating_sub(0x7fu).bits_not_set_anywhere(); } simdjson_unused simdjson_really_inline simd8 must_be_continuation(const simd8 prev1, const simd8 prev2, const simd8 prev3) { - simd8 is_second_byte = prev1.saturating_sub(0b11000000u-1); // Only 11______ will be > 0 - simd8 is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0 - simd8 is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0 + simd8 is_second_byte = prev1.saturating_sub(0xc0u-1); // Only 11______ will be > 0 + simd8 is_third_byte = prev2.saturating_sub(0xe0u-1); // Only 111_____ will be > 0 + simd8 is_fourth_byte = prev3.saturating_sub(0xf0u-1); // Only 1111____ will be > 0 // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine. return simd8(is_second_byte | is_third_byte | is_fourth_byte) > int8_t(0); } simdjson_really_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { - simd8 is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0 - simd8 is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0 + simd8 is_third_byte = prev2.saturating_sub(0xe0u-1); // Only 111_____ will be > 0 + simd8 is_fourth_byte = prev3.saturating_sub(0xf0u-1); // Only 1111____ will be > 0 // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine. return simd8(is_third_byte | is_fourth_byte) > int8_t(0); } diff --git a/src/westmere/dom_parser_implementation.cpp b/src/westmere/dom_parser_implementation.cpp index fe3b2a321..2bfdc9a8f 100644 --- a/src/westmere/dom_parser_implementation.cpp +++ b/src/westmere/dom_parser_implementation.cpp @@ -85,16 +85,16 @@ simdjson_really_inline bool is_ascii(const simd8x64& input) { } simdjson_unused simdjson_really_inline simd8 must_be_continuation(const simd8 prev1, const simd8 prev2, const simd8 prev3) { - simd8 is_second_byte = prev1.saturating_sub(0b11000000u-1); // Only 11______ will be > 0 - simd8 is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0 - simd8 is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0 + simd8 is_second_byte = prev1.saturating_sub(0xc0u-1); // Only 11______ will be > 0 + simd8 is_third_byte = prev2.saturating_sub(0xe0u-1); // Only 111_____ will be > 0 + simd8 is_fourth_byte = prev3.saturating_sub(0xf0u-1); // Only 1111____ will be > 0 // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine. return simd8(is_second_byte | is_third_byte | is_fourth_byte) > int8_t(0); } simdjson_really_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { - simd8 is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0 - simd8 is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0 + simd8 is_third_byte = prev2.saturating_sub(0xe0u-1); // Only 111_____ will be > 0 + simd8 is_fourth_byte = prev3.saturating_sub(0xf0u-1); // Only 1111____ will be > 0 // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine. return simd8(is_third_byte | is_fourth_byte) > int8_t(0); } diff --git a/tests/dom/errortests.cpp b/tests/dom/errortests.cpp index 6f12f7270..16bbd72f0 100644 --- a/tests/dom/errortests.cpp +++ b/tests/dom/errortests.cpp @@ -146,7 +146,7 @@ namespace parser_load { } namespace adversarial { - #define PADDING_FILLED_WITH_NUMBERS "222222222222222222222222222222222" + #define PADDING_FILLED_WITH_NUMBERS "22222222222222222222222222222222222222222222222222222222222222222" bool number_overrun_at_root() { TEST_START(); constexpr const char *json = "1" PADDING_FILLED_WITH_NUMBERS ","; @@ -179,8 +179,9 @@ namespace adversarial { TEST_SUCCEED(); } bool run() { - static_assert(33 > SIMDJSON_PADDING, "corruption test doesn't have enough padding"); // 33 = std::strlen(PADDING_FILLED_WITH_NUMBERS) - return true + constexpr size_t filler_size = 65; + static_assert(filler_size > SIMDJSON_PADDING, "corruption test doesn't have enough padding"); // 33 = std::strlen(PADDING_FILLED_WITH_NUMBERS) + return (std::strlen(PADDING_FILLED_WITH_NUMBERS) == filler_size) && number_overrun_at_root() && number_overrun_in_array() && number_overrun_in_object() diff --git a/tests/ondemand/ondemand_parse_api_tests.cpp b/tests/ondemand/ondemand_parse_api_tests.cpp index d8a4570ec..09b4cd900 100644 --- a/tests/ondemand/ondemand_parse_api_tests.cpp +++ b/tests/ondemand/ondemand_parse_api_tests.cpp @@ -46,8 +46,8 @@ namespace parse_api_tests { bool parser_iterate_padded() { TEST_START(); ondemand::parser parser; - const char json_str[] = "12\0 "; // 32 padding - ASSERT_EQUAL(sizeof(json_str), 34); + const char json_str[] = "12\0 ";// 64 bytes of padding + ASSERT_EQUAL(sizeof(json_str), 66); ASSERT_EQUAL(strlen(json_str), 2); { @@ -84,8 +84,8 @@ namespace parse_api_tests { bool parser_iterate_padded_string_view() { TEST_START(); ondemand::parser parser; - const char json_str[] = "12\0 "; // 32 padding - ASSERT_EQUAL(sizeof(json_str), 34); + const char json_str[] = "12\0 "; // 64 bytes of padding + ASSERT_EQUAL(sizeof(json_str), 66); ASSERT_EQUAL(strlen(json_str), 2); { @@ -122,14 +122,14 @@ namespace parse_api_tests { bool parser_iterate_insufficient_padding() { TEST_START(); ondemand::parser parser; - constexpr char json_str[] = "12\0 "; // 31 padding - ASSERT_EQUAL(sizeof(json_str), 33); + constexpr char json_str[] = "12\0 "; // 63 bytes of padding + ASSERT_EQUAL(sizeof(json_str), 65); ASSERT_EQUAL(strlen(json_str), 2); - ASSERT_EQUAL(padded_string_view(json_str, strlen(json_str), sizeof(json_str)).padding(), 31); - ASSERT_EQUAL(SIMDJSON_PADDING, 32); + ASSERT_EQUAL(padded_string_view(json_str, strlen(json_str), sizeof(json_str)).padding(), 63); + ASSERT_EQUAL(SIMDJSON_PADDING, 64); { - cout << "- char*, 31 padding" << endl; + cout << "- char*, 63 padding" << endl; ASSERT_ERROR( parser.iterate(json_str, strlen(json_str), sizeof(json_str)), INSUFFICIENT_PADDING ); cout << "- char*, 0 padding" << endl; ASSERT_ERROR( parser.iterate(json_str, strlen(json_str), strlen(json_str)), INSUFFICIENT_PADDING ); @@ -137,7 +137,7 @@ namespace parse_api_tests { { std::string_view json(json_str); - cout << "- string_view, 31 padding" << endl; + cout << "- string_view, 63 padding" << endl; ASSERT_ERROR( parser.iterate(json, sizeof(json_str)), INSUFFICIENT_PADDING ); cout << "- string_view, 0 padding" << endl; ASSERT_ERROR( parser.iterate(json, strlen(json_str)), INSUFFICIENT_PADDING );