admin/simdjson-simdjson
1
0
Fork 0
mirror of https://github.com/simdjson/simdjson synced 2026-06-08 17:27:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

New release candidate (#1856)

Browse Source 
* Patch for possible AVX-512 overflow.

* Updating the test for new padding.

* Preparing new version.

* replace binary integer literals with hex literals for C++11 compatibility (#1855)

Binary integer literals are a C++14 feature, so those are not supported
in C++11 and should be replaced by hexadecimal literals instead.

Fixes #1854.

Co-authored-by: Dirk Stolle <striezel-dev@web.de>
This commit is contained in:
Daniel Lemire
2022-07-04 21:54:19 -04:00
committed by GitHub
parent 62bdb9a2f7
commit 3fde8a4eac
22 changed files with 110 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CMakeLists.txt
+3 -3
View File
@@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.14)
project(
simdjson
# The version number is modified by tools/release.py
VERSION 2.1.0
VERSION 2.2.0
DESCRIPTION "Parsing gigabytes of JSON per second"
HOMEPAGE_URL "https://simdjson.org/"
LANGUAGES CXX C
@@ -20,8 +20,8 @@ string(
# ---- Options, variables ----
# These version numbers are modified by tools/release.py
set(SIMDJSON_LIB_VERSION "12.0.0" CACHE STRING "simdjson library version")
set(SIMDJSON_LIB_SOVERSION "12" CACHE STRING "simdjson library soversion")
set(SIMDJSON_LIB_VERSION "13.0.0" CACHE STRING "simdjson library version")
set(SIMDJSON_LIB_SOVERSION "13" CACHE STRING "simdjson library soversion")
option(SIMDJSON_ENABLE_THREADS "Link with thread support" ON)
Doxyfile
+1 -1
View File
@@ -38,7 +38,7 @@ PROJECT_NAME = simdjson
# could be handy for archiving the generated documentation or if some version
# control system is used.
PROJECT_NUMBER = "2.1.0"
PROJECT_NUMBER = "2.2.0"
# Using the PROJECT_BRIEF tag one can provide an optional one line description
# for a project that appears at the top of each page and should give viewer a
benchmark/json2msgpack/simdjson_ondemand.h
+2 -2
View File
@@ -34,8 +34,8 @@ struct simdjson2msgpack {
* @brief Converts the provided JSON into msgpack.
*
* @param json JSON input
* @param buf temporary buffer (must be large enough, with 32 bytes of
* padding)
* @param buf temporary buffer (must be large enough, with simdjson::SIMDJSON_PADDING bytes
* of padding)
* @return std::string_view msgpack output, writting to the temporary buffer
*/
inline std::string_view to_msgpack(const simdjson::padded_string &json,
include/simdjson/arm64/intrinsics.h
+2
View File
@@ -5,4 +5,6 @@
// you use visual studio or other compilers.
#include <arm_neon.h>
static_assert(sizeof(uint8x16_t) <= simdjson::SIMDJSON_PADDING, "insufficient padding for arm64");
#endif // SIMDJSON_ARM64_INTRINSICS_H
include/simdjson/common_defs.h
+2 -2
View File
@@ -37,12 +37,12 @@ constexpr size_t SIMDJSON_MAXSIZE_BYTES = 0xFFFFFFFF;
/**
* The amount of padding needed in a buffer to parse JSON.
*
* the input buf should be readable up to buf + SIMDJSON_PADDING
* The input buf should be readable up to buf + SIMDJSON_PADDING
* this is a stopgap; there should be a better description of the
* main loop and its behavior that abstracts over this
* See https://github.com/simdjson/simdjson/issues/174
*/
constexpr size_t SIMDJSON_PADDING = 32;
constexpr size_t SIMDJSON_PADDING = 64;
/**
* By default, simdjson supports this many nested objects and arrays.
include/simdjson/haswell/intrinsics.h
+2
View File
@@ -45,4 +45,6 @@
#endif // _blsr_u64
#endif // SIMDJSON_CLANG_VISUAL_STUDIO
static_assert(sizeof(__m256i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for haswell kernel.");
#endif // SIMDJSON_HASWELL_INTRINSICS_H
include/simdjson/icelake/intrinsics.h
+2
View File
@@ -53,4 +53,6 @@
#endif // _blsr_u64
#endif // SIMDJSON_CLANG_VISUAL_STUDIO
static_assert(sizeof(__m512i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for icelake");
#endif // SIMDJSON_ICELAKE_INTRINSICS_H
include/simdjson/ppc64/intrinsics.h
+2
View File
@@ -16,4 +16,6 @@
#undef vector
#endif
static_assert(sizeof(__vector unsigned char) <= simdjson::SIMDJSON_PADDING, "insufficient padding for ppc64");
#endif // SIMDJSON_PPC64_INTRINSICS_H
include/simdjson/simdjson_version.h
+2 -2
View File
@@ -4,7 +4,7 @@
#define SIMDJSON_SIMDJSON_VERSION_H
/** The version of simdjson being used (major.minor.revision) */
#define SIMDJSON_VERSION 2.1.0
#define SIMDJSON_VERSION 2.2.0
namespace simdjson {
enum {
@@ -15,7 +15,7 @@ enum {
/**
* The minor version (major.MINOR.revision) of simdjson being used.
*/
SIMDJSON_VERSION_MINOR = 1,
SIMDJSON_VERSION_MINOR = 2,
/**
* The revision (major.minor.REVISION) of simdjson being used.
*/
include/simdjson/westmere/intrinsics.h
+1 -1
View File
@@ -22,6 +22,6 @@
#include <wmmintrin.h> // for _mm_clmulepi64_si128
#endif
static_assert(sizeof(__m128i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for westmere");
#endif // SIMDJSON_WESTMERE_INTRINSICS_H
singleheader/simdjson.cpp
+1 -1
View File
@@ -1,4 +1,4 @@
/* auto-generated on 2022-06-30 10:29:50 -0400. Do not edit! */
/* auto-generated on 2022-07-04 20:04:25 -0400. Do not edit! */
/* begin file src/simdjson.cpp */
#include "simdjson.h"
singleheader/simdjson.h
+14 -6
View File
@@ -1,4 +1,4 @@
/* auto-generated on 2022-06-30 10:29:50 -0400. Do not edit! */
/* auto-generated on 2022-07-04 20:04:25 -0400. Do not edit! */
/* begin file include/simdjson.h */
#ifndef SIMDJSON_H
#define SIMDJSON_H
@@ -43,7 +43,7 @@
#define SIMDJSON_SIMDJSON_VERSION_H
/** The version of simdjson being used (major.minor.revision) */
#define SIMDJSON_VERSION 2.1.0
#define SIMDJSON_VERSION 2.2.0
namespace simdjson {
enum {
@@ -54,7 +54,7 @@ enum {
/**
* The minor version (major.MINOR.revision) of simdjson being used.
*/
SIMDJSON_VERSION_MINOR = 1,
SIMDJSON_VERSION_MINOR = 2,
/**
* The revision (major.minor.REVISION) of simdjson being used.
*/
@@ -329,12 +329,12 @@ constexpr size_t SIMDJSON_MAXSIZE_BYTES = 0xFFFFFFFF;
/**
* The amount of padding needed in a buffer to parse JSON.
*
* the input buf should be readable up to buf + SIMDJSON_PADDING
* The input buf should be readable up to buf + SIMDJSON_PADDING
* this is a stopgap; there should be a better description of the
* main loop and its behavior that abstracts over this
* See https://github.com/simdjson/simdjson/issues/174
*/
constexpr size_t SIMDJSON_PADDING = 32;
constexpr size_t SIMDJSON_PADDING = 64;
/**
* By default, simdjson supports this many nested objects and arrays.
@@ -9711,6 +9711,8 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(
// you use visual studio or other compilers.
#include <arm_neon.h>
static_assert(sizeof(uint8x16_t) <= simdjson::SIMDJSON_PADDING, "insufficient padding for arm64");
#endif // SIMDJSON_ARM64_INTRINSICS_H
/* end file include/simdjson/arm64/intrinsics.h */
/* begin file include/simdjson/arm64/bitmanipulation.h */
@@ -13724,6 +13726,8 @@ public:
#endif // _blsr_u64
#endif // SIMDJSON_CLANG_VISUAL_STUDIO
static_assert(sizeof(__m512i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for icelake");
#endif // SIMDJSON_ICELAKE_INTRINSICS_H
/* end file include/simdjson/icelake/intrinsics.h */
@@ -15914,6 +15918,8 @@ public:
#endif // _blsr_u64
#endif // SIMDJSON_CLANG_VISUAL_STUDIO
static_assert(sizeof(__m256i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for haswell kernel.");
#endif // SIMDJSON_HASWELL_INTRINSICS_H
/* end file include/simdjson/haswell/intrinsics.h */
@@ -18148,6 +18154,8 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(
#undef vector
#endif
static_assert(sizeof(__vector unsigned char) <= simdjson::SIMDJSON_PADDING, "insufficient padding for ppc64");
#endif // SIMDJSON_PPC64_INTRINSICS_H
/* end file include/simdjson/ppc64/intrinsics.h */
/* begin file include/simdjson/ppc64/bitmanipulation.h */
@@ -20369,7 +20377,7 @@ public:
#include <wmmintrin.h> // for _mm_clmulepi64_si128
#endif
static_assert(sizeof(__m128i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for westmere");
#endif // SIMDJSON_WESTMERE_INTRINSICS_H
/* end file include/simdjson/westmere/intrinsics.h */
src/arm64/dom_parser_implementation.cpp
+6 -6
View File
@@ -76,13 +76,13 @@ simdjson_really_inline json_character_block json_character_block::classify(const
simdjson_really_inline bool is_ascii(const simd8x64<uint8_t>& input) {
simd8<uint8_t> bits = input.reduce_or();
return bits.max_val() < 0b10000000u;
return bits.max_val() < 0x80u;
}
simdjson_unused simdjson_really_inline simd8<bool> must_be_continuation(const simd8<uint8_t> prev1, const simd8<uint8_t> prev2, const simd8<uint8_t> prev3) {
simd8<bool> is_second_byte = prev1 >= uint8_t(0b11000000u);
simd8<bool> is_third_byte = prev2 >= uint8_t(0b11100000u);
simd8<bool> is_fourth_byte = prev3 >= uint8_t(0b11110000u);
simd8<bool> is_second_byte = prev1 >= uint8_t(0xc0u);
simd8<bool> is_third_byte = prev2 >= uint8_t(0xe0u);
simd8<bool> is_fourth_byte = prev3 >= uint8_t(0xf0u);
// Use ^ instead of | for is_*_byte, because ^ is commutative, and the caller is using ^ as well.
// This will work fine because we only have to report errors for cases with 0-1 lead bytes.
// Multiple lead bytes implies 2 overlapping multibyte characters, and if that happens, there is
@@ -92,8 +92,8 @@ simdjson_unused simdjson_really_inline simd8<bool> must_be_continuation(const si
}
simdjson_really_inline simd8<bool> must_be_2_3_continuation(const simd8<uint8_t> prev2, const simd8<uint8_t> prev3) {
simd8<bool> is_third_byte = prev2 >= uint8_t(0b11100000u);
simd8<bool> is_fourth_byte = prev3 >= uint8_t(0b11110000u);
simd8<bool> is_third_byte = prev2 >= uint8_t(0xe0u);
simd8<bool> is_fourth_byte = prev3 >= uint8_t(0xf0u);
return is_third_byte ^ is_fourth_byte;
}
src/fallback/dom_parser_implementation.cpp
+27 -27
View File
@@ -27,12 +27,12 @@ simdjson_really_inline void add_structural() {
}
simdjson_really_inline bool is_continuation(uint8_t c) {
return (c & 0b11000000) == 0b10000000;
return (c & 0xc0) == 0x80;
}
simdjson_really_inline void validate_utf8_character() {
// Continuation
if (simdjson_unlikely((buf[idx] & 0b01000000) == 0)) {
if (simdjson_unlikely((buf[idx] & 0x40) == 0)) {
// extra continuation
error = UTF8_ERROR;
idx++;
@@ -40,7 +40,7 @@ simdjson_really_inline void validate_utf8_character() {
}
// 2-byte
if ((buf[idx] & 0b00100000) == 0) {
if ((buf[idx] & 0x20) == 0) {
// missing continuation
if (simdjson_unlikely(idx+1 > len || !is_continuation(buf[idx+1]))) {
if (idx+1 > len && is_streaming(partial)) { idx = len; return; }
@@ -49,13 +49,13 @@ simdjson_really_inline void validate_utf8_character() {
return;
}
// overlong: 1100000_ 10______
if (buf[idx] <= 0b11000001) { error = UTF8_ERROR; }
if (buf[idx] <= 0xc1) { error = UTF8_ERROR; }
idx += 2;
return;
}
// 3-byte
if ((buf[idx] & 0b00010000) == 0) {
if ((buf[idx] & 0x10) == 0) {
// missing continuation
if (simdjson_unlikely(idx+2 > len || !is_continuation(buf[idx+1]) || !is_continuation(buf[idx+2]))) {
if (idx+2 > len && is_streaming(partial)) { idx = len; return; }
@@ -64,9 +64,9 @@ simdjson_really_inline void validate_utf8_character() {
return;
}
// overlong: 11100000 100_____ ________
if (buf[idx] == 0b11100000 && buf[idx+1] <= 0b10011111) { error = UTF8_ERROR; }
if (buf[idx] == 0xe0 && buf[idx+1] <= 0x9f) { error = UTF8_ERROR; }
// surrogates: U+D800-U+DFFF 11101101 101_____
if (buf[idx] == 0b11101101 && buf[idx+1] >= 0b10100000) { error = UTF8_ERROR; }
if (buf[idx] == 0xed && buf[idx+1] >= 0xa0) { error = UTF8_ERROR; }
idx += 3;
return;
}
@@ -80,14 +80,14 @@ simdjson_really_inline void validate_utf8_character() {
return;
}
// overlong: 11110000 1000____ ________ ________
if (buf[idx] == 0b11110000 && buf[idx+1] <= 0b10001111) { error = UTF8_ERROR; }
if (buf[idx] == 0xf0 && buf[idx+1] <= 0x8f) { error = UTF8_ERROR; }
// too large: > U+10FFFF:
// 11110100 (1001|101_)____
// 1111(1___|011_|0101) 10______
// also includes 5, 6, 7 and 8 byte characters:
// 11111___
if (buf[idx] == 0b11110100 && buf[idx+1] >= 0b10010000) { error = UTF8_ERROR; }
if (buf[idx] >= 0b11110101) { error = UTF8_ERROR; }
if (buf[idx] == 0xf4 && buf[idx+1] >= 0x90) { error = UTF8_ERROR; }
if (buf[idx] >= 0xf5) { error = UTF8_ERROR; }
idx += 4;
}
@@ -97,7 +97,7 @@ simdjson_really_inline bool validate_string() {
while (idx < len && buf[idx] != '"') {
if (buf[idx] == '\\') {
idx += 2;
} else if (simdjson_unlikely(buf[idx] & 0b10000000)) {
} else if (simdjson_unlikely(buf[idx] & 0x80)) {
validate_utf8_character();
} else {
if (buf[idx] < 0x20) { error = UNESCAPED_CHARS; }
@@ -297,39 +297,39 @@ simdjson_warn_unused bool implementation::validate_utf8(const char *buf, size_t
}
}
unsigned char byte = data[pos];
if (byte < 0b10000000) {
if (byte < 0x80) {
pos++;
continue;
} else if ((byte & 0b11100000) == 0b11000000) {
} else if ((byte & 0xe0) == 0xc0) {
next_pos = pos + 2;
if (next_pos > len) { return false; }
if ((data[pos + 1] & 0b11000000) != 0b10000000) { return false; }
if ((data[pos + 1] & 0xc0) != 0x80) { return false; }
// range check
code_point = (byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111);
code_point = (byte & 0x1f) << 6 | (data[pos + 1] & 0x3f);
if (code_point < 0x80 || 0x7ff < code_point) { return false; }
} else if ((byte & 0b11110000) == 0b11100000) {
} else if ((byte & 0xf0) == 0xe0) {
next_pos = pos + 3;
if (next_pos > len) { return false; }
if ((data[pos + 1] & 0b11000000) != 0b10000000) { return false; }
if ((data[pos + 2] & 0b11000000) != 0b10000000) { return false; }
if ((data[pos + 1] & 0xc0) != 0x80) { return false; }
if ((data[pos + 2] & 0xc0) != 0x80) { return false; }
// range check
code_point = (byte & 0b00001111) << 12 |
(data[pos + 1] & 0b00111111) << 6 |
(data[pos + 2] & 0b00111111);
code_point = (byte & 0x0f) << 12 |
(data[pos + 1] & 0x3f) << 6 |
(data[pos + 2] & 0x3f);
if (code_point < 0x800 || 0xffff < code_point ||
(0xd7ff < code_point && code_point < 0xe000)) {
return false;
}
} else if ((byte & 0b11111000) == 0b11110000) { // 0b11110000
} else if ((byte & 0xf8) == 0xf0) { // 0b11110000
next_pos = pos + 4;
if (next_pos > len) { return false; }
if ((data[pos + 1] & 0b11000000) != 0b10000000) { return false; }
if ((data[pos + 2] & 0b11000000) != 0b10000000) { return false; }
if ((data[pos + 3] & 0b11000000) != 0b10000000) { return false; }
if ((data[pos + 1] & 0xc0) != 0x80) { return false; }
if ((data[pos + 2] & 0xc0) != 0x80) { return false; }
if ((data[pos + 3] & 0xc0) != 0x80) { return false; }
// range check
code_point =
(byte & 0b00000111) << 18 | (data[pos + 1] & 0b00111111) << 12 |
(data[pos + 2] & 0b00111111) << 6 | (data[pos + 3] & 0b00111111);
(byte & 0x07) << 18 | (data[pos + 1] & 0x3f) << 12 |
(data[pos + 2] & 0x3f) << 6 | (data[pos + 3] & 0x3f);
if (code_point <= 0xffff || 0x10ffff < code_point) { return false; }
} else {
// we may have a continuation
src/generic/stage1/json_structural_indexer.h
+6 -6
View File
@@ -160,19 +160,19 @@ simdjson_really_inline size_t trim_partial_utf8(const uint8_t *buf, size_t len)
if (simdjson_unlikely(len < 3)) {
switch (len) {
case 2:
if (buf[len-1] >= 0b11000000) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left
if (buf[len-2] >= 0b11100000) { return len-2; } // 3- and 4-byte characters with only 2 bytes left
if (buf[len-1] >= 0xc0) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left
if (buf[len-2] >= 0xe0) { return len-2; } // 3- and 4-byte characters with only 2 bytes left
return len;
case 1:
if (buf[len-1] >= 0b11000000) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left
if (buf[len-1] >= 0xc0) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left
return len;
case 0:
return len;
}
}
if (buf[len-1] >= 0b11000000) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left
if (buf[len-2] >= 0b11100000) { return len-2; } // 3- and 4-byte characters with only 1 byte left
if (buf[len-3] >= 0b11110000) { return len-3; } // 4-byte characters with only 3 bytes left
if (buf[len-1] >= 0xc0) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left
if (buf[len-2] >= 0xe0) { return len-2; } // 3- and 4-byte characters with only 1 byte left
if (buf[len-3] >= 0xf0) { return len-3; } // 4-byte characters with only 3 bytes left
return len;
}
src/generic/stage1/utf8_lookup4_algorithm.h
+2 -2
View File
@@ -120,14 +120,14 @@ using namespace simd;
255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 0b11110000u-1, 0b11100000u-1, 0b11000000u-1
255, 255, 255, 255, 255, 0xf0u-1, 0xe0u-1, 0xc0u-1
};
#else
static const uint8_t max_array[32] = {
255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 0b11110000u-1, 0b11100000u-1, 0b11000000u-1
255, 255, 255, 255, 255, 0xf0u-1, 0xe0u-1, 0xc0u-1
};
#endif
const simd8<uint8_t> max_value(&max_array[sizeof(max_array)-sizeof(simd8<uint8_t>)]);
src/haswell/dom_parser_implementation.cpp
+5 -5
View File
@@ -87,16 +87,16 @@ simdjson_really_inline bool is_ascii(const simd8x64<uint8_t>& input) {
}
simdjson_unused simdjson_really_inline simd8<bool> must_be_continuation(const simd8<uint8_t> prev1, const simd8<uint8_t> prev2, const simd8<uint8_t> prev3) {
simd8<uint8_t> is_second_byte = prev1.saturating_sub(0b11000000u-1); // Only 11______ will be > 0
simd8<uint8_t> is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0
simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0
simd8<uint8_t> is_second_byte = prev1.saturating_sub(0xc0u-1); // Only 11______ will be > 0
simd8<uint8_t> is_third_byte = prev2.saturating_sub(0xe0u-1); // Only 111_____ will be > 0
simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0xf0u-1); // Only 1111____ will be > 0
// Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine.
return simd8<int8_t>(is_second_byte | is_third_byte | is_fourth_byte) > int8_t(0);
}
simdjson_really_inline simd8<bool> must_be_2_3_continuation(const simd8<uint8_t> prev2, const simd8<uint8_t> prev3) {
simd8<uint8_t> is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0
simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0
simd8<uint8_t> is_third_byte = prev2.saturating_sub(0xe0u-1); // Only 111_____ will be > 0
simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0xf0u-1); // Only 1111____ will be > 0
// Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine.
return simd8<int8_t>(is_third_byte | is_fourth_byte) > int8_t(0);
}
src/icelake/dom_parser_implementation.cpp
+5 -5
View File
@@ -84,16 +84,16 @@ simdjson_really_inline bool is_ascii(const simd8x64<uint8_t>& input) {
}
simdjson_unused simdjson_really_inline simd8<bool> must_be_continuation(const simd8<uint8_t> prev1, const simd8<uint8_t> prev2, const simd8<uint8_t> prev3) {
simd8<uint8_t> is_second_byte = prev1.saturating_sub(0b11000000u-1); // Only 11______ will be > 0
simd8<uint8_t> is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0
simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0
simd8<uint8_t> is_second_byte = prev1.saturating_sub(0xc0u-1); // Only 11______ will be > 0
simd8<uint8_t> is_third_byte = prev2.saturating_sub(0xe0u-1); // Only 111_____ will be > 0
simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0xf0u-1); // Only 1111____ will be > 0
// Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine.
return simd8<int8_t>(is_second_byte | is_third_byte | is_fourth_byte) > int8_t(0);
}
simdjson_really_inline simd8<bool> must_be_2_3_continuation(const simd8<uint8_t> prev2, const simd8<uint8_t> prev3) {
simd8<uint8_t> is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0
simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0
simd8<uint8_t> is_third_byte = prev2.saturating_sub(0xe0u-1); // Only 111_____ will be > 0
simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0xf0u-1); // Only 1111____ will be > 0
// Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine.
return simd8<int8_t>(is_third_byte | is_fourth_byte) > int8_t(0);
}
src/ppc64/dom_parser_implementation.cpp
+6 -6
View File
@@ -50,20 +50,20 @@ simdjson_really_inline json_character_block json_character_block::classify(const
simdjson_really_inline bool is_ascii(const simd8x64<uint8_t>& input) {
// careful: 0x80 is not ascii.
return input.reduce_or().saturating_sub(0b01111111u).bits_not_set_anywhere();
return input.reduce_or().saturating_sub(0x7fu).bits_not_set_anywhere();
}
simdjson_unused simdjson_really_inline simd8<bool> must_be_continuation(const simd8<uint8_t> prev1, const simd8<uint8_t> prev2, const simd8<uint8_t> prev3) {
simd8<uint8_t> is_second_byte = prev1.saturating_sub(0b11000000u-1); // Only 11______ will be > 0
simd8<uint8_t> is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0
simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0
simd8<uint8_t> is_second_byte = prev1.saturating_sub(0xc0u-1); // Only 11______ will be > 0
simd8<uint8_t> is_third_byte = prev2.saturating_sub(0xe0u-1); // Only 111_____ will be > 0
simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0xf0u-1); // Only 1111____ will be > 0
// Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine.
return simd8<int8_t>(is_second_byte | is_third_byte | is_fourth_byte) > int8_t(0);
}
simdjson_really_inline simd8<bool> must_be_2_3_continuation(const simd8<uint8_t> prev2, const simd8<uint8_t> prev3) {
simd8<uint8_t> is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0
simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0
simd8<uint8_t> is_third_byte = prev2.saturating_sub(0xe0u-1); // Only 111_____ will be > 0
simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0xf0u-1); // Only 1111____ will be > 0
// Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine.
return simd8<int8_t>(is_third_byte | is_fourth_byte) > int8_t(0);
}
src/westmere/dom_parser_implementation.cpp
+5 -5
View File
@@ -85,16 +85,16 @@ simdjson_really_inline bool is_ascii(const simd8x64<uint8_t>& input) {
}
simdjson_unused simdjson_really_inline simd8<bool> must_be_continuation(const simd8<uint8_t> prev1, const simd8<uint8_t> prev2, const simd8<uint8_t> prev3) {
simd8<uint8_t> is_second_byte = prev1.saturating_sub(0b11000000u-1); // Only 11______ will be > 0
simd8<uint8_t> is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0
simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0
simd8<uint8_t> is_second_byte = prev1.saturating_sub(0xc0u-1); // Only 11______ will be > 0
simd8<uint8_t> is_third_byte = prev2.saturating_sub(0xe0u-1); // Only 111_____ will be > 0
simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0xf0u-1); // Only 1111____ will be > 0
// Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine.
return simd8<int8_t>(is_second_byte | is_third_byte | is_fourth_byte) > int8_t(0);
}
simdjson_really_inline simd8<bool> must_be_2_3_continuation(const simd8<uint8_t> prev2, const simd8<uint8_t> prev3) {
simd8<uint8_t> is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0
simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0
simd8<uint8_t> is_third_byte = prev2.saturating_sub(0xe0u-1); // Only 111_____ will be > 0
simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0xf0u-1); // Only 1111____ will be > 0
// Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine.
return simd8<int8_t>(is_third_byte | is_fourth_byte) > int8_t(0);
}
tests/dom/errortests.cpp
+4 -3
View File
@@ -146,7 +146,7 @@ namespace parser_load {
}
namespace adversarial {
#define PADDING_FILLED_WITH_NUMBERS "222222222222222222222222222222222"
#define PADDING_FILLED_WITH_NUMBERS "22222222222222222222222222222222222222222222222222222222222222222"
bool number_overrun_at_root() {
TEST_START();
constexpr const char *json = "1" PADDING_FILLED_WITH_NUMBERS ",";
@@ -179,8 +179,9 @@ namespace adversarial {
TEST_SUCCEED();
}
bool run() {
static_assert(33 > SIMDJSON_PADDING, "corruption test doesn't have enough padding"); // 33 = std::strlen(PADDING_FILLED_WITH_NUMBERS)
return true
constexpr size_t filler_size = 65;
static_assert(filler_size > SIMDJSON_PADDING, "corruption test doesn't have enough padding"); // 33 = std::strlen(PADDING_FILLED_WITH_NUMBERS)
return (std::strlen(PADDING_FILLED_WITH_NUMBERS) == filler_size)
&& number_overrun_at_root()
&& number_overrun_in_array()
&& number_overrun_in_object()
tests/ondemand/ondemand_parse_api_tests.cpp
+10 -10
View File
@@ -46,8 +46,8 @@ namespace parse_api_tests {
bool parser_iterate_padded() {
TEST_START();
ondemand::parser parser;
const char json_str[] = "12\0 "; // 32 padding
ASSERT_EQUAL(sizeof(json_str), 34);
const char json_str[] = "12\0 ";// 64 bytes of padding
ASSERT_EQUAL(sizeof(json_str), 66);
ASSERT_EQUAL(strlen(json_str), 2);
{
@@ -84,8 +84,8 @@ namespace parse_api_tests {
bool parser_iterate_padded_string_view() {
TEST_START();
ondemand::parser parser;
const char json_str[] = "12\0 "; // 32 padding
ASSERT_EQUAL(sizeof(json_str), 34);
const char json_str[] = "12\0 "; // 64 bytes of padding
ASSERT_EQUAL(sizeof(json_str), 66);
ASSERT_EQUAL(strlen(json_str), 2);
{
@@ -122,14 +122,14 @@ namespace parse_api_tests {
bool parser_iterate_insufficient_padding() {
TEST_START();
ondemand::parser parser;
constexpr char json_str[] = "12\0 "; // 31 padding
ASSERT_EQUAL(sizeof(json_str), 33);
constexpr char json_str[] = "12\0 "; // 63 bytes of padding
ASSERT_EQUAL(sizeof(json_str), 65);
ASSERT_EQUAL(strlen(json_str), 2);
ASSERT_EQUAL(padded_string_view(json_str, strlen(json_str), sizeof(json_str)).padding(), 31);
ASSERT_EQUAL(SIMDJSON_PADDING, 32);
ASSERT_EQUAL(padded_string_view(json_str, strlen(json_str), sizeof(json_str)).padding(), 63);
ASSERT_EQUAL(SIMDJSON_PADDING, 64);
{
cout << "- char*, 31 padding" << endl;
cout << "- char*, 63 padding" << endl;
ASSERT_ERROR( parser.iterate(json_str, strlen(json_str), sizeof(json_str)), INSUFFICIENT_PADDING );
cout << "- char*, 0 padding" << endl;
ASSERT_ERROR( parser.iterate(json_str, strlen(json_str), strlen(json_str)), INSUFFICIENT_PADDING );
@@ -137,7 +137,7 @@ namespace parse_api_tests {
{
std::string_view json(json_str);
cout << "- string_view, 31 padding" << endl;
cout << "- string_view, 63 padding" << endl;
ASSERT_ERROR( parser.iterate(json, sizeof(json_str)), INSUFFICIENT_PADDING );
cout << "- string_view, 0 padding" << endl;
ASSERT_ERROR( parser.iterate(json, strlen(json_str)), INSUFFICIENT_PADDING );