admin/simdjson-simdjson
1
0
Fork 0
mirror of https://github.com/simdjson/simdjson synced 2026-06-08 17:27:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

New release candidate (#1856)

Browse Source 
* Patch for possible AVX-512 overflow.

* Updating the test for new padding.

* Preparing new version.

* replace binary integer literals with hex literals for C++11 compatibility (#1855)

Binary integer literals are a C++14 feature, so those are not supported
in C++11 and should be replaced by hexadecimal literals instead.

Fixes #1854.

Co-authored-by: Dirk Stolle <striezel-dev@web.de>
This commit is contained in:
Daniel Lemire
2022-07-04 21:54:19 -04:00
committed by GitHub
parent 62bdb9a2f7
commit 3fde8a4eac
22 changed files with 110 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/fallback/dom_parser_implementation.cpp
+27 -27
View File
@@ -27,12 +27,12 @@ simdjson_really_inline void add_structural() {
}
simdjson_really_inline bool is_continuation(uint8_t c) {
return (c & 0b11000000) == 0b10000000;
return (c & 0xc0) == 0x80;
}
simdjson_really_inline void validate_utf8_character() {
// Continuation
if (simdjson_unlikely((buf[idx] & 0b01000000) == 0)) {
if (simdjson_unlikely((buf[idx] & 0x40) == 0)) {
// extra continuation
error = UTF8_ERROR;
idx++;
@@ -40,7 +40,7 @@ simdjson_really_inline void validate_utf8_character() {
}
// 2-byte
if ((buf[idx] & 0b00100000) == 0) {
if ((buf[idx] & 0x20) == 0) {
// missing continuation
if (simdjson_unlikely(idx+1 > len || !is_continuation(buf[idx+1]))) {
if (idx+1 > len && is_streaming(partial)) { idx = len; return; }
@@ -49,13 +49,13 @@ simdjson_really_inline void validate_utf8_character() {
return;
}
// overlong: 1100000_ 10______
if (buf[idx] <= 0b11000001) { error = UTF8_ERROR; }
if (buf[idx] <= 0xc1) { error = UTF8_ERROR; }
idx += 2;
return;
}
// 3-byte
if ((buf[idx] & 0b00010000) == 0) {
if ((buf[idx] & 0x10) == 0) {
// missing continuation
if (simdjson_unlikely(idx+2 > len || !is_continuation(buf[idx+1]) || !is_continuation(buf[idx+2]))) {
if (idx+2 > len && is_streaming(partial)) { idx = len; return; }
@@ -64,9 +64,9 @@ simdjson_really_inline void validate_utf8_character() {
return;
}
// overlong: 11100000 100_____ ________
if (buf[idx] == 0b11100000 && buf[idx+1] <= 0b10011111) { error = UTF8_ERROR; }
if (buf[idx] == 0xe0 && buf[idx+1] <= 0x9f) { error = UTF8_ERROR; }
// surrogates: U+D800-U+DFFF 11101101 101_____
if (buf[idx] == 0b11101101 && buf[idx+1] >= 0b10100000) { error = UTF8_ERROR; }
if (buf[idx] == 0xed && buf[idx+1] >= 0xa0) { error = UTF8_ERROR; }
idx += 3;
return;
}
@@ -80,14 +80,14 @@ simdjson_really_inline void validate_utf8_character() {
return;
}
// overlong: 11110000 1000____ ________ ________
if (buf[idx] == 0b11110000 && buf[idx+1] <= 0b10001111) { error = UTF8_ERROR; }
if (buf[idx] == 0xf0 && buf[idx+1] <= 0x8f) { error = UTF8_ERROR; }
// too large: > U+10FFFF:
// 11110100 (1001|101_)____
// 1111(1___|011_|0101) 10______
// also includes 5, 6, 7 and 8 byte characters:
// 11111___
if (buf[idx] == 0b11110100 && buf[idx+1] >= 0b10010000) { error = UTF8_ERROR; }
if (buf[idx] >= 0b11110101) { error = UTF8_ERROR; }
if (buf[idx] == 0xf4 && buf[idx+1] >= 0x90) { error = UTF8_ERROR; }
if (buf[idx] >= 0xf5) { error = UTF8_ERROR; }
idx += 4;
}
@@ -97,7 +97,7 @@ simdjson_really_inline bool validate_string() {
while (idx < len && buf[idx] != '"') {
if (buf[idx] == '\\') {
idx += 2;
} else if (simdjson_unlikely(buf[idx] & 0b10000000)) {
} else if (simdjson_unlikely(buf[idx] & 0x80)) {
validate_utf8_character();
} else {
if (buf[idx] < 0x20) { error = UNESCAPED_CHARS; }
@@ -297,39 +297,39 @@ simdjson_warn_unused bool implementation::validate_utf8(const char *buf, size_t
}
}
unsigned char byte = data[pos];
if (byte < 0b10000000) {
if (byte < 0x80) {
pos++;
continue;
} else if ((byte & 0b11100000) == 0b11000000) {
} else if ((byte & 0xe0) == 0xc0) {
next_pos = pos + 2;
if (next_pos > len) { return false; }
if ((data[pos + 1] & 0b11000000) != 0b10000000) { return false; }
if ((data[pos + 1] & 0xc0) != 0x80) { return false; }
// range check
code_point = (byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111);
code_point = (byte & 0x1f) << 6 | (data[pos + 1] & 0x3f);
if (code_point < 0x80 || 0x7ff < code_point) { return false; }
} else if ((byte & 0b11110000) == 0b11100000) {
} else if ((byte & 0xf0) == 0xe0) {
next_pos = pos + 3;
if (next_pos > len) { return false; }
if ((data[pos + 1] & 0b11000000) != 0b10000000) { return false; }
if ((data[pos + 2] & 0b11000000) != 0b10000000) { return false; }
if ((data[pos + 1] & 0xc0) != 0x80) { return false; }
if ((data[pos + 2] & 0xc0) != 0x80) { return false; }
// range check
code_point = (byte & 0b00001111) << 12 |
(data[pos + 1] & 0b00111111) << 6 |
(data[pos + 2] & 0b00111111);
code_point = (byte & 0x0f) << 12 |
(data[pos + 1] & 0x3f) << 6 |
(data[pos + 2] & 0x3f);
if (code_point < 0x800 || 0xffff < code_point ||
(0xd7ff < code_point && code_point < 0xe000)) {
return false;
}
} else if ((byte & 0b11111000) == 0b11110000) { // 0b11110000
} else if ((byte & 0xf8) == 0xf0) { // 0b11110000
next_pos = pos + 4;
if (next_pos > len) { return false; }
if ((data[pos + 1] & 0b11000000) != 0b10000000) { return false; }
if ((data[pos + 2] & 0b11000000) != 0b10000000) { return false; }
if ((data[pos + 3] & 0b11000000) != 0b10000000) { return false; }
if ((data[pos + 1] & 0xc0) != 0x80) { return false; }
if ((data[pos + 2] & 0xc0) != 0x80) { return false; }
if ((data[pos + 3] & 0xc0) != 0x80) { return false; }
// range check
code_point =
(byte & 0b00000111) << 18 | (data[pos + 1] & 0b00111111) << 12 |
(data[pos + 2] & 0b00111111) << 6 | (data[pos + 3] & 0b00111111);
(byte & 0x07) << 18 | (data[pos + 1] & 0x3f) << 12 |
(data[pos + 2] & 0x3f) << 6 | (data[pos + 3] & 0x3f);
if (code_point <= 0xffff || 0x10ffff < code_point) { return false; }
} else {
// we may have a continuation