admin/simdjson-simdjson
1
0
Fork 0
mirror of https://github.com/simdjson/simdjson synced 2026-06-08 17:27:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into release_candidate_4_0_0

Browse Source
This commit is contained in:
Daniel Lemire
2025-07-31 10:24:54 -04:00
parent 97eb557388 d365cfb4a1
commit 3d1ab87ecb
18 changed files with 1207 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/fallback.cpp
+88 -32
View File
@@ -121,10 +121,78 @@ simdjson_inline void validate_utf8_character() {
idx += 4;
}
static const uint8_t CHAR_TYPE_SPACE = 1 << 0;
static const uint8_t CHAR_TYPE_OPERATOR = 1 << 1;
static const uint8_t CHAR_TYPE_ESC_ASCII = 1 << 2;
static const uint8_t CHAR_TYPE_NON_ASCII = 1 << 3;
const uint8_t char_table[256] = {
0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04,
0x04, 0x05, 0x05, 0x04, 0x04, 0x05, 0x04, 0x04,
0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04,
0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04,
0x01, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x02, 0x04, 0x02, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x02, 0x00, 0x02, 0x00, 0x00,
0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08
};
simdjson_inline bool char_is_type(uint8_t c, uint8_t type) {
return (char_table[c] & type);
}
simdjson_inline bool char_is_space(uint8_t c) {
return char_is_type(c, CHAR_TYPE_SPACE);
}
simdjson_inline bool char_is_operator(uint8_t c) {
return char_is_type(c, CHAR_TYPE_OPERATOR);
}
simdjson_inline bool char_is_space_or_operator(uint8_t c) {
return char_is_type(c, CHAR_TYPE_SPACE | CHAR_TYPE_OPERATOR);
}
simdjson_inline bool char_is_ascii_stop(uint8_t c) {
return char_is_type(c, CHAR_TYPE_ESC_ASCII | CHAR_TYPE_NON_ASCII);
}
// Returns true if the string is unclosed.
simdjson_inline bool validate_string() {
idx++; // skip first quote
while (idx < len && buf[idx] != '"') {
while (idx < len) {
do {
if (char_is_ascii_stop(buf[idx])) { break; }
idx++;
} while (idx < len);
if (idx >= len) { return true; }
if (buf[idx] == '"') {
return false;
}
if (buf[idx] == '\\') {
idx += 2;
} else if (simdjson_unlikely(buf[idx] & 0x80)) {
@@ -138,43 +206,31 @@ simdjson_inline bool validate_string() {
return false;
}
simdjson_inline bool is_whitespace_or_operator(uint8_t c) {
switch (c) {
case '{': case '}': case '[': case ']': case ',': case ':':
case ' ': case '\r': case '\n': case '\t':
return true;
default:
return false;
}
}
//
// Parse the entire input in STEP_SIZE-byte chunks.
//
simdjson_inline error_code scan() {
bool unclosed_string = false;
for (;idx<len;idx++) {
switch (buf[idx]) {
// String
case '"':
add_structural();
unclosed_string |= validate_string();
break;
// Operator
case '{': case '}': case '[': case ']': case ',': case ':':
add_structural();
break;
// Whitespace
case ' ': case '\r': case '\n': case '\t':
break;
// Primitive or invalid character (invalid characters will be checked in stage 2)
default:
// Anything else, add the structural and go until we find the next one
add_structural();
while (idx+1<len && !is_whitespace_or_operator(buf[idx+1])) {
idx++;
};
break;
do {
if (!char_is_space(buf[idx])) { break; }
idx++;
} while (idx < len);
if (idx >= len) { break; }
// String
if (buf[idx] == '"') {
add_structural();
unclosed_string |= validate_string();
// Operator
} else if (char_is_operator(buf[idx])) {
add_structural();
// Primitive or invalid character (invalid characters will be checked in stage 2)
} else {
// Anything else, add the structural and go until we find the next one
add_structural();
while (idx+1<len && !char_is_space_or_operator(buf[idx+1])) {
idx++;
};
}
}
// We pad beyond.