admin/simdjson-simdjson
1
0
Fork 0
mirror of https://github.com/simdjson/simdjson synced 2026-06-08 17:27:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #939 from simdjson/dlemire/lookup3

Browse Source 
Introducing lookup3 (UTF-8 validation).
This commit is contained in:
Daniel Lemire
2020-06-17 11:19:09 -04:00
committed by GitHub
parent 7339f67dd7 71a889ed73
commit 942ef3b7f2
4 changed files with 255 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/westmere/dom_parser_implementation.cpp
+9 -1
View File
@@ -62,6 +62,14 @@ really_inline simd8<bool> must_be_continuation(simd8<uint8_t> prev1, simd8<uint8
return simd8<int8_t>(is_second_byte | is_third_byte | is_fourth_byte) > int8_t(0);
}
really_inline simd8<bool> must_be_2_3_continuation(simd8<uint8_t> prev2, simd8<uint8_t> prev3) {
simd8<uint8_t> is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0
simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0
// Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine.
return simd8<int8_t>(is_third_byte | is_fourth_byte) > int8_t(0);
}
#include "generic/stage1/buf_block_reader.h"
#include "generic/stage1/json_string_scanner.h"
#include "generic/stage1/json_scanner.h"
@@ -79,7 +87,7 @@ WARN_UNUSED error_code implementation::minify(const uint8_t *buf, size_t len, ui
}
#include "generic/stage1/find_next_document_index.h"
#include "generic/stage1/utf8_lookup2_algorithm.h"
#include "generic/stage1/utf8_lookup3_algorithm.h"
#include "generic/stage1/json_structural_indexer.h"
WARN_UNUSED error_code dom_parser_implementation::stage1(const uint8_t *_buf, size_t _len, bool streaming) noexcept {
this->buf = _buf;