diff --git a/.drone.yml b/.drone.yml index 79023e61c..dfe02a233 100644 --- a/.drone.yml +++ b/.drone.yml @@ -5,6 +5,6 @@ steps: - name: test image: gcc:8 commands: - - make -j2 - - make quiettest -j2 + - make + - make quiettest - make amalgamate diff --git a/Makefile b/Makefile index 223064cad..e6b1c4bfa 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ architecture:=$(shell arch) ifeq ($(architecture),aarch64) ARCHFLAGS ?= -march=armv8-a+crc+crypto else -ARCHFLAGS ?= -march=native +ARCHFLAGS ?= -msse4.2 -mpclmul # lowest supported feature set? endif CXXFLAGS = $(ARCHFLAGS) -std=c++17 -Wall -Wextra -Wshadow -Iinclude -Ibenchmark/linux $(EXTRAFLAGS) @@ -62,8 +62,8 @@ TESTEXECUTABLES=jsoncheck numberparsingcheck stringparsingcheck pointercheck COMPARISONEXECUTABLES=minifiercompetition parsingcompetition parseandstatcompetition distinctuseridcompetition allparserscheckfile allparsingcompetition SUPPLEMENTARYEXECUTABLES=parse_noutf8validation parse_nonumberparsing parse_nostringparsing -HEADERS= include/simdjson/simdutf8check.h include/simdjson/stringparsing.h include/simdjson/numberparsing.h include/simdjson/jsonparser.h include/simdjson/common_defs.h include/simdjson/jsonioutil.h benchmark/benchmark.h benchmark/linux/linux-perf-events.h include/simdjson/parsedjson.h include/simdjson/stage1_find_marks.h include/simdjson/stage2_build_tape.h include/simdjson/jsoncharutils.h include/simdjson/jsonformatutils.h -LIBFILES=src/jsonioutil.cpp src/jsonparser.cpp src/simdjson.cpp src/stage1_find_marks.cpp src/stage2_build_tape.cpp src/parsedjson.cpp src/parsedjsoniterator.cpp +HEADERS= include/simdjson/simdutf8check_haswell.h include/simdjson/simdutf8check_westmere.h include/simdjson/simdutf8check_arm64.h include/simdjson/stringparsing.h include/simdjson/stringparsing_arm64.h include/simdjson/stringparsing_haswell.h include/simdjson/stringparsing_macros.h include/simdjson/stringparsing_westmere.h include/simdjson/numberparsing.h include/simdjson/jsonparser.h include/simdjson/common_defs.h include/simdjson/jsonioutil.h benchmark/benchmark.h benchmark/linux/linux-perf-events.h include/simdjson/parsedjson.h include/simdjson/stage1_find_marks.h include/simdjson/stage1_find_marks_arm64.h include/simdjson/stage1_find_marks_haswell.h include/simdjson/stage1_find_marks_westmere.h include/simdjson/stage1_find_marks_macros.h include/simdjson/stage2_build_tape.h include/simdjson/jsoncharutils.h include/simdjson/jsonformatutils.h +LIBFILES=src/jsonioutil.cpp src/jsonparser.cpp src/simdjson.cpp src/stage1_find_marks.cpp src/stage2_build_tape.cpp src/parsedjson.cpp src/parsedjsoniterator.cpp MINIFIERHEADERS=include/simdjson/jsonminifier.h include/simdjson/simdprune_tables.h MINIFIERLIBFILES=src/jsonminifier.cpp @@ -152,7 +152,7 @@ stringparsingcheck:tests/stringparsingcheck.cpp $(HEADERS) $(LIBFILES) $(CXX) $(CXXFLAGS) -o stringparsingcheck tests/stringparsingcheck.cpp src/jsonioutil.cpp src/jsonparser.cpp src/simdjson.cpp src/stage1_find_marks.cpp src/parsedjson.cpp -I. $(LIBFLAGS) -DJSON_TEST_STRINGS pointercheck:tests/pointercheck.cpp $(HEADERS) $(LIBFILES) - $(CXX) $(CXXFLAGS) -o pointercheck tests/pointercheck.cpp src/jsonioutil.cpp src/jsonparser.cpp src/simdjson.cpp src/stage1_find_marks.cpp src/parsedjson.cpp src/parsedjsoniterator.cpp -I. $(LIBFLAGS) + $(CXX) $(CXXFLAGS) -o pointercheck tests/pointercheck.cpp src/stage2_build_tape.cpp src/jsonioutil.cpp src/jsonparser.cpp src/simdjson.cpp src/stage1_find_marks.cpp src/parsedjson.cpp src/parsedjsoniterator.cpp -I. $(LIBFLAGS) minifiercompetition: benchmark/minifiercompetition.cpp $(HEADERS) submodules $(MINIFIERHEADERS) $(LIBFILES) $(MINIFIERLIBFILES) $(CXX) $(CXXFLAGS) -o minifiercompetition $(LIBFILES) $(MINIFIERLIBFILES) benchmark/minifiercompetition.cpp -I. $(LIBFLAGS) $(COREDEPSINCLUDE) diff --git a/README.md b/README.md index f41907959..1b8e468e4 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ On a Skylake processor, the parsing speeds (in GB/s) of various processors on th - We support platforms like Linux or macOS, as well as Windows through Visual Studio 2017 or later. - A processor with - AVX2 (i.e., Intel processors starting with the Haswell microarchitecture released 2013 and AMD processors starting with the Zen microarchitecture released 2017), - - or SSE 4.2 (i.e., Intel processors going back to Nehalem released in 2008 or AMD processors starting with the Jaguar used in the PS4 and XBox One) + - or SSE 4.2 and CLMUL (i.e., Intel processors going back to Westmere released in 2010 or AMD processors starting with the Jaguar used in the PS4 and XBox One) - or a 64-bit ARM processor (ARMv8-A): this covers a wide range of mobile processors, including all Apple processors currently available for sale, going back as far back as the iPhone 5s (2013). - A recent C++ compiler (e.g., GNU GCC or LLVM CLANG or Visual Studio 2017), we assume C++17. GNU GCC 7 or better or LLVM's clang 6 or better. - Some benchmark scripts assume bash and other common utilities, but they are optional. @@ -174,16 +174,19 @@ int main(int argc, char *argv[]) { } ``` -On Intel and AMD processors, we get best performance by using the hardware support for AVX2 instructions. You have to make sure that you instruct your -compiler to use these instructions as needed. Under compilers such as GNU GCC or LLVM clang, the -flag `-march=native` used on a recent Intel processor (Haswell or better) is sufficient. For portability -of the binary files you can also specify directly the Haswell processor (`-march=haswell`). You may -also use the flags `-mavx2 -mbmi2`. Under Visual Studio, you need to target x64 and add the -flag `/arch:AVX2`. - Note: In some settings, it might be desirable to precompile `simdjson.cpp` instead of including it. +## Runtime dispatch + +On Intel and AMD processors, we get best performance by using the hardware support for AVX2 instructions. However, simdjson also +runs on older Intel and AMD processors. We require a minimum feature support of SSE 4.2 and CLMUL (2010 Intel Westmere or better). +The code automatically detects the feature set of your processor and switches to the right function at runtime (a technical +sometimes called runtime dispatch). + + +We also support 64-bit ARM. We assume NEON support, and if the cryptographic extension is available, we leverage it, at compile-time. +There is no runtime dispatch on ARM. ## Usage (old-school Makefile on platforms like Linux or macOS) @@ -266,13 +269,13 @@ make test ## Usage (CMake on Windows using Visual Studio) -We assume you have a common Windows PC with at least Visual Studio 2017 and an x64 processor with AVX2 support (2013 Intel Haswell or later) or SSE 4.2 (2008 Nehalem or later). +We assume you have a common Windows PC with at least Visual Studio 2017 and an x64 processor with AVX2 support (2013 Intel Haswell or later) or SSE 4.2 + CLMUL (2010 Westmere or later). - Grab the simdjson code from GitHub, e.g., by cloning it using [GitHub Desktop](https://desktop.github.com/). - Install [CMake](https://cmake.org/download/). When you install it, make sure to ask that `cmake` be made available from the command line. Please choose a recent version of cmake. - Create a subdirectory within simdjson, such as `VisualStudio`. - Using a shell, go to this newly created directory. -- Type `cmake -DCMAKE_GENERATOR_PLATFORM=x64 ..` in the shell while in the `VisualStudio` repository. (Alternatively, if you want to build a DLL, you may use the command line `cmake -DCMAKE_GENERATOR_PLATFORM=x64 -DSIMDJSON_BUILD_STATIC=OFF ..`.) This will build the code with AVX2 instructions. If your target processor does not support AVX2, you need to replace `cmake -DCMAKE_GENERATOR_PLATFORM=x64 ..` by `cmake -DSIMDJSON_DISABLE_AVX=on -DCMAKE_GENERATOR_PLATFORM=x64 ..` . That is, you need to set the flag to forcefully disable AVX support since we compile with AVX2 instructions *by default*. +- Type `cmake -DCMAKE_GENERATOR_PLATFORM=x64 ..` in the shell while in the `VisualStudio` repository. (Alternatively, if you want to build a DLL, you may use the command line `cmake -DCMAKE_GENERATOR_PLATFORM=x64 -DSIMDJSON_BUILD_STATIC=OFF ..`.) - This last command (`cmake ...`) created a Visual Studio solution file in the newly created directory (e.g., `simdjson.sln`). Open this file in Visual Studio. You should now be able to build the project and run the tests. For example, in the `Solution Explorer` window (available from the `View` menu), right-click `ALL_BUILD` and select `Build`. To test the code, still in the `Solution Explorer` window, select `RUN_TESTS` and select `Build`. @@ -324,7 +327,6 @@ To simplify the engineering, we make some assumptions. - We support UTF-8 (and thus ASCII), nothing else (no Latin, no UTF-16). We do not believe this is a genuine limitation, because we do not think there is any serious application that needs to process JSON data without an ASCII or UTF-8 encoding. - All strings in the JSON document may have up to 4294967295 bytes in UTF-8 (4GB). To enforce this constraint, we refuse to parse a document that contains more than 4294967295 bytes (4GB). This should accommodate most JSON documents. -- We assume AVX2 support, which is available in all recent mainstream x86 processors produced by AMD and Intel. No support for non-x86 processors is included, though it can be done. We plan to support ARM processors (help is invited). - In cases of failure, we report a failure without any indication to the nature of the problem. (This can be easily improved without affecting performance.) - As allowed by the specification, we allow repeated keys within an object (other parsers like sajson do the same). - Performance is optimized for JSON documents spanning at least a tens kilobytes up to many megabytes: the performance issues with having to parse many tiny JSON documents or one truly enormous JSON document are different. diff --git a/amalgamation.sh b/amalgamation.sh index 486ce9e4d..2266c0619 100755 --- a/amalgamation.sh +++ b/amalgamation.sh @@ -26,20 +26,31 @@ $SCRIPTPATH/src/parsedjsoniterator.cpp # order matters ALLCHEADERS=" $SCRIPTPATH/include/simdjson/simdjson_version.h -$SCRIPTPATH/include/simdjson/simdjson.h $SCRIPTPATH/include/simdjson/portability.h +$SCRIPTPATH/include/simdjson/isadetection.h +$SCRIPTPATH/include/simdjson/simdjson.h $SCRIPTPATH/include/simdjson/common_defs.h $SCRIPTPATH/include/simdjson/padded_string.h $SCRIPTPATH/include/simdjson/jsoncharutils.h $SCRIPTPATH/include/simdjson/jsonformatutils.h $SCRIPTPATH/include/simdjson/jsonioutil.h $SCRIPTPATH/include/simdjson/simdprune_tables.h -$SCRIPTPATH/include/simdjson/simdutf8check.h -$SCRIPTPATH/include/simdjson/simdutf8check_neon.h +$SCRIPTPATH/include/simdjson/simdutf8check_haswell.h +$SCRIPTPATH/include/simdjson/simdutf8check_westmere.h +$SCRIPTPATH/include/simdjson/simdutf8check_arm64.h $SCRIPTPATH/include/simdjson/jsonminifier.h $SCRIPTPATH/include/simdjson/parsedjson.h $SCRIPTPATH/include/simdjson/stage1_find_marks.h +$SCRIPTPATH/include/simdjson/stage1_find_marks_flatten.h +$SCRIPTPATH/include/simdjson/stage1_find_marks_macros.h +$SCRIPTPATH/include/simdjson/stage1_find_marks_westmere.h +$SCRIPTPATH/include/simdjson/stage1_find_marks_haswell.h +$SCRIPTPATH/include/simdjson/stage1_find_marks_arm64.h $SCRIPTPATH/include/simdjson/stringparsing.h +$SCRIPTPATH/include/simdjson/stringparsing_macros.h +$SCRIPTPATH/include/simdjson/stringparsing_westmere.h +$SCRIPTPATH/include/simdjson/stringparsing_haswell.h +$SCRIPTPATH/include/simdjson/stringparsing_arm64.h $SCRIPTPATH/include/simdjson/numberparsing.h $SCRIPTPATH/include/simdjson/stage2_build_tape.h $SCRIPTPATH/include/simdjson/jsonparser.h @@ -125,16 +136,16 @@ echo "Giving final instructions:" CPPBIN=${DEMOCPP%%.*} echo "Try :" -echo "c++ -march=native -O3 -std=c++17 -o ${CPPBIN} ${DEMOCPP} && ./${CPPBIN} ../jsonexamples/twitter.json " +echo "c++ -O3 -std=c++17 -o ${CPPBIN} ${DEMOCPP} && ./${CPPBIN} ../jsonexamples/twitter.json " SINGLEHDR=$SCRIPTPATH/singleheader echo "Copying files to $SCRIPTPATH/singleheader " mkdir -p $SINGLEHDR -echo "c++ -march=native -O3 -std=c++17 -o ${CPPBIN} ${DEMOCPP} && ./${CPPBIN} ../jsonexamples/twitter.json " > $SINGLEHDR/README.md +echo "c++ -O3 -std=c++17 -o ${CPPBIN} ${DEMOCPP} && ./${CPPBIN} ../jsonexamples/twitter.json " > $SINGLEHDR/README.md cp ${AMAL_C} ${AMAL_H} ${DEMOCPP} $SINGLEHDR ls $SINGLEHDR -cd $SINGLEHDR && c++ -march=native -O3 -std=c++17 -o ${CPPBIN} ${DEMOCPP} && ./${CPPBIN} ../jsonexamples/twitter.json +cd $SINGLEHDR && c++ -O3 -std=c++17 -o ${CPPBIN} ${DEMOCPP} && ./${CPPBIN} ../jsonexamples/twitter.json lowercase(){ echo "$1" | tr 'A-Z' 'a-z' diff --git a/benchmark/minifiercompetition.cpp b/benchmark/minifiercompetition.cpp index aae2d2838..c662a2203 100644 --- a/benchmark/minifiercompetition.cpp +++ b/benchmark/minifiercompetition.cpp @@ -14,7 +14,7 @@ #include "rapidjson/writer.h" #include "sajson.h" - +using namespace simdjson; using namespace rapidjson; std::string rapidstringmeInsitu(char *json) { diff --git a/benchmark/parse.cpp b/benchmark/parse.cpp index f614519fe..44e20d81e 100644 --- a/benchmark/parse.cpp +++ b/benchmark/parse.cpp @@ -33,6 +33,90 @@ #include "simdjson/parsedjson.h" #include "simdjson/stage1_find_marks.h" #include "simdjson/stage2_build_tape.h" +#include "simdjson/isadetection.h" +namespace simdjson { +architecture _find_best_supported_implementation() { + constexpr uint32_t haswell_flags = SIMDExtensions::AVX2 | SIMDExtensions::PCLMULQDQ + | SIMDExtensions::BMI1 | SIMDExtensions::BMI2; + constexpr uint32_t westmere_flags = SIMDExtensions::SSE42 | SIMDExtensions::PCLMULQDQ; + uint32_t supports = detect_supported_architectures(); + // Order from best to worst (within architecture) + if ((haswell_flags & supports) == haswell_flags) { + return architecture::haswell; + } + if ((westmere_flags & supports) == westmere_flags) { + return architecture::westmere; + } + if (SIMDExtensions::NEON) return architecture::arm64; + + return architecture::none; +} + + +using unified_functype = int (const uint8_t *buf, size_t len, ParsedJson &pj); +using stage1_functype = int (const uint8_t *buf, size_t len, ParsedJson &pj); + + +extern unified_functype *unified_ptr; + +extern stage1_functype *stage1_ptr; + +int unified_machine_dispatch(const uint8_t *buf, size_t len, ParsedJson &pj) { + architecture best_implementation = _find_best_supported_implementation(); + // Selecting the best implementation + switch (best_implementation) { +#ifdef IS_X86_64 + case architecture::haswell: + unified_ptr = &unified_machine; + break; + case architecture::westmere: + unified_ptr = &unified_machine; + break; +#endif +#ifdef IS_ARM64 + case architecture::arm64: + unified_ptr = &unified_machine; + break; +#endif + default : + std::cerr << "The processor is not supported by simdjson." << std::endl; + return simdjson::UNEXPECTED_ERROR; + } + + return unified_ptr(buf, len, pj); +} + +// Responsible to select the best json_parse implementation +int find_structural_bits_dispatch(const uint8_t *buf, size_t len, ParsedJson &pj) { + architecture best_implementation = _find_best_supported_implementation(); + // Selecting the best implementation + switch (best_implementation) { +#ifdef IS_X86_64 + case architecture::haswell: + stage1_ptr = &find_structural_bits; + break; + case architecture::westmere: + stage1_ptr = &find_structural_bits; + break; +#endif +#ifdef IS_ARM64 + case architecture::arm64: + stage1_ptr = &find_structural_bits; + break; +#endif + default : + std::cerr << "The processor is not supported by simdjson." << std::endl; + return simdjson::UNEXPECTED_ERROR; + } + + return stage1_ptr(buf, len, pj); +} + +stage1_functype *stage1_ptr = &find_structural_bits_dispatch; +unified_functype *unified_ptr = &unified_machine_dispatch; +} + + int main(int argc, char *argv[]) { bool verbose = false; @@ -104,7 +188,14 @@ int main(int argc, char *argv[]) { printf("justdata (-t) flag only works under linux.\n"); } #endif - + {// practice run + simdjson::ParsedJson pj; + bool allocok = pj.allocateCapacity(p.size()); + if(allocok) { + simdjson::stage1_ptr((const uint8_t*)p.data(), p.size(), pj); + simdjson::unified_ptr((const uint8_t*)(const uint8_t*)(const uint8_t*)(const uint8_t*)(const uint8_t*)(const uint8_t*)(const uint8_t*)(const uint8_t*)p.data(), p.size(), pj); + } + } #ifndef SQUASH_COUNTERS std::vector evts; evts.push_back(PERF_COUNT_HW_CPU_CYCLES); @@ -144,8 +235,7 @@ int main(int argc, char *argv[]) { std::cout << "[verbose] allocated memory for parsed JSON " << std::endl; } unified.start(); - // The default template is simdjson::instruction_set::native. - isok = (simdjson::find_structural_bits<>(p.data(), p.size(), pj) == simdjson::SUCCESS); + isok = (simdjson::stage1_ptr((const uint8_t*)p.data(), p.size(), pj) == simdjson::SUCCESS); unified.end(results); cy1 += results[0]; cl1 += results[1]; @@ -157,8 +247,7 @@ int main(int argc, char *argv[]) { break; } unified.start(); - // The default template is simdjson::instruction_set::native. - isok = isok && (simdjson::SUCCESS == simdjson::unified_machine<>(p.data(), p.size(), pj)); + isok = isok && (simdjson::SUCCESS == simdjson::unified_ptr((const uint8_t*)p.data(), p.size(), pj)); unified.end(results); cy2 += results[0]; cl2 += results[1]; @@ -187,9 +276,8 @@ int main(int argc, char *argv[]) { } auto start = std::chrono::steady_clock::now(); - // The default template is simdjson::instruction_set::native. - isok = (simdjson::find_structural_bits<>(p.data(), p.size(), pj) == simdjson::SUCCESS); - isok = isok && (simdjson::SUCCESS == simdjson::unified_machine<>(p.data(), p.size(), pj)); + isok = (simdjson::stage1_ptr((const uint8_t*)p.data(), p.size(), pj) == simdjson::SUCCESS); + isok = isok && (simdjson::SUCCESS == simdjson::unified_ptr((const uint8_t*)p.data(), p.size(), pj)); auto end = std::chrono::steady_clock::now(); std::chrono::duration secs = end - start; res[i] = secs.count(); diff --git a/benchmark/parseandstatcompetition.cpp b/benchmark/parseandstatcompetition.cpp index 1f5916d8e..4ea121320 100644 --- a/benchmark/parseandstatcompetition.cpp +++ b/benchmark/parseandstatcompetition.cpp @@ -12,7 +12,7 @@ #include "sajson.h" using namespace rapidjson; - +using namespace simdjson; struct stat_s { size_t number_count; size_t object_count; diff --git a/benchmark/statisticalmodel.cpp b/benchmark/statisticalmodel.cpp index 3d6a1b1c0..d79c853e4 100644 --- a/benchmark/statisticalmodel.cpp +++ b/benchmark/statisticalmodel.cpp @@ -180,7 +180,7 @@ int main(int argc, char *argv[]) { results.resize(evts.size()); for (uint32_t i = 0; i < iterations; i++) { unified.start(); - // The default template is simdjson::instruction_set::native. + // The default template is simdjson::architecture::native. bool isok = (simdjson::find_structural_bits<>(p.data(), p.size(), pj) == simdjson::SUCCESS); unified.end(results); diff --git a/include/simdjson/common_defs.h b/include/simdjson/common_defs.h index 44fcef7b0..eb9369843 100644 --- a/include/simdjson/common_defs.h +++ b/include/simdjson/common_defs.h @@ -31,7 +31,7 @@ #define ISALIGNED_N(ptr, n) (((uintptr_t)(ptr) & ((n)-1)) == 0) #ifdef _MSC_VER -#define really_inline inline +#define really_inline __forceinline #define never_inline __declspec(noinline) #define UNUSED @@ -44,20 +44,26 @@ #define unlikely(x) x #endif +// For Visual Studio compilers, same-page buffer overrun is not fine. +#define ALLOW_SAME_PAGE_BUFFER_OVERRUN false + #else // For non-Visual Studio compilers, we may assume that same-page buffer overrun is fine. // However, it will make it difficult to be "valgrind clean". //#ifndef ALLOW_SAME_PAGE_BUFFER_OVERRUN -//#define ALLOW_SAME_PAGE_BUFFER_OVERRUN +//#define ALLOW_SAME_PAGE_BUFFER_OVERRUN true +//#else +#define ALLOW_SAME_PAGE_BUFFER_OVERRUN false //#endif // The following is likely unnecessarily complex. #ifdef __SANITIZE_ADDRESS__ // we have GCC, stuck with https://gcc.gnu.org/bugzilla/show_bug.cgi?id=67368 -#undef ALLOW_SAME_PAGE_BUFFER_OVERRUN +#define ALLOW_SAME_PAGE_BUFFER_OVERRUN false #elif defined(__has_feature) // we have CLANG? +// todo: if we're setting ALLOW_SAME_PAGE_BUFFER_OVERRUN to false, why do we have a non-empty qualifier? # if (__has_feature(address_sanitizer)) #define ALLOW_SAME_PAGE_BUFFER_OVERRUN_QUALIFIER __attribute__((no_sanitize("address"))) # endif diff --git a/include/simdjson/isadetection.h b/include/simdjson/isadetection.h new file mode 100644 index 000000000..903c81188 --- /dev/null +++ b/include/simdjson/isadetection.h @@ -0,0 +1,154 @@ +/* From https://github.com/endorno/pytorch/blob/master/torch/lib/TH/generic/simd/simd.h +Highly modified. + +Copyright (c) 2016- Facebook, Inc (Adam Paszke) +Copyright (c) 2014- Facebook, Inc (Soumith Chintala) +Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert) +Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu) +Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu) +Copyright (c) 2011-2013 NYU (Clement Farabet) +Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, Iain Melvin, Jason Weston) +Copyright (c) 2006 Idiap Research Institute (Samy Bengio) +Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, Samy Bengio, Johnny Mariethoz) + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories America + and IDIAP Research Institute nor the names of its contributors may be + used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef SIMDJSON_ISADETECTION_H +#define SIMDJSON_ISADETECTION_H + +#include +#include +#if defined(_MSC_VER) +#include +#elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID) +#include +#endif + +namespace simdjson { +// Can be found on Intel ISA Reference for CPUID +constexpr uint32_t cpuid_avx2_bit = 1 << 5; // Bit 5 of EBX for EAX=0x7 +constexpr uint32_t cpuid_bmi1_bit = 1 << 3; // bit 3 of EBX for EAX=0x7 +constexpr uint32_t cpuid_bmi2_bit = 1 << 8; // bit 8 of EBX for EAX=0x7 +constexpr uint32_t cpuid_sse42_bit = 1 << 20; // bit 20 of ECX for EAX=0x1 +constexpr uint32_t cpuid_pclmulqdq_bit = 1 << 1; // bit 1 of ECX for EAX=0x1 + +enum SIMDExtensions { + DEFAULT = 0x0, + NEON = 0x1, + AVX2 = 0x4, + SSE42 = 0x8, + PCLMULQDQ = 0x10, + BMI1 = 0x20, + BMI2 = 0x40 +}; + +#if defined(__arm__) || defined(__aarch64__) // incl. armel, armhf, arm64 + + #if defined(__NEON__) + +static inline uint32_t detect_supported_architectures() +{ + return SIMDExtensions::NEON; +} + + #else //ARM without NEON + +static inline uint32_t detect_supported_architectures() +{ + return SIMDExtensions::DEFAULT; +} + + #endif + +#else // x86 +static inline void cpuid(uint32_t *eax, uint32_t *ebx, uint32_t *ecx, uint32_t *edx) +{ +#if defined(_MSC_VER) + int cpuInfo[4]; + __cpuid(cpuInfo, *eax); + *eax = cpuInfo[0]; + *ebx = cpuInfo[1]; + *ecx = cpuInfo[2]; + *edx = cpuInfo[3]; +#elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID) + uint32_t level = *eax; + __get_cpuid (level, eax, ebx, ecx, edx); +#else + uint32_t a = *eax, b, c = *ecx, d; + asm volatile ( "cpuid\n\t" + : "+a"(a), "=b"(b), "+c"(c), "=d"(d) ); + *eax = a; + *ebx = b; + *ecx = c; + *edx = d; +#endif +} + +static inline uint32_t detect_supported_architectures() +{ + uint32_t eax, ebx, ecx, edx; + uint32_t hostSimdExts = 0x0; + + // ECX for EAX=0x7 + eax = 0x7; + ecx = 0x0; + cpuid(&eax, &ebx, &ecx, &edx); + + if (ebx & cpuid_avx2_bit) { + hostSimdExts |= SIMDExtensions::AVX2; + } + + if (ebx & cpuid_bmi1_bit) { + hostSimdExts |= SIMDExtensions::BMI1; + } + + if (ebx & cpuid_bmi2_bit) { + hostSimdExts |= SIMDExtensions::BMI2; + } + + // EBX for EAX=0x1 + eax = 0x1; + cpuid(&eax, &ebx, &ecx, &edx); + + if (ecx & cpuid_sse42_bit) { + hostSimdExts |= SIMDExtensions::SSE42; + } + + if (ecx & cpuid_pclmulqdq_bit) { + hostSimdExts |= SIMDExtensions::PCLMULQDQ; + } + + return hostSimdExts; +} + +#endif // end SIMD extension detection code +} +#endif diff --git a/include/simdjson/jsonparser.h b/include/simdjson/jsonparser.h index dd67f83fb..060a8bce0 100644 --- a/include/simdjson/jsonparser.h +++ b/include/simdjson/jsonparser.h @@ -27,16 +27,16 @@ using json_parse_functype = int (const uint8_t *buf, size_t len, ParsedJson &pj, extern json_parse_functype *json_parse_ptr; // json_parse_implementation is the generic function, it is specialized for various -// SIMD instruction sets, e.g., as json_parse_implementation -// or json_parse_implementation -template +// architectures, e.g., as json_parse_implementation +// or json_parse_implementation +template int json_parse_implementation(const uint8_t *buf, size_t len, ParsedJson &pj, bool reallocifneeded = true) { if (pj.bytecapacity < len) { return simdjson::CAPACITY; } bool reallocated = false; if(reallocifneeded) { -#ifdef ALLOW_SAME_PAGE_BUFFER_OVERRUN +#if ALLOW_SAME_PAGE_BUFFER_OVERRUN // realloc is needed if the end of the memory crosses a page #ifdef _MSC_VER SYSTEM_INFO sysInfo; @@ -63,7 +63,7 @@ int json_parse_implementation(const uint8_t *buf, size_t len, ParsedJson &pj, bo reallocated = true; } // if (true) OR if ( (reinterpret_cast(buf + len - 1) % pagesize ) + SIMDJSON_PADDING < static_cast(pagesize) ) { } // if(reallocifneeded) { - int stage1_is_ok = find_structural_bits(buf, len, pj); + int stage1_is_ok = simdjson::find_structural_bits(buf, len, pj); if(stage1_is_ok != simdjson::SUCCESS) { pj.errorcode = stage1_is_ok; return pj.errorcode; diff --git a/include/simdjson/numberparsing.h b/include/simdjson/numberparsing.h index 8b6a9f531..9a3519a52 100644 --- a/include/simdjson/numberparsing.h +++ b/include/simdjson/numberparsing.h @@ -113,14 +113,14 @@ really_inline bool is_not_structural_or_whitespace_or_exponent_or_decimal(unsigned char c) { return structural_or_whitespace_or_exponent_or_decimal_negated[c]; } - +}// simdjson #ifndef SIMDJSON_DISABLE_SWAR_NUMBER_PARSING -// #if defined (__AVX2__) || defined (__SSE4_2__) #define SWAR_NUMBER_PARSING #endif #ifdef SWAR_NUMBER_PARSING +namespace simdjson { // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html @@ -138,9 +138,10 @@ static inline bool is_made_of_eight_digits_fast(const char *chars) { (((val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0) >> 4)) == 0x3333333333333333); } - -#if defined (__AVX2__) || defined (__SSE4_2__) - +} +#ifdef IS_X86_64 +TARGET_WESTMERE +namespace simdjson { static inline uint32_t parse_eight_digits_unrolled(const char *chars) { // this actually computes *16* values so we are being wasteful. const __m128i ascii0 = _mm_set1_epi8('0'); @@ -157,7 +158,12 @@ static inline uint32_t parse_eight_digits_unrolled(const char *chars) { return _mm_cvtsi128_si32( t4); // only captures the sum of the first 8 digits, drop the rest } -#else +} +UNTARGET_REGION +#endif + +namespace simdjson { +#ifdef IS_ARM64 // we don't have SSE, so let us use a scalar function // credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/ static inline uint32_t parse_eight_digits_unrolled(const char *chars) { @@ -167,9 +173,9 @@ static inline uint32_t parse_eight_digits_unrolled(const char *chars) { val = (val & 0x00FF00FF00FF00FF) * 6553601 >> 16; return (val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32; } - - #endif + + #endif // @@ -565,5 +571,5 @@ static really_inline bool parse_number(const uint8_t *const buf, return is_structural_or_whitespace(*p); #endif // SIMDJSON_SKIPNUMBERPARSING } -} +}//simdjson #endif diff --git a/include/simdjson/portability.h b/include/simdjson/portability.h index 8781655d1..8963d29a4 100644 --- a/include/simdjson/portability.h +++ b/include/simdjson/portability.h @@ -1,9 +1,58 @@ #ifndef SIMDJSON_PORTABILITY_H #define SIMDJSON_PORTABILITY_H +#if defined(__x86_64__) || defined(_M_AMD64) +# define IS_X86_64 1 +#endif +#if defined(__aarch64__) || defined(_M_ARM64) +# define IS_ARM64 1 +#endif + +// this is almost standard? +#define STRINGIFY(a) #a + + + +// we are going to use runtime dispatch +#ifdef IS_X86_64 +#ifdef __clang__ +// clang does not have GCC push pop +// warning: clang attribute push can't be used within a namespace in clang up til 8.0 so TARGET_REGION and +// UNTARGET_REGION must be *outside* of a namespace. +#define TARGET_REGION(T) _Pragma(STRINGIFY(clang attribute push(__attribute__((target(T))), apply_to=function))) +#define UNTARGET_REGION _Pragma("clang attribute pop") +#elif defined(__GNUC__) +// GCC is easier +#define TARGET_REGION(T) \ +_Pragma("GCC push_options") \ +_Pragma(STRINGIFY(GCC target(T))) +#define UNTARGET_REGION \ +_Pragma("GCC pop_options") +#else +#define TARGET_REGION(T) +#define UNTARGET_REGION +#endif // clang then gcc + +// under GCC and CLANG, we use these two macros +#define TARGET_HASWELL TARGET_REGION("avx2,bmi,pclmul") +#define TARGET_WESTMERE TARGET_REGION("sse4.2,pclmul") + +#endif // x86 + + + +#ifdef _MSC_VER +# include +#else +# if IS_X86_64 +# include +# elif IS_ARM64 +# include +# endif +#endif + #ifdef _MSC_VER /* Microsoft C/C++-compatible compiler */ -#include #include #include @@ -40,9 +89,6 @@ static inline int hamming(uint64_t input_num) { #include #include -#if defined(__BMI2__) || defined(__POPCOUNT__) || defined(__AVX2__) || defined(__SSE4_2__) -#include -#endif namespace simdjson { static inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t *result) { return __builtin_uaddll_overflow(value1, value2, (unsigned long long*)result); @@ -101,30 +147,6 @@ static inline char *aligned_malloc_char(size_t alignment, size_t size) { return (char*)aligned_malloc(alignment, size); } -#ifdef __AVX2__ - -#ifndef __clang__ -#ifndef _MSC_VER -static __m256i inline _mm256_loadu2_m128i(__m128i const *__addr_hi, - __m128i const *__addr_lo) { - __m256i __v256 = _mm256_castsi128_si256(_mm_loadu_si128(__addr_lo)); - return _mm256_insertf128_si256(__v256, _mm_loadu_si128(__addr_hi), 1); -} - -static inline void _mm256_storeu2_m128i(__m128i *__addr_hi, __m128i *__addr_lo, - __m256i __a) { - __m128i __v128; - - __v128 = _mm256_castsi256_si128(__a); - _mm_storeu_si128(__addr_lo, __v128); - __v128 = _mm256_extractf128_si256(__a, 1); - _mm_storeu_si128(__addr_hi, __v128); -} -#endif -#endif - -#endif // AVX_2 - static inline void aligned_free(void *memblock) { if(memblock == nullptr) { return; } #ifdef _MSC_VER @@ -142,5 +164,4 @@ static inline void aligned_free_char(char *memblock) { aligned_free((void*)memblock); } } - #endif // SIMDJSON_PORTABILITY_H diff --git a/include/simdjson/simdjson.h b/include/simdjson/simdjson.h index 4190aeafe..ff2aaa3bc 100644 --- a/include/simdjson/simdjson.h +++ b/include/simdjson/simdjson.h @@ -1,26 +1,21 @@ -#ifndef SIMDJSON_ERR_H -# define SIMDJSON_ERR_H +#ifndef SIMDJSON_ERR_H +#define SIMDJSON_ERR_H #include namespace simdjson { -enum class instruction_set { - avx2, - sse4_2, - neon, +// Represents the minimal architecture that would support an implementation +enum class architecture { + westmere, + haswell, + arm64, none, -// the 'native' enum class value should point at a good default on the current machine -#ifdef __AVX2__ - native = avx2 -#elif defined(__ARM_NEON) || (defined(_MSC_VER) && defined(_M_ARM64)) - native = neon -#else - // Let us assume that we have an old x64 processor, but one that has SSE (i.e., something - // that came out in the second decade of the XXIst century. - // It would be nicer to check explicitly, but there many not be a good way to do so - // that is cross-platform. - // Under Visual Studio, there is no way to check for SSE4.2 support at compile-time. - native = sse4_2 + // TODO remove 'native' in favor of runtime dispatch? + // the 'native' enum class value should point at a good default on the current machine +#ifdef IS_X86_64 + native = westmere +#elif defined(IS_ARM64) + native = arm64 #endif }; diff --git a/include/simdjson/simdprune_tables.h b/include/simdjson/simdprune_tables.h index 77b78e8a7..8191b268a 100644 --- a/include/simdjson/simdprune_tables.h +++ b/include/simdjson/simdprune_tables.h @@ -4,8 +4,6 @@ #include "simdjson/portability.h" - -#ifdef __AVX__ namespace simdjson { static const unsigned char mask128_epi8[] = { 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe, @@ -34987,9 +34985,7 @@ static const unsigned char mask128_epi32[] = { 0xff, 0xff, 0xff, 0xff, }; } // namespace simdjson -#endif //__SSE3__ -#ifdef __AVX2__ namespace simdjson { static const uint32_t mask256_epi32[] = { 0, 1, 2, 3, 4, 5, 6, 7, 1, 2, 3, 4, 5, 6, 7, 7, 0, 2, 3, 4, 5, 6, 7, 7, 2, @@ -35075,6 +35071,4 @@ static const uint32_t mask256_epi32[] = { 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; } -#endif //__AVX2__ - #endif diff --git a/include/simdjson/simdutf8check_neon.h b/include/simdjson/simdutf8check_arm64.h similarity index 98% rename from include/simdjson/simdutf8check_neon.h rename to include/simdjson/simdutf8check_arm64.h index ab0f53758..198fc7990 100644 --- a/include/simdjson/simdutf8check_neon.h +++ b/include/simdjson/simdutf8check_arm64.h @@ -1,8 +1,8 @@ // From https://github.com/cyb70289/utf8/blob/master/lemire-neon.c // Adapted from https://github.com/lemire/fastvalidate-utf-8 -#ifndef SIMDJSON_SIMDUTF8CHECK_NEON_H -#define SIMDJSON_SIMDUTF8CHECK_NEON_H +#ifndef SIMDJSON_SIMDUTF8CHECK_ARM64_H +#define SIMDJSON_SIMDUTF8CHECK_ARM64_H #if defined(_ARM_NEON) || defined(__aarch64__) || (defined(_MSC_VER) && defined(_M_ARM64)) @@ -29,6 +29,7 @@ * U+100000..U+10FFFF F4 80..8F 80..BF 80..BF * */ +namespace simdjson { // all byte values must be no larger than 0xF4 static inline void checkSmallerThan0xF4(int8x16_t current_bytes, @@ -167,6 +168,6 @@ checkUTF8Bytes(int8x16_t current_bytes, struct processed_utf_bytes *previous, previous->high_nibbles, has_error); return pb; } - +}// simdjson #endif #endif diff --git a/include/simdjson/simdutf8check.h b/include/simdjson/simdutf8check_haswell.h similarity index 57% rename from include/simdjson/simdutf8check.h rename to include/simdjson/simdutf8check_haswell.h index a32b29593..f896a3dcb 100644 --- a/include/simdjson/simdutf8check.h +++ b/include/simdjson/simdutf8check_haswell.h @@ -1,13 +1,13 @@ +#ifndef SIMDJSON_SIMDUTF8CHECK_HASWELL_H +#define SIMDJSON_SIMDUTF8CHECK_HASWELL_H -#ifndef SIMDJSON_SIMDUTF8CHECK_H -#define SIMDJSON_SIMDUTF8CHECK_H - -#if defined (__AVX2__) || defined (__SSE4_2__) || (defined(_MSC_VER) && defined(_M_AMD64)) #include #include #include #include "simdjson/portability.h" + +#ifdef IS_X86_64 /* * legal utf-8 byte sequence * http://www.unicode.org/versions/Unicode6.0.0/ch03.pdf - page 94 @@ -27,141 +27,8 @@ // all byte values must be no larger than 0xF4 +TARGET_HASWELL namespace simdjson { -// all byte values must be no larger than 0xF4 -static inline void checkSmallerThan0xF4(__m128i current_bytes, - __m128i *has_error) { - // unsigned, saturates to 0 below max - *has_error = _mm_or_si128(*has_error, - _mm_subs_epu8(current_bytes, _mm_set1_epi8(0xF4))); -} - -static inline __m128i continuationLengths(__m128i high_nibbles) { - return _mm_shuffle_epi8( - _mm_setr_epi8(1, 1, 1, 1, 1, 1, 1, 1, // 0xxx (ASCII) - 0, 0, 0, 0, // 10xx (continuation) - 2, 2, // 110x - 3, // 1110 - 4), // 1111, next should be 0 (not checked here) - high_nibbles); -} - -static inline __m128i carryContinuations(__m128i initial_lengths, - __m128i previous_carries) { - - __m128i right1 = - _mm_subs_epu8(_mm_alignr_epi8(initial_lengths, previous_carries, 16 - 1), - _mm_set1_epi8(1)); - __m128i sum = _mm_add_epi8(initial_lengths, right1); - - __m128i right2 = _mm_subs_epu8(_mm_alignr_epi8(sum, previous_carries, 16 - 2), - _mm_set1_epi8(2)); - return _mm_add_epi8(sum, right2); -} - -static inline void checkContinuations(__m128i initial_lengths, __m128i carries, - __m128i *has_error) { - - // overlap || underlap - // carry > length && length > 0 || !(carry > length) && !(length > 0) - // (carries > length) == (lengths > 0) - __m128i overunder = - _mm_cmpeq_epi8(_mm_cmpgt_epi8(carries, initial_lengths), - _mm_cmpgt_epi8(initial_lengths, _mm_setzero_si128())); - - *has_error = _mm_or_si128(*has_error, overunder); -} - -// when 0xED is found, next byte must be no larger than 0x9F -// when 0xF4 is found, next byte must be no larger than 0x8F -// next byte must be continuation, ie sign bit is set, so signed < is ok -static inline void checkFirstContinuationMax(__m128i current_bytes, - __m128i off1_current_bytes, - __m128i *has_error) { - __m128i maskED = _mm_cmpeq_epi8(off1_current_bytes, _mm_set1_epi8(0xED)); - __m128i maskF4 = _mm_cmpeq_epi8(off1_current_bytes, _mm_set1_epi8(0xF4)); - - __m128i badfollowED = - _mm_and_si128(_mm_cmpgt_epi8(current_bytes, _mm_set1_epi8(0x9F)), maskED); - __m128i badfollowF4 = - _mm_and_si128(_mm_cmpgt_epi8(current_bytes, _mm_set1_epi8(0x8F)), maskF4); - - *has_error = _mm_or_si128(*has_error, _mm_or_si128(badfollowED, badfollowF4)); -} - -// map off1_hibits => error condition -// hibits off1 cur -// C => < C2 && true -// E => < E1 && < A0 -// F => < F1 && < 90 -// else false && false -static inline void checkOverlong(__m128i current_bytes, - __m128i off1_current_bytes, __m128i hibits, - __m128i previous_hibits, __m128i *has_error) { - __m128i off1_hibits = _mm_alignr_epi8(hibits, previous_hibits, 16 - 1); - __m128i initial_mins = _mm_shuffle_epi8( - _mm_setr_epi8(-128, -128, -128, -128, -128, -128, -128, -128, -128, -128, - -128, -128, // 10xx => false - 0xC2, -128, // 110x - 0xE1, // 1110 - 0xF1), - off1_hibits); - - __m128i initial_under = _mm_cmpgt_epi8(initial_mins, off1_current_bytes); - - __m128i second_mins = _mm_shuffle_epi8( - _mm_setr_epi8(-128, -128, -128, -128, -128, -128, -128, -128, -128, -128, - -128, -128, // 10xx => false - 127, 127, // 110x => true - 0xA0, // 1110 - 0x90), - off1_hibits); - __m128i second_under = _mm_cmpgt_epi8(second_mins, current_bytes); - *has_error = - _mm_or_si128(*has_error, _mm_and_si128(initial_under, second_under)); -} - -struct processed_utf_bytes { - __m128i rawbytes; - __m128i high_nibbles; - __m128i carried_continuations; -}; - -static inline void count_nibbles(__m128i bytes, - struct processed_utf_bytes *answer) { - answer->rawbytes = bytes; - answer->high_nibbles = - _mm_and_si128(_mm_srli_epi16(bytes, 4), _mm_set1_epi8(0x0F)); -} - -// check whether the current bytes are valid UTF-8 -// at the end of the function, previous gets updated -static struct processed_utf_bytes -checkUTF8Bytes(__m128i current_bytes, struct processed_utf_bytes *previous, - __m128i *has_error) { - struct processed_utf_bytes pb; - count_nibbles(current_bytes, &pb); - - checkSmallerThan0xF4(current_bytes, has_error); - - __m128i initial_lengths = continuationLengths(pb.high_nibbles); - - pb.carried_continuations = - carryContinuations(initial_lengths, previous->carried_continuations); - - checkContinuations(initial_lengths, pb.carried_continuations, has_error); - - __m128i off1_current_bytes = - _mm_alignr_epi8(pb.rawbytes, previous->rawbytes, 16 - 1); - checkFirstContinuationMax(current_bytes, off1_current_bytes, has_error); - - checkOverlong(current_bytes, off1_current_bytes, pb.high_nibbles, - previous->high_nibbles, has_error); - return pb; -} - -#ifdef __AVX2__ -/*****************************/ static inline __m256i push_last_byte_of_a_to_b(__m256i a, __m256i b) { return _mm256_alignr_epi8(b, _mm256_permute2x128_si256(a, b, 0x21), 15); } @@ -322,10 +189,10 @@ avxcheckUTF8Bytes(__m256i current_bytes, previous->high_nibbles, has_error); return pb; } +}// simdjson +UNTARGET_REGION // haswell -#endif // __AVX2__ -} -#endif // defined (__AVX2__) || defined (__SSE4_2__) || (defined(_MSC_VER) && defined(_M_AMD64)) +#endif // IS_X86_64 #endif diff --git a/include/simdjson/simdutf8check_westmere.h b/include/simdjson/simdutf8check_westmere.h new file mode 100644 index 000000000..73beab9dc --- /dev/null +++ b/include/simdjson/simdutf8check_westmere.h @@ -0,0 +1,170 @@ +#ifndef SIMDJSON_SIMDUTF8CHECK_WESTMERE_H +#define SIMDJSON_SIMDUTF8CHECK_WESTMERE_H + +#include +#include +#include +#include "simdjson/portability.h" +#ifdef IS_X86_64 + +/* + * legal utf-8 byte sequence + * http://www.unicode.org/versions/Unicode6.0.0/ch03.pdf - page 94 + * + * Code Points 1st 2s 3s 4s + * U+0000..U+007F 00..7F + * U+0080..U+07FF C2..DF 80..BF + * U+0800..U+0FFF E0 A0..BF 80..BF + * U+1000..U+CFFF E1..EC 80..BF 80..BF + * U+D000..U+D7FF ED 80..9F 80..BF + * U+E000..U+FFFF EE..EF 80..BF 80..BF + * U+10000..U+3FFFF F0 90..BF 80..BF 80..BF + * U+40000..U+FFFFF F1..F3 80..BF 80..BF 80..BF + * U+100000..U+10FFFF F4 80..8F 80..BF 80..BF + * + */ + +// all byte values must be no larger than 0xF4 + +/********** sse code **********/ +TARGET_WESTMERE + +namespace simdjson{ +// all byte values must be no larger than 0xF4 +static inline void checkSmallerThan0xF4(__m128i current_bytes, + __m128i *has_error) { + // unsigned, saturates to 0 below max + *has_error = _mm_or_si128(*has_error, + _mm_subs_epu8(current_bytes, _mm_set1_epi8(0xF4))); +} + +static inline __m128i continuationLengths(__m128i high_nibbles) { + return _mm_shuffle_epi8( + _mm_setr_epi8(1, 1, 1, 1, 1, 1, 1, 1, // 0xxx (ASCII) + 0, 0, 0, 0, // 10xx (continuation) + 2, 2, // 110x + 3, // 1110 + 4), // 1111, next should be 0 (not checked here) + high_nibbles); +} + +static inline __m128i carryContinuations(__m128i initial_lengths, + __m128i previous_carries) { + + __m128i right1 = + _mm_subs_epu8(_mm_alignr_epi8(initial_lengths, previous_carries, 16 - 1), + _mm_set1_epi8(1)); + __m128i sum = _mm_add_epi8(initial_lengths, right1); + + __m128i right2 = _mm_subs_epu8(_mm_alignr_epi8(sum, previous_carries, 16 - 2), + _mm_set1_epi8(2)); + return _mm_add_epi8(sum, right2); +} + +static inline void checkContinuations(__m128i initial_lengths, __m128i carries, + __m128i *has_error) { + + // overlap || underlap + // carry > length && length > 0 || !(carry > length) && !(length > 0) + // (carries > length) == (lengths > 0) + __m128i overunder = + _mm_cmpeq_epi8(_mm_cmpgt_epi8(carries, initial_lengths), + _mm_cmpgt_epi8(initial_lengths, _mm_setzero_si128())); + + *has_error = _mm_or_si128(*has_error, overunder); +} + +// when 0xED is found, next byte must be no larger than 0x9F +// when 0xF4 is found, next byte must be no larger than 0x8F +// next byte must be continuation, ie sign bit is set, so signed < is ok +static inline void checkFirstContinuationMax(__m128i current_bytes, + __m128i off1_current_bytes, + __m128i *has_error) { + __m128i maskED = _mm_cmpeq_epi8(off1_current_bytes, _mm_set1_epi8(0xED)); + __m128i maskF4 = _mm_cmpeq_epi8(off1_current_bytes, _mm_set1_epi8(0xF4)); + + __m128i badfollowED = + _mm_and_si128(_mm_cmpgt_epi8(current_bytes, _mm_set1_epi8(0x9F)), maskED); + __m128i badfollowF4 = + _mm_and_si128(_mm_cmpgt_epi8(current_bytes, _mm_set1_epi8(0x8F)), maskF4); + + *has_error = _mm_or_si128(*has_error, _mm_or_si128(badfollowED, badfollowF4)); +} + +// map off1_hibits => error condition +// hibits off1 cur +// C => < C2 && true +// E => < E1 && < A0 +// F => < F1 && < 90 +// else false && false +static inline void checkOverlong(__m128i current_bytes, + __m128i off1_current_bytes, __m128i hibits, + __m128i previous_hibits, __m128i *has_error) { + __m128i off1_hibits = _mm_alignr_epi8(hibits, previous_hibits, 16 - 1); + __m128i initial_mins = _mm_shuffle_epi8( + _mm_setr_epi8(-128, -128, -128, -128, -128, -128, -128, -128, -128, -128, + -128, -128, // 10xx => false + 0xC2, -128, // 110x + 0xE1, // 1110 + 0xF1), + off1_hibits); + + __m128i initial_under = _mm_cmpgt_epi8(initial_mins, off1_current_bytes); + + __m128i second_mins = _mm_shuffle_epi8( + _mm_setr_epi8(-128, -128, -128, -128, -128, -128, -128, -128, -128, -128, + -128, -128, // 10xx => false + 127, 127, // 110x => true + 0xA0, // 1110 + 0x90), + off1_hibits); + __m128i second_under = _mm_cmpgt_epi8(second_mins, current_bytes); + *has_error = + _mm_or_si128(*has_error, _mm_and_si128(initial_under, second_under)); +} + +struct processed_utf_bytes { + __m128i rawbytes; + __m128i high_nibbles; + __m128i carried_continuations; +}; + +static inline void count_nibbles(__m128i bytes, + struct processed_utf_bytes *answer) { + answer->rawbytes = bytes; + answer->high_nibbles = + _mm_and_si128(_mm_srli_epi16(bytes, 4), _mm_set1_epi8(0x0F)); +} + +// check whether the current bytes are valid UTF-8 +// at the end of the function, previous gets updated +static struct processed_utf_bytes +checkUTF8Bytes(__m128i current_bytes, struct processed_utf_bytes *previous, + __m128i *has_error) { + struct processed_utf_bytes pb; + count_nibbles(current_bytes, &pb); + + checkSmallerThan0xF4(current_bytes, has_error); + + __m128i initial_lengths = continuationLengths(pb.high_nibbles); + + pb.carried_continuations = + carryContinuations(initial_lengths, previous->carried_continuations); + + checkContinuations(initial_lengths, pb.carried_continuations, has_error); + + __m128i off1_current_bytes = + _mm_alignr_epi8(pb.rawbytes, previous->rawbytes, 16 - 1); + checkFirstContinuationMax(current_bytes, off1_current_bytes, has_error); + + checkOverlong(current_bytes, off1_current_bytes, pb.high_nibbles, + previous->high_nibbles, has_error); + return pb; +} +}//simdjson +UNTARGET_REGION // westmere + + +#endif // IS_X86_64 + +#endif diff --git a/include/simdjson/stage1_find_marks.h b/include/simdjson/stage1_find_marks.h index d204b2739..daf95415a 100644 --- a/include/simdjson/stage1_find_marks.h +++ b/include/simdjson/stage1_find_marks.h @@ -3,120 +3,16 @@ #include #include "simdjson/common_defs.h" +#include "simdjson/simdjson.h" #include "simdjson/parsedjson.h" #include "simdjson/portability.h" -#if defined (__AVX2__) -#elif defined (__SSE4_2__) || (defined(_MSC_VER) && defined(_M_AMD64)) -#elif defined(__ARM_NEON) || (defined(_MSC_VER) && defined(_M_ARM64)) -#include -#else -#warning It appears that neither ARM NEON nor AVX2 nor SSE are detected. -#endif // (__AVX2__) - -#ifndef SIMDJSON_SKIPUTF8VALIDATION -#define SIMDJSON_UTF8VALIDATE -#endif - -// It seems that many parsers do UTF-8 validation. -// RapidJSON does not do it by default, but a flag -// allows it. -#ifdef SIMDJSON_UTF8VALIDATE -#if defined (__AVX2__) -#include "simdjson/simdutf8check.h" -#elif defined (__SSE4_2__) || (defined(_MSC_VER) && defined(_M_AMD64)) -#include "simdjson/simdutf8check.h" -#elif defined(__ARM_NEON) || (defined(_MSC_VER) && defined(_M_ARM64)) -#include "simdjson/simdutf8check_neon.h" -#endif // (__AVX2__) -#endif // SIMDJSON_UTF8VALIDATE - -//#define TRANSPOSE - namespace simdjson { -template + +template struct simd_input; -#ifdef __AVX2__ -template<> -struct simd_input -{ - __m256i lo; - __m256i hi; -}; -#endif - -#if defined(__SSE4_2__) || (defined(_MSC_VER) && defined(_M_AMD64)) -template<> -struct simd_input -{ - __m128i v0; - __m128i v1; - __m128i v2; - __m128i v3; -}; -#endif - -#if defined(__ARM_NEON) || (defined(_MSC_VER) && defined(_M_ARM64)) -template<> struct simd_input -{ -#ifndef TRANSPOSE - uint8x16_t i0; - uint8x16_t i1; - uint8x16_t i2; - uint8x16_t i3; -#else - uint8x16x4_t i; -#endif -}; -#endif - -#if defined(__ARM_NEON) || (defined(_MSC_VER) && defined(_M_ARM64)) -really_inline -uint16_t neonmovemask(uint8x16_t input) { - const uint8x16_t bitmask = { 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80, - 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80}; - uint8x16_t minput = vandq_u8(input, bitmask); - uint8x16_t tmp = vpaddq_u8(minput, minput); - tmp = vpaddq_u8(tmp, tmp); - tmp = vpaddq_u8(tmp, tmp); - return vgetq_lane_u16(vreinterpretq_u16_u8(tmp), 0); -} - -really_inline -uint64_t neonmovemask_bulk(uint8x16_t p0, uint8x16_t p1, uint8x16_t p2, uint8x16_t p3) { -#ifndef TRANSPOSE - const uint8x16_t bitmask = { 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80, - 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80}; - uint8x16_t t0 = vandq_u8(p0, bitmask); - uint8x16_t t1 = vandq_u8(p1, bitmask); - uint8x16_t t2 = vandq_u8(p2, bitmask); - uint8x16_t t3 = vandq_u8(p3, bitmask); - uint8x16_t sum0 = vpaddq_u8(t0, t1); - uint8x16_t sum1 = vpaddq_u8(t2, t3); - sum0 = vpaddq_u8(sum0, sum1); - sum0 = vpaddq_u8(sum0, sum0); - return vgetq_lane_u64(vreinterpretq_u64_u8(sum0), 0); -#else - const uint8x16_t bitmask1 = { 0x01, 0x10, 0x01, 0x10, 0x01, 0x10, 0x01, 0x10, - 0x01, 0x10, 0x01, 0x10, 0x01, 0x10, 0x01, 0x10}; - const uint8x16_t bitmask2 = { 0x02, 0x20, 0x02, 0x20, 0x02, 0x20, 0x02, 0x20, - 0x02, 0x20, 0x02, 0x20, 0x02, 0x20, 0x02, 0x20}; - const uint8x16_t bitmask3 = { 0x04, 0x40, 0x04, 0x40, 0x04, 0x40, 0x04, 0x40, - 0x04, 0x40, 0x04, 0x40, 0x04, 0x40, 0x04, 0x40}; - const uint8x16_t bitmask4 = { 0x08, 0x80, 0x08, 0x80, 0x08, 0x80, 0x08, 0x80, - 0x08, 0x80, 0x08, 0x80, 0x08, 0x80, 0x08, 0x80}; - uint8x16_t t0 = vandq_u8(p0, bitmask1); - uint8x16_t t1 = vbslq_u8(bitmask2, p1, t0); - uint8x16_t t2 = vbslq_u8(bitmask3, p2, t1); - uint8x16_t tmp = vbslq_u8(bitmask4, p3, t2); - uint8x16_t sum = vpaddq_u8(tmp, tmp); - return vgetq_lane_u64(vreinterpretq_u64_u8(sum), 0); -#endif -} -#endif - -template +template uint64_t compute_quote_mask(uint64_t quote_bits); namespace { @@ -132,428 +28,40 @@ namespace { } } -// In practice, if you have NEON or __PCLMUL__, you would -// always want to use them, but it might be useful, for research -// purposes, to disable it willingly, that's what SIMDJSON_AVOID_CLMUL -// does. -// Also: we don't know of an instance where AVX2 is supported but -// where clmul is not supported, so check for both, to be sure. -#ifdef SIMDJSON_AVOID_CLMUL -template really_inline -uint64_t compute_quote_mask(uint64_t quote_bits) { - return portable_compute_quote_mask(quote_bits); -} -#else -template -uint64_t compute_quote_mask(uint64_t quote_bits); - -#ifdef __AVX2__ -template<> really_inline -uint64_t compute_quote_mask(uint64_t quote_bits) { - // There should be no such thing with a processing supporting avx2 - // but not clmul. - uint64_t quote_mask = _mm_cvtsi128_si64(_mm_clmulepi64_si128( - _mm_set_epi64x(0ULL, quote_bits), _mm_set1_epi8(0xFF), 0)); - return quote_mask; -} -#endif - -#if defined(__SSE4_2__) || (defined(_MSC_VER) && defined(_M_AMD64)) -template<> really_inline -uint64_t compute_quote_mask(uint64_t quote_bits) { - // CLMUL is supported on some SSE42 hardware such as Sandy Bridge, - // but not on others. -#ifdef __PCLMUL__ - return _mm_cvtsi128_si64(_mm_clmulepi64_si128( - _mm_set_epi64x(0ULL, quote_bits), _mm_set1_epi8(0xFF), 0)); -#else - return portable_compute_quote_mask(quote_bits); -#endif -} -#endif - -#if defined(__ARM_NEON) || (defined(_MSC_VER) && defined(_M_ARM64)) -template<> really_inline -uint64_t compute_quote_mask(uint64_t quote_bits) { -#ifdef __ARM_FEATURE_CRYPTO // some ARM processors lack this extension - return vmull_p64( -1ULL, quote_bits); -#else - return portable_compute_quote_mask(quote_bits); -#endif -} -#endif -#endif // SIMDJSON_AVOID_CLMUL - -#ifdef SIMDJSON_UTF8VALIDATE // Holds the state required to perform check_utf8(). -template +template struct utf8_checking_state; -#ifdef __AVX2__ -template<> -struct utf8_checking_state -{ - __m256i has_error = _mm256_setzero_si256(); - avx_processed_utf_bytes previous { - _mm256_setzero_si256(), // rawbytes - _mm256_setzero_si256(), // high_nibbles - _mm256_setzero_si256() // carried_continuations - }; -}; -#endif -#if defined(__SSE4_2__) || (defined(_MSC_VER) && defined(_M_AMD64)) -template<> -struct utf8_checking_state -{ - __m128i has_error = _mm_setzero_si128(); - processed_utf_bytes previous { - _mm_setzero_si128(), // rawbytes - _mm_setzero_si128(), // high_nibbles - _mm_setzero_si128() // carried_continuations - }; -}; -#endif - -#if defined(__ARM_NEON) || (defined(_MSC_VER) && defined(_M_ARM64)) -template<> -struct utf8_checking_state -{ - int8x16_t has_error {}; - processed_utf_bytes previous {}; -}; -#endif - -#if defined(__ARM_NEON) || (defined(_MSC_VER) && defined(_M_ARM64)) -// Checks that all bytes are ascii -really_inline -bool check_ascii_neon(simd_input in) { - // checking if the most significant bit is always equal to 0. - uint8x16_t highbit = vdupq_n_u8(0x80); - uint8x16_t t0 = vorrq_u8(in.i0, in.i1); - uint8x16_t t1 = vorrq_u8(in.i2, in.i3); - uint8x16_t t3 = vorrq_u8(t0, t1); - uint8x16_t t4 = vandq_u8(t3, highbit); - uint64x2_t v64 = vreinterpretq_u64_u8(t4); - uint32x2_t v32 = vqmovn_u64(v64); - uint64x1_t result = vreinterpret_u64_u32(v32); - return vget_lane_u64(result, 0) == 0; -} -#endif - -template +template void check_utf8(simd_input in, utf8_checking_state& state); -#ifdef __AVX2__ -template<> really_inline -void check_utf8(simd_input in, - utf8_checking_state& state) { - __m256i highbit = _mm256_set1_epi8(0x80); - if ((_mm256_testz_si256(_mm256_or_si256(in.lo, in.hi), highbit)) == 1) { - // it is ascii, we just check continuation - state.has_error = _mm256_or_si256( - _mm256_cmpgt_epi8( - state.previous.carried_continuations, - _mm256_setr_epi8(9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 1)), - state.has_error); - } else { - // it is not ascii so we have to do heavy work - state.previous = avxcheckUTF8Bytes(in.lo, &(state.previous), &(state.has_error)); - state.previous = avxcheckUTF8Bytes(in.hi, &(state.previous), &(state.has_error)); - } -} -#endif //__AVX2__ - -#if defined(__SSE4_2__) || (defined(_MSC_VER) && defined(_M_AMD64)) -template<> really_inline -void check_utf8(simd_input in, - utf8_checking_state& state) { - __m128i highbit = _mm_set1_epi8(0x80); - if ((_mm_testz_si128(_mm_or_si128(in.v0, in.v1), highbit)) == 1) { - // it is ascii, we just check continuation - state.has_error = _mm_or_si128( - _mm_cmpgt_epi8( - state.previous.carried_continuations, - _mm_setr_epi8(9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 1)), - state.has_error); - } else { - // it is not ascii so we have to do heavy work - state.previous = checkUTF8Bytes(in.v0, &(state.previous), &(state.has_error)); - state.previous = checkUTF8Bytes(in.v1, &(state.previous), &(state.has_error)); - } - - if ((_mm_testz_si128(_mm_or_si128(in.v2, in.v3), highbit)) == 1) { - // it is ascii, we just check continuation - state.has_error = _mm_or_si128( - _mm_cmpgt_epi8( - state.previous.carried_continuations, - _mm_setr_epi8(9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 1)), - state.has_error); - } else { - // it is not ascii so we have to do heavy work - state.previous = checkUTF8Bytes(in.v2, &(state.previous), &(state.has_error)); - state.previous = checkUTF8Bytes(in.v3, &(state.previous), &(state.has_error)); - } -} -#endif // __SSE4_2 - -#if defined(__ARM_NEON) || (defined(_MSC_VER) && defined(_M_ARM64)) -template<> really_inline -void check_utf8(simd_input in, - utf8_checking_state& state) { - if (check_ascii_neon(in)) { - // All bytes are ascii. Therefore the byte that was just before must be ascii too. - // We only check the byte that was just before simd_input. Nines are arbitrary values. - const int8x16_t verror = (int8x16_t){9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 1}; - state.has_error = - vorrq_s8(vreinterpretq_s8_u8(vcgtq_s8(state.previous.carried_continuations, - verror)), - state.has_error); - } else { - // it is not ascii so we have to do heavy work - state.previous = checkUTF8Bytes(vreinterpretq_s8_u8(in.i0), &(state.previous), &(state.has_error)); - state.previous = checkUTF8Bytes(vreinterpretq_s8_u8(in.i1), &(state.previous), &(state.has_error)); - state.previous = checkUTF8Bytes(vreinterpretq_s8_u8(in.i2), &(state.previous), &(state.has_error)); - state.previous = checkUTF8Bytes(vreinterpretq_s8_u8(in.i3), &(state.previous), &(state.has_error)); - } -} -#endif // __ARM_NEON - // Checks if the utf8 validation has found any error. -template +template errorValues check_utf8_errors(utf8_checking_state& state); -#ifdef __AVX2__ -template<> really_inline -errorValues check_utf8_errors(utf8_checking_state& state) { - return _mm256_testz_si256(state.has_error, state.has_error) == 0 ? simdjson::UTF8_ERROR : simdjson::SUCCESS; -} -#endif - -#if defined(__SSE4_2__) || (defined(_MSC_VER) && defined(_M_AMD64)) -template<> really_inline -errorValues check_utf8_errors(utf8_checking_state& state) { - return _mm_testz_si128(state.has_error, state.has_error) == 0 ? simdjson::UTF8_ERROR : simdjson::SUCCESS; -} -#endif - -#if defined(__ARM_NEON) || (defined(_MSC_VER) && defined(_M_ARM64)) -template<> really_inline -errorValues check_utf8_errors(utf8_checking_state& state) { - uint64x2_t v64 = vreinterpretq_u64_s8(state.has_error); - uint32x2_t v32 = vqmovn_u64(v64); - uint64x1_t result = vreinterpret_u64_u32(v32); - return vget_lane_u64(result, 0) != 0 ? simdjson::UTF8_ERROR : simdjson::SUCCESS; -} -#endif -#endif // SIMDJSON_UTF8VALIDATE - -template -simd_input fill_input(const uint8_t * ptr); - -#ifdef __AVX2__ -template<> really_inline -simd_input fill_input(const uint8_t * ptr) { - struct simd_input in; - in.lo = _mm256_loadu_si256(reinterpret_cast(ptr + 0)); - in.hi = _mm256_loadu_si256(reinterpret_cast(ptr + 32)); - return in; -} -#endif - -#if defined(__SSE4_2__) || (defined(_MSC_VER) && defined(_M_AMD64)) -template<> really_inline -simd_input fill_input(const uint8_t * ptr) { - struct simd_input in; - in.v0 = _mm_loadu_si128(reinterpret_cast(ptr + 0)); - in.v1 = _mm_loadu_si128(reinterpret_cast(ptr + 16)); - in.v2 = _mm_loadu_si128(reinterpret_cast(ptr + 32)); - in.v3 = _mm_loadu_si128(reinterpret_cast(ptr + 48)); - return in; -} -#endif - -#if defined(__ARM_NEON) || (defined(_MSC_VER) && defined(_M_ARM64)) -template<> really_inline -simd_input fill_input(const uint8_t * ptr) { - struct simd_input in; -#ifndef TRANSPOSE - in.i0 = vld1q_u8(ptr + 0); - in.i1 = vld1q_u8(ptr + 16); - in.i2 = vld1q_u8(ptr + 32); - in.i3 = vld1q_u8(ptr + 48); -#else - in.i = vld4q_u8(ptr); -#endif - return in; -} -#endif - -// a straightforward comparison of a mask against input. 5 uops; would be -// cheaper in AVX512. -template +// a straightforward comparison of a mask against input. +template uint64_t cmp_mask_against_input(simd_input in, uint8_t m); -#ifdef __AVX2__ -template<> really_inline -uint64_t cmp_mask_against_input(simd_input in, uint8_t m) { - const __m256i mask = _mm256_set1_epi8(m); - __m256i cmp_res_0 = _mm256_cmpeq_epi8(in.lo, mask); - uint64_t res_0 = static_cast(_mm256_movemask_epi8(cmp_res_0)); - __m256i cmp_res_1 = _mm256_cmpeq_epi8(in.hi, mask); - uint64_t res_1 = _mm256_movemask_epi8(cmp_res_1); - return res_0 | (res_1 << 32); -} -#endif + +template +simd_input fill_input(const uint8_t * ptr); -#if defined(__SSE4_2__) || (defined(_MSC_VER) && defined(_M_AMD64)) -template<> really_inline -uint64_t cmp_mask_against_input(simd_input in, uint8_t m) { - const __m128i mask = _mm_set1_epi8(m); - __m128i cmp_res_0 = _mm_cmpeq_epi8(in.v0, mask); - uint64_t res_0 = _mm_movemask_epi8(cmp_res_0); - __m128i cmp_res_1 = _mm_cmpeq_epi8(in.v1, mask); - uint64_t res_1 = _mm_movemask_epi8(cmp_res_1); - __m128i cmp_res_2 = _mm_cmpeq_epi8(in.v2, mask); - uint64_t res_2 = _mm_movemask_epi8(cmp_res_2); - __m128i cmp_res_3 = _mm_cmpeq_epi8(in.v3, mask); - uint64_t res_3 = _mm_movemask_epi8(cmp_res_3); - return res_0 | (res_1 << 16) | (res_2 << 32) | (res_3 << 48); -} -#endif - -#if defined(__ARM_NEON) || (defined(_MSC_VER) && defined(_M_ARM64)) -template<> really_inline -uint64_t cmp_mask_against_input(simd_input in, uint8_t m) { - const uint8x16_t mask = vmovq_n_u8(m); - uint8x16_t cmp_res_0 = vceqq_u8(in.i0, mask); - uint8x16_t cmp_res_1 = vceqq_u8(in.i1, mask); - uint8x16_t cmp_res_2 = vceqq_u8(in.i2, mask); - uint8x16_t cmp_res_3 = vceqq_u8(in.i3, mask); - return neonmovemask_bulk(cmp_res_0, cmp_res_1, cmp_res_2, cmp_res_3); -} -#endif // find all values less than or equal than the content of maxval (using unsigned arithmetic) -template +template uint64_t unsigned_lteq_against_input(simd_input in, uint8_t m); -#ifdef __AVX2__ -template<> really_inline -uint64_t unsigned_lteq_against_input(simd_input in, uint8_t m) { - const __m256i maxval = _mm256_set1_epi8(m); - __m256i cmp_res_0 = _mm256_cmpeq_epi8(_mm256_max_epu8(maxval,in.lo),maxval); - uint64_t res_0 = static_cast(_mm256_movemask_epi8(cmp_res_0)); - __m256i cmp_res_1 = _mm256_cmpeq_epi8(_mm256_max_epu8(maxval,in.hi),maxval); - uint64_t res_1 = _mm256_movemask_epi8(cmp_res_1); - return res_0 | (res_1 << 32); -} -#endif -#if defined(__SSE4_2__) || (defined(_MSC_VER) && defined(_M_AMD64)) -template<> really_inline -uint64_t unsigned_lteq_against_input(simd_input in, uint8_t m) { - const __m128i maxval = _mm_set1_epi8(m); - __m128i cmp_res_0 = _mm_cmpeq_epi8(_mm_max_epu8(maxval,in.v0),maxval); - uint64_t res_0 = _mm_movemask_epi8(cmp_res_0); - __m128i cmp_res_1 = _mm_cmpeq_epi8(_mm_max_epu8(maxval,in.v1),maxval); - uint64_t res_1 = _mm_movemask_epi8(cmp_res_1); - __m128i cmp_res_2 = _mm_cmpeq_epi8(_mm_max_epu8(maxval,in.v2),maxval); - uint64_t res_2 = _mm_movemask_epi8(cmp_res_2); - __m128i cmp_res_3 = _mm_cmpeq_epi8(_mm_max_epu8(maxval,in.v3),maxval); - uint64_t res_3 = _mm_movemask_epi8(cmp_res_3); - return res_0 | (res_1 << 16) | (res_2 << 32) | (res_3 << 48); -} -#endif +template really_inline +uint64_t find_odd_backslash_sequences(simd_input in, uint64_t &prev_iter_ends_odd_backslash); -#if defined(__ARM_NEON) || (defined(_MSC_VER) && defined(_M_ARM64)) -template<> really_inline -uint64_t unsigned_lteq_against_input(simd_input in, uint8_t m) { - const uint8x16_t mask = vmovq_n_u8(m); - uint8x16_t cmp_res_0 = vcleq_u8(in.i0, mask); - uint8x16_t cmp_res_1 = vcleq_u8(in.i1, mask); - uint8x16_t cmp_res_2 = vcleq_u8(in.i2, mask); - uint8x16_t cmp_res_3 = vcleq_u8(in.i3, mask); - return neonmovemask_bulk(cmp_res_0, cmp_res_1, cmp_res_2, cmp_res_3); -} -#endif -// return a bitvector indicating where we have characters that end an odd-length -// sequence of backslashes (and thus change the behavior of the next character -// to follow). A even-length sequence of backslashes, and, for that matter, the -// largest even-length prefix of our odd-length sequence of backslashes, simply -// modify the behavior of the backslashes themselves. -// We also update the prev_iter_ends_odd_backslash reference parameter to -// indicate whether we end an iteration on an odd-length sequence of -// backslashes, which modifies our subsequent search for odd-length -// sequences of backslashes in an obvious way. -template really_inline -uint64_t find_odd_backslash_sequences(simd_input in, uint64_t &prev_iter_ends_odd_backslash) { - const uint64_t even_bits = 0x5555555555555555ULL; - const uint64_t odd_bits = ~even_bits; - uint64_t bs_bits = cmp_mask_against_input(in, '\\'); - uint64_t start_edges = bs_bits & ~(bs_bits << 1); - // flip lowest if we have an odd-length run at the end of the prior - // iteration - uint64_t even_start_mask = even_bits ^ prev_iter_ends_odd_backslash; - uint64_t even_starts = start_edges & even_start_mask; - uint64_t odd_starts = start_edges & ~even_start_mask; - uint64_t even_carries = bs_bits + even_starts; - - uint64_t odd_carries; - // must record the carry-out of our odd-carries out of bit 63; this - // indicates whether the sense of any edge going to the next iteration - // should be flipped - bool iter_ends_odd_backslash = - add_overflow(bs_bits, odd_starts, &odd_carries); - - odd_carries |= - prev_iter_ends_odd_backslash; // push in bit zero as a potential end - // if we had an odd-numbered run at the - // end of the previous iteration - prev_iter_ends_odd_backslash = iter_ends_odd_backslash ? 0x1ULL : 0x0ULL; - uint64_t even_carry_ends = even_carries & ~bs_bits; - uint64_t odd_carry_ends = odd_carries & ~bs_bits; - uint64_t even_start_odd_end = even_carry_ends & odd_bits; - uint64_t odd_start_even_end = odd_carry_ends & even_bits; - uint64_t odd_ends = even_start_odd_end | odd_start_even_end; - return odd_ends; -} - -// return both the quote mask (which is a half-open mask that covers the first -// quote -// in an unescaped quote pair and everything in the quote pair) and the quote -// bits, which are the simple -// unescaped quoted bits. We also update the prev_iter_inside_quote value to -// tell the next iteration -// whether we finished the final iteration inside a quote pair; if so, this -// inverts our behavior of -// whether we're inside quotes for the next iteration. -// Note that we don't do any error checking to see if we have backslash -// sequences outside quotes; these -// backslash sequences (of any length) will be detected elsewhere. -template really_inline +template really_inline uint64_t find_quote_mask_and_bits(simd_input in, uint64_t odd_ends, - uint64_t &prev_iter_inside_quote, uint64_t "e_bits, uint64_t &error_mask) { - quote_bits = cmp_mask_against_input(in, '"'); - quote_bits = quote_bits & ~odd_ends; - uint64_t quote_mask = compute_quote_mask(quote_bits); - quote_mask ^= prev_iter_inside_quote; - // All Unicode characters may be placed within the - // quotation marks, except for the characters that MUST be escaped: - // quotation mark, reverse solidus, and the control characters (U+0000 - //through U+001F). - // https://tools.ietf.org/html/rfc8259 - uint64_t unescaped = unsigned_lteq_against_input(in, 0x1F); - error_mask |= quote_mask & unescaped; - // right shift of a signed value expected to be well-defined and standard - // compliant as of C++20, - // John Regher from Utah U. says this is fine code - prev_iter_inside_quote = - static_cast(static_cast(quote_mask) >> 63); - return quote_mask; -} + uint64_t &prev_iter_inside_quote, uint64_t "e_bits, uint64_t &error_mask); + // do a 'shufti' to detect structural JSON characters // they are { 0x7b } 0x7d : 0x3a [ 0x5b ] 0x5d , 0x2c @@ -562,285 +70,11 @@ uint64_t find_quote_mask_and_bits(simd_input in, uint64_t odd_ends, // we are also interested in the four whitespace characters // space 0x20, linefeed 0x0a, horizontal tab 0x09 and carriage return 0x0d // these go into the next 2 buckets of the comparison (8/16) -template +template void find_whitespace_and_structurals(simd_input in, uint64_t &whitespace, uint64_t &structurals); -#ifdef __AVX2__ -template<> really_inline -void find_whitespace_and_structurals(simd_input in, - uint64_t &whitespace, - uint64_t &structurals) { -#ifdef SIMDJSON_NAIVE_STRUCTURAL - // You should never need this naive approach, but it can be useful - // for research purposes - const __m256i mask_open_brace = _mm256_set1_epi8(0x7b); - __m256i struct_lo = _mm256_cmpeq_epi8(in.lo, mask_open_brace); - __m256i struct_hi = _mm256_cmpeq_epi8(in.hi, mask_open_brace); - const __m256i mask_close_brace = _mm256_set1_epi8(0x7d); - struct_lo = _mm256_or_si256(struct_lo,_mm256_cmpeq_epi8(in.lo, mask_close_brace)); - struct_hi = _mm256_or_si256(struct_hi,_mm256_cmpeq_epi8(in.hi, mask_close_brace)); - const __m256i mask_open_bracket = _mm256_set1_epi8(0x5b); - struct_lo = _mm256_or_si256(struct_lo,_mm256_cmpeq_epi8(in.lo, mask_open_bracket)); - struct_hi = _mm256_or_si256(struct_hi,_mm256_cmpeq_epi8(in.hi, mask_open_bracket)); - const __m256i mask_close_bracket = _mm256_set1_epi8(0x5d); - struct_lo = _mm256_or_si256(struct_lo,_mm256_cmpeq_epi8(in.lo, mask_close_bracket)); - struct_hi = _mm256_or_si256(struct_hi,_mm256_cmpeq_epi8(in.hi, mask_close_bracket)); - const __m256i mask_column = _mm256_set1_epi8(0x3a); - struct_lo = _mm256_or_si256(struct_lo,_mm256_cmpeq_epi8(in.lo, mask_column)); - struct_hi = _mm256_or_si256(struct_hi,_mm256_cmpeq_epi8(in.hi, mask_column)); - const __m256i mask_comma = _mm256_set1_epi8(0x2c); - struct_lo = _mm256_or_si256(struct_lo,_mm256_cmpeq_epi8(in.lo, mask_comma)); - struct_hi = _mm256_or_si256(struct_hi,_mm256_cmpeq_epi8(in.hi, mask_comma)); - uint64_t structural_res_0 = static_cast(_mm256_movemask_epi8(struct_lo)); - uint64_t structural_res_1 = _mm256_movemask_epi8(struct_hi); - structurals = (structural_res_0 | (structural_res_1 << 32)); - - const __m256i mask_space = _mm256_set1_epi8(0x20); - __m256i space_lo = _mm256_cmpeq_epi8(in.lo, mask_space); - __m256i space_hi = _mm256_cmpeq_epi8(in.hi, mask_space); - const __m256i mask_linefeed = _mm256_set1_epi8(0x0a); - space_lo = _mm256_or_si256(space_lo,_mm256_cmpeq_epi8(in.lo, mask_linefeed)); - space_hi = _mm256_or_si256(space_hi,_mm256_cmpeq_epi8(in.hi, mask_linefeed)); - const __m256i mask_tab = _mm256_set1_epi8(0x09); - space_lo = _mm256_or_si256(space_lo,_mm256_cmpeq_epi8(in.lo, mask_tab)); - space_hi = _mm256_or_si256(space_hi,_mm256_cmpeq_epi8(in.hi, mask_tab)); - const __m256i mask_carriage = _mm256_set1_epi8(0x0d); - space_lo = _mm256_or_si256(space_lo,_mm256_cmpeq_epi8(in.lo, mask_carriage)); - space_hi = _mm256_or_si256(space_hi,_mm256_cmpeq_epi8(in.hi, mask_carriage)); - - uint64_t ws_res_0 = static_cast(_mm256_movemask_epi8(space_lo)); - uint64_t ws_res_1 = _mm256_movemask_epi8(space_hi); - whitespace = (ws_res_0 | (ws_res_1 << 32)); - // end of naive approach - -#else // SIMDJSON_NAIVE_STRUCTURAL - const __m256i structural_table = _mm256_setr_epi8( - 44, 125, 0, 0, 0xc0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 123, - 44, 125, 0, 0, 0xc0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 123); - const __m256i white_table = _mm256_setr_epi8( - 32, 100, 100, 100, 17, 100, 113, 2, 100, 9, 10, 112, 100, 13, 100, 100, - 32, 100, 100, 100, 17, 100, 113, 2, 100, 9, 10, 112, 100, 13, 100, 100); - const __m256i struct_offset = _mm256_set1_epi8(0xd4); - const __m256i struct_mask = _mm256_set1_epi8(32); - - __m256i lo_white = _mm256_cmpeq_epi8(in.lo, - _mm256_shuffle_epi8(white_table, in.lo)); - __m256i hi_white = _mm256_cmpeq_epi8(in.hi, - _mm256_shuffle_epi8(white_table, in.hi)); - uint64_t ws_res_0 = static_cast(_mm256_movemask_epi8(lo_white)); - uint64_t ws_res_1 = _mm256_movemask_epi8(hi_white); - whitespace = (ws_res_0 | (ws_res_1 << 32)); - __m256i lo_struct_r1 = _mm256_add_epi8(struct_offset, in.lo); - __m256i hi_struct_r1 = _mm256_add_epi8(struct_offset, in.hi); - __m256i lo_struct_r2 = _mm256_or_si256(in.lo, struct_mask); - __m256i hi_struct_r2 = _mm256_or_si256(in.hi, struct_mask); - __m256i lo_struct_r3 = _mm256_shuffle_epi8(structural_table, lo_struct_r1); - __m256i hi_struct_r3 = _mm256_shuffle_epi8(structural_table, hi_struct_r1); - __m256i lo_struct = _mm256_cmpeq_epi8(lo_struct_r2, lo_struct_r3); - __m256i hi_struct = _mm256_cmpeq_epi8(hi_struct_r2, hi_struct_r3); - - uint64_t structural_res_0 = - static_cast(_mm256_movemask_epi8(lo_struct)); - uint64_t structural_res_1 = _mm256_movemask_epi8(hi_struct); - structurals = (structural_res_0 | (structural_res_1 << 32)); -#endif // SIMDJSON_NAIVE_STRUCTURAL -} -#endif // __AVX2__ - -#if defined(__SSE4_2__) || (defined(_MSC_VER) && defined(_M_AMD64)) -template<> really_inline -void find_whitespace_and_structurals(simd_input in, - uint64_t &whitespace, uint64_t &structurals) { - const __m128i structural_table = _mm_setr_epi8(44, 125, 0, 0, 0xc0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 123); - const __m128i white_table = _mm_setr_epi8( - 32, 100, 100, 100, 17, 100, 113, 2, 100, 9, 10, 112, 100, 13, 100, 100); - const __m128i struct_offset = _mm_set1_epi8(0xd4); - const __m128i struct_mask = _mm_set1_epi8(32); - - __m128i white0 = _mm_cmpeq_epi8(in.v0, - _mm_shuffle_epi8(white_table, in.v0)); - __m128i white1 = _mm_cmpeq_epi8(in.v1, - _mm_shuffle_epi8(white_table, in.v1)); - __m128i white2 = _mm_cmpeq_epi8(in.v2, - _mm_shuffle_epi8(white_table, in.v2)); - __m128i white3 = _mm_cmpeq_epi8(in.v3, - _mm_shuffle_epi8(white_table, in.v3)); - uint64_t ws_res_0 = _mm_movemask_epi8(white0); - uint64_t ws_res_1 = _mm_movemask_epi8(white1); - uint64_t ws_res_2 = _mm_movemask_epi8(white2); - uint64_t ws_res_3 = _mm_movemask_epi8(white3); - - whitespace = (ws_res_0 | (ws_res_1 << 16) | (ws_res_2 << 32) | (ws_res_3 << 48)); - - __m128i struct1_r1 = _mm_add_epi8(struct_offset, in.v0); - __m128i struct2_r1 = _mm_add_epi8(struct_offset, in.v1); - __m128i struct3_r1 = _mm_add_epi8(struct_offset, in.v2); - __m128i struct4_r1 = _mm_add_epi8(struct_offset, in.v3); - - __m128i struct1_r2 = _mm_or_si128(in.v0, struct_mask); - __m128i struct2_r2 = _mm_or_si128(in.v1, struct_mask); - __m128i struct3_r2 = _mm_or_si128(in.v2, struct_mask); - __m128i struct4_r2 = _mm_or_si128(in.v3, struct_mask); - - __m128i struct1_r3 = _mm_shuffle_epi8(structural_table, struct1_r1); - __m128i struct2_r3 = _mm_shuffle_epi8(structural_table, struct2_r1); - __m128i struct3_r3 = _mm_shuffle_epi8(structural_table, struct3_r1); - __m128i struct4_r3 = _mm_shuffle_epi8(structural_table, struct4_r1); - - __m128i struct1 = _mm_cmpeq_epi8(struct1_r2, struct1_r3); - __m128i struct2 = _mm_cmpeq_epi8(struct2_r2, struct2_r3); - __m128i struct3 = _mm_cmpeq_epi8(struct3_r2, struct3_r3); - __m128i struct4 = _mm_cmpeq_epi8(struct4_r2, struct4_r3); - - uint64_t structural_res_0 = _mm_movemask_epi8(struct1); - uint64_t structural_res_1 = _mm_movemask_epi8(struct2); - uint64_t structural_res_2 = _mm_movemask_epi8(struct3); - uint64_t structural_res_3 = _mm_movemask_epi8(struct4); - - structurals = (structural_res_0 | (structural_res_1 << 16) | (structural_res_2 << 32) | (structural_res_3 << 48)); -} -#endif // __SSE4_2__ - -#if defined(__ARM_NEON) || (defined(_MSC_VER) && defined(_M_ARM64)) -template<> really_inline -void find_whitespace_and_structurals( - simd_input in, - uint64_t &whitespace, - uint64_t &structurals) { - const uint8x16_t low_nibble_mask = (uint8x16_t){ - 16, 0, 0, 0, 0, 0, 0, 0, 0, 8, 12, 1, 2, 9, 0, 0}; - const uint8x16_t high_nibble_mask = (uint8x16_t){ - 8, 0, 18, 4, 0, 1, 0, 1, 0, 0, 0, 3, 2, 1, 0, 0}; - const uint8x16_t structural_shufti_mask = vmovq_n_u8(0x7); - const uint8x16_t whitespace_shufti_mask = vmovq_n_u8(0x18); - const uint8x16_t low_nib_and_mask = vmovq_n_u8(0xf); - - uint8x16_t nib_0_lo = vandq_u8(in.i0, low_nib_and_mask); - uint8x16_t nib_0_hi = vshrq_n_u8(in.i0, 4); - uint8x16_t shuf_0_lo = vqtbl1q_u8(low_nibble_mask, nib_0_lo); - uint8x16_t shuf_0_hi = vqtbl1q_u8(high_nibble_mask, nib_0_hi); - uint8x16_t v_0 = vandq_u8(shuf_0_lo, shuf_0_hi); - - uint8x16_t nib_1_lo = vandq_u8(in.i1, low_nib_and_mask); - uint8x16_t nib_1_hi = vshrq_n_u8(in.i1, 4); - uint8x16_t shuf_1_lo = vqtbl1q_u8(low_nibble_mask, nib_1_lo); - uint8x16_t shuf_1_hi = vqtbl1q_u8(high_nibble_mask, nib_1_hi); - uint8x16_t v_1 = vandq_u8(shuf_1_lo, shuf_1_hi); - - uint8x16_t nib_2_lo = vandq_u8(in.i2, low_nib_and_mask); - uint8x16_t nib_2_hi = vshrq_n_u8(in.i2, 4); - uint8x16_t shuf_2_lo = vqtbl1q_u8(low_nibble_mask, nib_2_lo); - uint8x16_t shuf_2_hi = vqtbl1q_u8(high_nibble_mask, nib_2_hi); - uint8x16_t v_2 = vandq_u8(shuf_2_lo, shuf_2_hi); - - uint8x16_t nib_3_lo = vandq_u8(in.i3, low_nib_and_mask); - uint8x16_t nib_3_hi = vshrq_n_u8(in.i3, 4); - uint8x16_t shuf_3_lo = vqtbl1q_u8(low_nibble_mask, nib_3_lo); - uint8x16_t shuf_3_hi = vqtbl1q_u8(high_nibble_mask, nib_3_hi); - uint8x16_t v_3 = vandq_u8(shuf_3_lo, shuf_3_hi); - - uint8x16_t tmp_0 = vtstq_u8(v_0, structural_shufti_mask); - uint8x16_t tmp_1 = vtstq_u8(v_1, structural_shufti_mask); - uint8x16_t tmp_2 = vtstq_u8(v_2, structural_shufti_mask); - uint8x16_t tmp_3 = vtstq_u8(v_3, structural_shufti_mask); - structurals = neonmovemask_bulk(tmp_0, tmp_1, tmp_2, tmp_3); - - uint8x16_t tmp_ws_0 = vtstq_u8(v_0, whitespace_shufti_mask); - uint8x16_t tmp_ws_1 = vtstq_u8(v_1, whitespace_shufti_mask); - uint8x16_t tmp_ws_2 = vtstq_u8(v_2, whitespace_shufti_mask); - uint8x16_t tmp_ws_3 = vtstq_u8(v_3, whitespace_shufti_mask); - whitespace = neonmovemask_bulk(tmp_ws_0, tmp_ws_1, tmp_ws_2, tmp_ws_3); -} -#endif // __ARM_NEON - - -#ifdef SIMDJSON_NAIVE_FLATTEN // useful for benchmarking -// -// This is just a naive implementation. It should be normally -// disable, but can be used for research purposes to compare -// again our optimized version. -really_inline void flatten_bits(uint32_t *base_ptr, uint32_t &base, - uint32_t idx, uint64_t bits) { - uint32_t * out_ptr = base_ptr + base; - idx -= 64; - while(bits != 0) { - out_ptr[0] = idx + trailingzeroes(bits); - bits = bits & (bits - 1); - out_ptr++; - } - base = (out_ptr - base_ptr); -} - -#else -// flatten out values in 'bits' assuming that they are are to have values of idx -// plus their position in the bitvector, and store these indexes at -// base_ptr[base] incrementing base as we go -// will potentially store extra values beyond end of valid bits, so base_ptr -// needs to be large enough to handle this -really_inline void flatten_bits(uint32_t *base_ptr, uint32_t &base, - uint32_t idx, uint64_t bits) { - // In some instances, the next branch is expensive because it is mispredicted. - // Unfortunately, in other cases, - // it helps tremendously. - if(bits == 0) return; - uint32_t cnt = hamming(bits); - uint32_t next_base = base + cnt; - idx -= 64; - base_ptr += base; - { - base_ptr[0] = idx + trailingzeroes(bits); - bits = bits & (bits - 1); - base_ptr[1] = idx + trailingzeroes(bits); - bits = bits & (bits - 1); - base_ptr[2] = idx + trailingzeroes(bits); - bits = bits & (bits - 1); - base_ptr[3] = idx + trailingzeroes(bits); - bits = bits & (bits - 1); - base_ptr[4] = idx + trailingzeroes(bits); - bits = bits & (bits - 1); - base_ptr[5] = idx + trailingzeroes(bits); - bits = bits & (bits - 1); - base_ptr[6] = idx + trailingzeroes(bits); - bits = bits & (bits - 1); - base_ptr[7] = idx + trailingzeroes(bits); - bits = bits & (bits - 1); - base_ptr += 8; - } - // We hope that the next branch is easily predicted. - if (cnt > 8) { - base_ptr[0] = idx + trailingzeroes(bits); - bits = bits & (bits - 1); - base_ptr[1] = idx + trailingzeroes(bits); - bits = bits & (bits - 1); - base_ptr[2] = idx + trailingzeroes(bits); - bits = bits & (bits - 1); - base_ptr[3] = idx + trailingzeroes(bits); - bits = bits & (bits - 1); - base_ptr[4] = idx + trailingzeroes(bits); - bits = bits & (bits - 1); - base_ptr[5] = idx + trailingzeroes(bits); - bits = bits & (bits - 1); - base_ptr[6] = idx + trailingzeroes(bits); - bits = bits & (bits - 1); - base_ptr[7] = idx + trailingzeroes(bits); - bits = bits & (bits - 1); - base_ptr += 8; - } - if (cnt > 16) { // unluckly: we rarely get here - // since it means having one structural or pseudo-structral element - // every 4 characters (possible with inputs like "","","",...). - do { - base_ptr[0] = idx + trailingzeroes(bits); - bits = bits & (bits - 1); - base_ptr++; - } while(bits != 0); - } - base = next_base; -} -#endif // SIMDJSON_NAIVE_FLATTEN - // return a updated structural bit vector with quoted contents cleared out and // pseudo-structural characters added to the mask // updates prev_iter_ends_pseudo_pred which tells us whether the previous @@ -880,154 +114,21 @@ really_inline uint64_t finalize_structurals( return structurals; } -template -WARN_UNUSED -/*never_inline*/ int find_structural_bits(const uint8_t *buf, size_t len, - ParsedJson &pj) { - if (len > pj.bytecapacity) { - std::cerr << "Your ParsedJson object only supports documents up to " - << pj.bytecapacity << " bytes but you are trying to process " << len - << " bytes" << std::endl; - return simdjson::CAPACITY; - } - uint32_t *base_ptr = pj.structural_indexes; - uint32_t base = 0; -#ifdef SIMDJSON_UTF8VALIDATE - utf8_checking_state state; -#endif +template +int find_structural_bits(const uint8_t *buf, size_t len, simdjson::ParsedJson &pj); - // we have padded the input out to 64 byte multiple with the remainder being - // zeros - - // persistent state across loop - // does the last iteration end with an odd-length sequence of backslashes? - // either 0 or 1, but a 64-bit value - uint64_t prev_iter_ends_odd_backslash = 0ULL; - // does the previous iteration end inside a double-quote pair? - uint64_t prev_iter_inside_quote = 0ULL; // either all zeros or all ones - // does the previous iteration end on something that is a predecessor of a - // pseudo-structural character - i.e. whitespace or a structural character - // effectively the very first char is considered to follow "whitespace" for - // the - // purposes of pseudo-structural character detection so we initialize to 1 - uint64_t prev_iter_ends_pseudo_pred = 1ULL; - - // structurals are persistent state across loop as we flatten them on the - // subsequent iteration into our array pointed to be base_ptr. - // This is harmless on the first iteration as structurals==0 - // and is done for performance reasons; we can hide some of the latency of the - // expensive carryless multiply in the previous step with this work - uint64_t structurals = 0; - - size_t lenminus64 = len < 64 ? 0 : len - 64; - size_t idx = 0; - uint64_t error_mask = 0; // for unescaped characters within strings (ASCII code points < 0x20) - - for (; idx < lenminus64; idx += 64) { -#ifndef _MSC_VER - __builtin_prefetch(buf + idx + 128); -#endif - simd_input in = fill_input(buf+idx); -#ifdef SIMDJSON_UTF8VALIDATE - check_utf8(in, state); -#endif - // detect odd sequences of backslashes - uint64_t odd_ends = find_odd_backslash_sequences( - in, prev_iter_ends_odd_backslash); - - // detect insides of quote pairs ("quote_mask") and also our quote_bits - // themselves - uint64_t quote_bits; - uint64_t quote_mask = find_quote_mask_and_bits( - in, odd_ends, prev_iter_inside_quote, quote_bits, error_mask); - - // take the previous iterations structural bits, not our current iteration, - // and flatten - flatten_bits(base_ptr, base, idx, structurals); - - uint64_t whitespace; - find_whitespace_and_structurals(in, whitespace, structurals); - - // fixup structurals to reflect quotes and add pseudo-structural characters - structurals = finalize_structurals(structurals, whitespace, quote_mask, - quote_bits, prev_iter_ends_pseudo_pred); - } - - //////////////// - /// we use a giant copy-paste which is ugly. - /// but otherwise the string needs to be properly padded or else we - /// risk invalidating the UTF-8 checks. - //////////// - if (idx < len) { - uint8_t tmpbuf[64]; - memset(tmpbuf, 0x20, 64); - memcpy(tmpbuf, buf + idx, len - idx); - simd_input in = fill_input(tmpbuf); -#ifdef SIMDJSON_UTF8VALIDATE - check_utf8(in, state); -#endif - - // detect odd sequences of backslashes - uint64_t odd_ends = find_odd_backslash_sequences( - in, prev_iter_ends_odd_backslash); - - // detect insides of quote pairs ("quote_mask") and also our quote_bits - // themselves - uint64_t quote_bits; - uint64_t quote_mask = find_quote_mask_and_bits( - in, odd_ends, prev_iter_inside_quote, quote_bits, error_mask); - - // take the previous iterations structural bits, not our current iteration, - // and flatten - flatten_bits(base_ptr, base, idx, structurals); - - uint64_t whitespace; - find_whitespace_and_structurals(in, whitespace, structurals); - - // fixup structurals to reflect quotes and add pseudo-structural characters - structurals = finalize_structurals(structurals, whitespace, quote_mask, - quote_bits, prev_iter_ends_pseudo_pred); - idx += 64; - } - - // is last string quote closed? - if (prev_iter_inside_quote) { - return simdjson::UNCLOSED_STRING; - } - - // finally, flatten out the remaining structurals from the last iteration - flatten_bits(base_ptr, base, idx, structurals); - - pj.n_structural_indexes = base; - // a valid JSON file cannot have zero structural indexes - we should have - // found something - if (pj.n_structural_indexes == 0u) { - return simdjson::EMPTY; - } - if (base_ptr[pj.n_structural_indexes - 1] > len) { - return simdjson::UNEXPECTED_ERROR; - } - if (len != base_ptr[pj.n_structural_indexes - 1]) { - // the string might not be NULL terminated, but we add a virtual NULL ending - // character. - base_ptr[pj.n_structural_indexes++] = len; - } - // make it safe to dereference one beyond this array - base_ptr[pj.n_structural_indexes] = 0; - if (error_mask) { - return simdjson::UNESCAPED_CHARS; - } -#ifdef SIMDJSON_UTF8VALIDATE - return check_utf8_errors(state); -#else - return simdjson::SUCCESS; -#endif +template +int find_structural_bits(const char *buf, size_t len, simdjson::ParsedJson &pj) { + return find_structural_bits((const uint8_t*)buf, len, pj); } -template -WARN_UNUSED -int find_structural_bits(const char *buf, size_t len, ParsedJson &pj) { - return find_structural_bits(reinterpret_cast(buf), len, pj); -} -} + +} // namespace simdjson + + + + + + + #endif diff --git a/include/simdjson/stage1_find_marks_arm64.h b/include/simdjson/stage1_find_marks_arm64.h new file mode 100644 index 000000000..69cb069c2 --- /dev/null +++ b/include/simdjson/stage1_find_marks_arm64.h @@ -0,0 +1,197 @@ +#ifndef SIMDJSON_STAGE1_FIND_MARKS_ARM64_H +#define SIMDJSON_STAGE1_FIND_MARKS_ARM64_H + +#include "simdjson/stage1_find_marks.h" +#include "simdjson/stage1_find_marks_macros.h" +#include "simdjson/stage1_find_marks_flatten.h" +#include "simdjson/simdutf8check_arm64.h" + +#ifdef IS_ARM64 +namespace simdjson { +template<> struct simd_input { + uint8x16_t i0; + uint8x16_t i1; + uint8x16_t i2; + uint8x16_t i3; +}; + +template<> really_inline +simd_input fill_input(const uint8_t * ptr) { + struct simd_input in; + in.i0 = vld1q_u8(ptr + 0); + in.i1 = vld1q_u8(ptr + 16); + in.i2 = vld1q_u8(ptr + 32); + in.i3 = vld1q_u8(ptr + 48); + return in; +} + + +really_inline +uint16_t neonmovemask(uint8x16_t input) { + const uint8x16_t bitmask = { 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80, + 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80}; + uint8x16_t minput = vandq_u8(input, bitmask); + uint8x16_t tmp = vpaddq_u8(minput, minput); + tmp = vpaddq_u8(tmp, tmp); + tmp = vpaddq_u8(tmp, tmp); + return vgetq_lane_u16(vreinterpretq_u16_u8(tmp), 0); +} + +really_inline +uint64_t neonmovemask_bulk(uint8x16_t p0, uint8x16_t p1, uint8x16_t p2, uint8x16_t p3) { + const uint8x16_t bitmask = { 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80, + 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80}; + uint8x16_t t0 = vandq_u8(p0, bitmask); + uint8x16_t t1 = vandq_u8(p1, bitmask); + uint8x16_t t2 = vandq_u8(p2, bitmask); + uint8x16_t t3 = vandq_u8(p3, bitmask); + uint8x16_t sum0 = vpaddq_u8(t0, t1); + uint8x16_t sum1 = vpaddq_u8(t2, t3); + sum0 = vpaddq_u8(sum0, sum1); + sum0 = vpaddq_u8(sum0, sum0); + return vgetq_lane_u64(vreinterpretq_u64_u8(sum0), 0); +} + +template<> really_inline +uint64_t compute_quote_mask(uint64_t quote_bits) { +#ifdef __ARM_FEATURE_CRYPTO // some ARM processors lack this extension + return vmull_p64( -1ULL, quote_bits); +#else + return portable_compute_quote_mask(quote_bits); +#endif +} + +template<> +struct utf8_checking_state +{ + int8x16_t has_error {}; + processed_utf_bytes previous {}; +}; + +// Checks that all bytes are ascii +really_inline +bool check_ascii_neon(simd_input in) { + // checking if the most significant bit is always equal to 0. + uint8x16_t highbit = vdupq_n_u8(0x80); + uint8x16_t t0 = vorrq_u8(in.i0, in.i1); + uint8x16_t t1 = vorrq_u8(in.i2, in.i3); + uint8x16_t t3 = vorrq_u8(t0, t1); + uint8x16_t t4 = vandq_u8(t3, highbit); + uint64x2_t v64 = vreinterpretq_u64_u8(t4); + uint32x2_t v32 = vqmovn_u64(v64); + uint64x1_t result = vreinterpret_u64_u32(v32); + return vget_lane_u64(result, 0) == 0; +} + +template<> really_inline +void check_utf8(simd_input in, + utf8_checking_state& state) { + if (check_ascii_neon(in)) { + // All bytes are ascii. Therefore the byte that was just before must be ascii too. + // We only check the byte that was just before simd_input. Nines are arbitrary values. + const int8x16_t verror = (int8x16_t){9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 1}; + state.has_error = + vorrq_s8(vreinterpretq_s8_u8(vcgtq_s8(state.previous.carried_continuations, + verror)), + state.has_error); + } else { + // it is not ascii so we have to do heavy work + state.previous = checkUTF8Bytes(vreinterpretq_s8_u8(in.i0), &(state.previous), &(state.has_error)); + state.previous = checkUTF8Bytes(vreinterpretq_s8_u8(in.i1), &(state.previous), &(state.has_error)); + state.previous = checkUTF8Bytes(vreinterpretq_s8_u8(in.i2), &(state.previous), &(state.has_error)); + state.previous = checkUTF8Bytes(vreinterpretq_s8_u8(in.i3), &(state.previous), &(state.has_error)); + } +} + +template<> really_inline +errorValues check_utf8_errors(utf8_checking_state& state) { + uint64x2_t v64 = vreinterpretq_u64_s8(state.has_error); + uint32x2_t v32 = vqmovn_u64(v64); + uint64x1_t result = vreinterpret_u64_u32(v32); + return vget_lane_u64(result, 0) != 0 ? simdjson::UTF8_ERROR : simdjson::SUCCESS; +} + +template<> really_inline +uint64_t cmp_mask_against_input(simd_input in, uint8_t m) { + const uint8x16_t mask = vmovq_n_u8(m); + uint8x16_t cmp_res_0 = vceqq_u8(in.i0, mask); + uint8x16_t cmp_res_1 = vceqq_u8(in.i1, mask); + uint8x16_t cmp_res_2 = vceqq_u8(in.i2, mask); + uint8x16_t cmp_res_3 = vceqq_u8(in.i3, mask); + return neonmovemask_bulk(cmp_res_0, cmp_res_1, cmp_res_2, cmp_res_3); +} + +template<> really_inline +uint64_t unsigned_lteq_against_input(simd_input in, uint8_t m) { + const uint8x16_t mask = vmovq_n_u8(m); + uint8x16_t cmp_res_0 = vcleq_u8(in.i0, mask); + uint8x16_t cmp_res_1 = vcleq_u8(in.i1, mask); + uint8x16_t cmp_res_2 = vcleq_u8(in.i2, mask); + uint8x16_t cmp_res_3 = vcleq_u8(in.i3, mask); + return neonmovemask_bulk(cmp_res_0, cmp_res_1, cmp_res_2, cmp_res_3); +} + +template<> really_inline +uint64_t find_odd_backslash_sequences(simd_input in, uint64_t &prev_iter_ends_odd_backslash) { + FIND_ODD_BACKSLASH_SEQUENCES(architecture::arm64, in, prev_iter_ends_odd_backslash); +} + +template<> really_inline +uint64_t find_quote_mask_and_bits(simd_input in, uint64_t odd_ends, + uint64_t &prev_iter_inside_quote, uint64_t "e_bits, uint64_t &error_mask) { + FIND_QUOTE_MASK_AND_BITS(architecture::arm64, in, odd_ends, prev_iter_inside_quote, quote_bits, error_mask) +} + +template<> really_inline +void find_whitespace_and_structurals( + simd_input in, + uint64_t &whitespace, + uint64_t &structurals) { + const uint8x16_t low_nibble_mask = (uint8x16_t){ + 16, 0, 0, 0, 0, 0, 0, 0, 0, 8, 12, 1, 2, 9, 0, 0}; + const uint8x16_t high_nibble_mask = (uint8x16_t){ + 8, 0, 18, 4, 0, 1, 0, 1, 0, 0, 0, 3, 2, 1, 0, 0}; + const uint8x16_t structural_shufti_mask = vmovq_n_u8(0x7); + const uint8x16_t whitespace_shufti_mask = vmovq_n_u8(0x18); + const uint8x16_t low_nib_and_mask = vmovq_n_u8(0xf); + + uint8x16_t nib_0_lo = vandq_u8(in.i0, low_nib_and_mask); + uint8x16_t nib_0_hi = vshrq_n_u8(in.i0, 4); + uint8x16_t shuf_0_lo = vqtbl1q_u8(low_nibble_mask, nib_0_lo); + uint8x16_t shuf_0_hi = vqtbl1q_u8(high_nibble_mask, nib_0_hi); + uint8x16_t v_0 = vandq_u8(shuf_0_lo, shuf_0_hi); + + uint8x16_t nib_1_lo = vandq_u8(in.i1, low_nib_and_mask); + uint8x16_t nib_1_hi = vshrq_n_u8(in.i1, 4); + uint8x16_t shuf_1_lo = vqtbl1q_u8(low_nibble_mask, nib_1_lo); + uint8x16_t shuf_1_hi = vqtbl1q_u8(high_nibble_mask, nib_1_hi); + uint8x16_t v_1 = vandq_u8(shuf_1_lo, shuf_1_hi); + + uint8x16_t nib_2_lo = vandq_u8(in.i2, low_nib_and_mask); + uint8x16_t nib_2_hi = vshrq_n_u8(in.i2, 4); + uint8x16_t shuf_2_lo = vqtbl1q_u8(low_nibble_mask, nib_2_lo); + uint8x16_t shuf_2_hi = vqtbl1q_u8(high_nibble_mask, nib_2_hi); + uint8x16_t v_2 = vandq_u8(shuf_2_lo, shuf_2_hi); + + uint8x16_t nib_3_lo = vandq_u8(in.i3, low_nib_and_mask); + uint8x16_t nib_3_hi = vshrq_n_u8(in.i3, 4); + uint8x16_t shuf_3_lo = vqtbl1q_u8(low_nibble_mask, nib_3_lo); + uint8x16_t shuf_3_hi = vqtbl1q_u8(high_nibble_mask, nib_3_hi); + uint8x16_t v_3 = vandq_u8(shuf_3_lo, shuf_3_hi); + + uint8x16_t tmp_0 = vtstq_u8(v_0, structural_shufti_mask); + uint8x16_t tmp_1 = vtstq_u8(v_1, structural_shufti_mask); + uint8x16_t tmp_2 = vtstq_u8(v_2, structural_shufti_mask); + uint8x16_t tmp_3 = vtstq_u8(v_3, structural_shufti_mask); + structurals = neonmovemask_bulk(tmp_0, tmp_1, tmp_2, tmp_3); + + uint8x16_t tmp_ws_0 = vtstq_u8(v_0, whitespace_shufti_mask); + uint8x16_t tmp_ws_1 = vtstq_u8(v_1, whitespace_shufti_mask); + uint8x16_t tmp_ws_2 = vtstq_u8(v_2, whitespace_shufti_mask); + uint8x16_t tmp_ws_3 = vtstq_u8(v_3, whitespace_shufti_mask); + whitespace = neonmovemask_bulk(tmp_ws_0, tmp_ws_1, tmp_ws_2, tmp_ws_3); +} +}// simdjson namespace + +#endif // IS_ARM64 +#endif // SIMDJSON_STAGE1_FIND_MARKS_ARM64_H \ No newline at end of file diff --git a/include/simdjson/stage1_find_marks_flatten.h b/include/simdjson/stage1_find_marks_flatten.h new file mode 100644 index 000000000..cb76f7c01 --- /dev/null +++ b/include/simdjson/stage1_find_marks_flatten.h @@ -0,0 +1,92 @@ +#ifndef SIMDJSON_STAGE1_FIND_MARKS_FLATTEN_H +#define SIMDJSON_STAGE1_FIND_MARKS_FLATTEN_H + +namespace simdjson { + +#ifdef SIMDJSON_NAIVE_FLATTEN // useful for benchmarking +// +// This is just a naive implementation. It should be normally +// disable, but can be used for research purposes to compare +// again our optimized version. +really_inline void flatten_bits(uint32_t *base_ptr, uint32_t &base, + uint32_t idx, uint64_t bits) { + uint32_t * out_ptr = base_ptr + base; + idx -= 64; + while(bits != 0) { + out_ptr[0] = idx + trailingzeroes(bits); + bits = bits & (bits - 1); + out_ptr++; + } + base = (out_ptr - base_ptr); +} + +#else +// flatten out values in 'bits' assuming that they are are to have values of idx +// plus their position in the bitvector, and store these indexes at +// base_ptr[base] incrementing base as we go +// will potentially store extra values beyond end of valid bits, so base_ptr +// needs to be large enough to handle this +really_inline void flatten_bits(uint32_t *base_ptr, uint32_t &base, + uint32_t idx, uint64_t bits) { + // In some instances, the next branch is expensive because it is mispredicted. + // Unfortunately, in other cases, + // it helps tremendously. + if(bits == 0) return; + uint32_t cnt = hamming(bits); + uint32_t next_base = base + cnt; + idx -= 64; + base_ptr += base; + { + base_ptr[0] = idx + trailingzeroes(bits); + bits = bits & (bits - 1); + base_ptr[1] = idx + trailingzeroes(bits); + bits = bits & (bits - 1); + base_ptr[2] = idx + trailingzeroes(bits); + bits = bits & (bits - 1); + base_ptr[3] = idx + trailingzeroes(bits); + bits = bits & (bits - 1); + base_ptr[4] = idx + trailingzeroes(bits); + bits = bits & (bits - 1); + base_ptr[5] = idx + trailingzeroes(bits); + bits = bits & (bits - 1); + base_ptr[6] = idx + trailingzeroes(bits); + bits = bits & (bits - 1); + base_ptr[7] = idx + trailingzeroes(bits); + bits = bits & (bits - 1); + base_ptr += 8; + } + // We hope that the next branch is easily predicted. + if (cnt > 8) { + base_ptr[0] = idx + trailingzeroes(bits); + bits = bits & (bits - 1); + base_ptr[1] = idx + trailingzeroes(bits); + bits = bits & (bits - 1); + base_ptr[2] = idx + trailingzeroes(bits); + bits = bits & (bits - 1); + base_ptr[3] = idx + trailingzeroes(bits); + bits = bits & (bits - 1); + base_ptr[4] = idx + trailingzeroes(bits); + bits = bits & (bits - 1); + base_ptr[5] = idx + trailingzeroes(bits); + bits = bits & (bits - 1); + base_ptr[6] = idx + trailingzeroes(bits); + bits = bits & (bits - 1); + base_ptr[7] = idx + trailingzeroes(bits); + bits = bits & (bits - 1); + base_ptr += 8; + } + if (cnt > 16) { // unluckly: we rarely get here + // since it means having one structural or pseudo-structral element + // every 4 characters (possible with inputs like "","","",...). + do { + base_ptr[0] = idx + trailingzeroes(bits); + bits = bits & (bits - 1); + base_ptr++; + } while(bits != 0); + } + base = next_base; +} +#endif // SIMDJSON_NAIVE_FLATTEN +} + +#endif // SIMDJSON_STAGE1_FIND_MARKS_FLATTEN_H \ No newline at end of file diff --git a/include/simdjson/stage1_find_marks_haswell.h b/include/simdjson/stage1_find_marks_haswell.h new file mode 100644 index 000000000..a2e707614 --- /dev/null +++ b/include/simdjson/stage1_find_marks_haswell.h @@ -0,0 +1,189 @@ +#ifndef SIMDJSON_STAGE1_FIND_MARKS_HASWELL_H +#define SIMDJSON_STAGE1_FIND_MARKS_HASWELL_H + +#include "simdjson/stage1_find_marks.h" +#include "simdjson/stage1_find_marks_macros.h" +#include "simdjson/stage1_find_marks_flatten.h" +#include "simdjson/simdutf8check_haswell.h" + +#ifdef IS_X86_64 + +TARGET_HASWELL +namespace simdjson { +template<> +struct simd_input { + __m256i lo; + __m256i hi; +}; + +template<> really_inline +simd_input fill_input(const uint8_t * ptr) { + struct simd_input in; + in.lo = _mm256_loadu_si256(reinterpret_cast(ptr + 0)); + in.hi = _mm256_loadu_si256(reinterpret_cast(ptr + 32)); + return in; +} + +template<> really_inline +uint64_t compute_quote_mask(uint64_t quote_bits) { + // There should be no such thing with a processing supporting avx2 + // but not clmul. + uint64_t quote_mask = _mm_cvtsi128_si64(_mm_clmulepi64_si128( + _mm_set_epi64x(0ULL, quote_bits), _mm_set1_epi8(0xFF), 0)); + return quote_mask; +} + +template<> +struct utf8_checking_state { + __m256i has_error; + avx_processed_utf_bytes previous; + utf8_checking_state() { + has_error = _mm256_setzero_si256(); + previous.rawbytes = _mm256_setzero_si256(); + previous.high_nibbles = _mm256_setzero_si256(); + previous.carried_continuations =_mm256_setzero_si256(); + } +}; + + +template<> really_inline +void check_utf8(simd_input in, + utf8_checking_state& state) { + __m256i highbit = _mm256_set1_epi8(0x80); + if ((_mm256_testz_si256(_mm256_or_si256(in.lo, in.hi), highbit)) == 1) { + // it is ascii, we just check continuation + state.has_error = _mm256_or_si256( + _mm256_cmpgt_epi8( + state.previous.carried_continuations, + _mm256_setr_epi8(9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 1)), + state.has_error); + } else { + // it is not ascii so we have to do heavy work + state.previous = avxcheckUTF8Bytes(in.lo, &(state.previous), &(state.has_error)); + state.previous = avxcheckUTF8Bytes(in.hi, &(state.previous), &(state.has_error)); + } +} + +template<> really_inline +errorValues check_utf8_errors(utf8_checking_state& state) { + return _mm256_testz_si256(state.has_error, state.has_error) == 0 ? simdjson::UTF8_ERROR : simdjson::SUCCESS; +} + +template<> really_inline +uint64_t cmp_mask_against_input(simd_input in, uint8_t m) { + const __m256i mask = _mm256_set1_epi8(m); + __m256i cmp_res_0 = _mm256_cmpeq_epi8(in.lo, mask); + uint64_t res_0 = static_cast(_mm256_movemask_epi8(cmp_res_0)); + __m256i cmp_res_1 = _mm256_cmpeq_epi8(in.hi, mask); + uint64_t res_1 = _mm256_movemask_epi8(cmp_res_1); + return res_0 | (res_1 << 32); +} + +template<> really_inline +uint64_t unsigned_lteq_against_input(simd_input in, uint8_t m) { + const __m256i maxval = _mm256_set1_epi8(m); + __m256i cmp_res_0 = _mm256_cmpeq_epi8(_mm256_max_epu8(maxval,in.lo),maxval); + uint64_t res_0 = static_cast(_mm256_movemask_epi8(cmp_res_0)); + __m256i cmp_res_1 = _mm256_cmpeq_epi8(_mm256_max_epu8(maxval,in.hi),maxval); + uint64_t res_1 = _mm256_movemask_epi8(cmp_res_1); + return res_0 | (res_1 << 32); +} + +template<> really_inline +uint64_t find_odd_backslash_sequences(simd_input in, uint64_t &prev_iter_ends_odd_backslash) { + FIND_ODD_BACKSLASH_SEQUENCES(architecture::haswell, in, prev_iter_ends_odd_backslash); +} + +template<> really_inline +uint64_t find_quote_mask_and_bits(simd_input in, uint64_t odd_ends, + uint64_t &prev_iter_inside_quote, uint64_t "e_bits, uint64_t &error_mask) { + FIND_QUOTE_MASK_AND_BITS(architecture::haswell, in, odd_ends, prev_iter_inside_quote, quote_bits, error_mask) +} + +template<> really_inline +void find_whitespace_and_structurals(simd_input in, + uint64_t &whitespace, + uint64_t &structurals) { +#ifdef SIMDJSON_NAIVE_STRUCTURAL + // You should never need this naive approach, but it can be useful + // for research purposes + const __m256i mask_open_brace = _mm256_set1_epi8(0x7b); + __m256i struct_lo = _mm256_cmpeq_epi8(in.lo, mask_open_brace); + __m256i struct_hi = _mm256_cmpeq_epi8(in.hi, mask_open_brace); + const __m256i mask_close_brace = _mm256_set1_epi8(0x7d); + struct_lo = _mm256_or_si256(struct_lo,_mm256_cmpeq_epi8(in.lo, mask_close_brace)); + struct_hi = _mm256_or_si256(struct_hi,_mm256_cmpeq_epi8(in.hi, mask_close_brace)); + const __m256i mask_open_bracket = _mm256_set1_epi8(0x5b); + struct_lo = _mm256_or_si256(struct_lo,_mm256_cmpeq_epi8(in.lo, mask_open_bracket)); + struct_hi = _mm256_or_si256(struct_hi,_mm256_cmpeq_epi8(in.hi, mask_open_bracket)); + const __m256i mask_close_bracket = _mm256_set1_epi8(0x5d); + struct_lo = _mm256_or_si256(struct_lo,_mm256_cmpeq_epi8(in.lo, mask_close_bracket)); + struct_hi = _mm256_or_si256(struct_hi,_mm256_cmpeq_epi8(in.hi, mask_close_bracket)); + const __m256i mask_column = _mm256_set1_epi8(0x3a); + struct_lo = _mm256_or_si256(struct_lo,_mm256_cmpeq_epi8(in.lo, mask_column)); + struct_hi = _mm256_or_si256(struct_hi,_mm256_cmpeq_epi8(in.hi, mask_column)); + const __m256i mask_comma = _mm256_set1_epi8(0x2c); + struct_lo = _mm256_or_si256(struct_lo,_mm256_cmpeq_epi8(in.lo, mask_comma)); + struct_hi = _mm256_or_si256(struct_hi,_mm256_cmpeq_epi8(in.hi, mask_comma)); + uint64_t structural_res_0 = static_cast(_mm256_movemask_epi8(struct_lo)); + uint64_t structural_res_1 = _mm256_movemask_epi8(struct_hi); + structurals = (structural_res_0 | (structural_res_1 << 32)); + + const __m256i mask_space = _mm256_set1_epi8(0x20); + __m256i space_lo = _mm256_cmpeq_epi8(in.lo, mask_space); + __m256i space_hi = _mm256_cmpeq_epi8(in.hi, mask_space); + const __m256i mask_linefeed = _mm256_set1_epi8(0x0a); + space_lo = _mm256_or_si256(space_lo,_mm256_cmpeq_epi8(in.lo, mask_linefeed)); + space_hi = _mm256_or_si256(space_hi,_mm256_cmpeq_epi8(in.hi, mask_linefeed)); + const __m256i mask_tab = _mm256_set1_epi8(0x09); + space_lo = _mm256_or_si256(space_lo,_mm256_cmpeq_epi8(in.lo, mask_tab)); + space_hi = _mm256_or_si256(space_hi,_mm256_cmpeq_epi8(in.hi, mask_tab)); + const __m256i mask_carriage = _mm256_set1_epi8(0x0d); + space_lo = _mm256_or_si256(space_lo,_mm256_cmpeq_epi8(in.lo, mask_carriage)); + space_hi = _mm256_or_si256(space_hi,_mm256_cmpeq_epi8(in.hi, mask_carriage)); + + uint64_t ws_res_0 = static_cast(_mm256_movemask_epi8(space_lo)); + uint64_t ws_res_1 = _mm256_movemask_epi8(space_hi); + whitespace = (ws_res_0 | (ws_res_1 << 32)); + // end of naive approach + +#else // SIMDJSON_NAIVE_STRUCTURAL + const __m256i structural_table = _mm256_setr_epi8( + 44, 125, 0, 0, 0xc0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 123, + 44, 125, 0, 0, 0xc0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 123); + const __m256i white_table = _mm256_setr_epi8( + 32, 100, 100, 100, 17, 100, 113, 2, 100, 9, 10, 112, 100, 13, 100, 100, + 32, 100, 100, 100, 17, 100, 113, 2, 100, 9, 10, 112, 100, 13, 100, 100); + const __m256i struct_offset = _mm256_set1_epi8(0xd4); + const __m256i struct_mask = _mm256_set1_epi8(32); + + __m256i lo_white = _mm256_cmpeq_epi8(in.lo, + _mm256_shuffle_epi8(white_table, in.lo)); + __m256i hi_white = _mm256_cmpeq_epi8(in.hi, + _mm256_shuffle_epi8(white_table, in.hi)); + uint64_t ws_res_0 = static_cast(_mm256_movemask_epi8(lo_white)); + uint64_t ws_res_1 = _mm256_movemask_epi8(hi_white); + whitespace = (ws_res_0 | (ws_res_1 << 32)); + __m256i lo_struct_r1 = _mm256_add_epi8(struct_offset, in.lo); + __m256i hi_struct_r1 = _mm256_add_epi8(struct_offset, in.hi); + __m256i lo_struct_r2 = _mm256_or_si256(in.lo, struct_mask); + __m256i hi_struct_r2 = _mm256_or_si256(in.hi, struct_mask); + __m256i lo_struct_r3 = _mm256_shuffle_epi8(structural_table, lo_struct_r1); + __m256i hi_struct_r3 = _mm256_shuffle_epi8(structural_table, hi_struct_r1); + __m256i lo_struct = _mm256_cmpeq_epi8(lo_struct_r2, lo_struct_r3); + __m256i hi_struct = _mm256_cmpeq_epi8(hi_struct_r2, hi_struct_r3); + + uint64_t structural_res_0 = + static_cast(_mm256_movemask_epi8(lo_struct)); + uint64_t structural_res_1 = _mm256_movemask_epi8(hi_struct); + structurals = (structural_res_0 | (structural_res_1 << 32)); +#endif // SIMDJSON_NAIVE_STRUCTURAL +} + +} // namespace simdjson +UNTARGET_REGION + + +#endif // IS_X86_64 +#endif // SIMDJSON_STAGE1_FIND_MARKS_HASWELL_H \ No newline at end of file diff --git a/include/simdjson/stage1_find_marks_macros.h b/include/simdjson/stage1_find_marks_macros.h new file mode 100644 index 000000000..5e43f2c7e --- /dev/null +++ b/include/simdjson/stage1_find_marks_macros.h @@ -0,0 +1,220 @@ +#ifndef SIMDJSON_STAGE1_FIND_MARKS_MACROS_H +#define SIMDJSON_STAGE1_FIND_MARKS_MACROS_H + +// return a bitvector indicating where we have characters that end an odd-length +// sequence of backslashes (and thus change the behavior of the next character +// to follow). A even-length sequence of backslashes, and, for that matter, the +// largest even-length prefix of our odd-length sequence of backslashes, simply +// modify the behavior of the backslashes themselves. +// We also update the prev_iter_ends_odd_backslash reference parameter to +// indicate whether we end an iteration on an odd-length sequence of +// backslashes, which modifies our subsequent search for odd-length +// sequences of backslashes in an obvious way. +// We need to compile that code for multiple architectures. However, target attributes can be used +// only once by function definition. Huge macro seemed better than huge code duplication. +// uint64_t FIND_ODD_BACKSLASH_SEQUENCES(architecture T, simd_input in, uint64_t &prev_iter_ends_odd_backslash) +#define FIND_ODD_BACKSLASH_SEQUENCES(T, in, prev_iter_ends_odd_backslash) { \ + const uint64_t even_bits = 0x5555555555555555ULL; \ + const uint64_t odd_bits = ~even_bits; \ + uint64_t bs_bits = cmp_mask_against_input(in, '\\'); \ + uint64_t start_edges = bs_bits & ~(bs_bits << 1); \ + /* flip lowest if we have an odd-length run at the end of the prior */ \ + /* iteration */ \ + uint64_t even_start_mask = even_bits ^ prev_iter_ends_odd_backslash; \ + uint64_t even_starts = start_edges & even_start_mask; \ + uint64_t odd_starts = start_edges & ~even_start_mask; \ + uint64_t even_carries = bs_bits + even_starts; \ + \ + uint64_t odd_carries; \ + /* must record the carry-out of our odd-carries out of bit 63; this */ \ + /* indicates whether the sense of any edge going to the next iteration */ \ + /* should be flipped */ \ + bool iter_ends_odd_backslash = \ + add_overflow(bs_bits, odd_starts, &odd_carries); \ + \ + odd_carries |= \ + prev_iter_ends_odd_backslash; /* push in bit zero as a potential end */ \ + /* if we had an odd-numbered run at the */ \ + /* end of the previous iteration */ \ + prev_iter_ends_odd_backslash = iter_ends_odd_backslash ? 0x1ULL : 0x0ULL; \ + uint64_t even_carry_ends = even_carries & ~bs_bits; \ + uint64_t odd_carry_ends = odd_carries & ~bs_bits; \ + uint64_t even_start_odd_end = even_carry_ends & odd_bits; \ + uint64_t odd_start_even_end = odd_carry_ends & even_bits; \ + uint64_t odd_ends = even_start_odd_end | odd_start_even_end; \ + return odd_ends; \ +} + + +// return both the quote mask (which is a half-open mask that covers the first +// quote +// in an unescaped quote pair and everything in the quote pair) and the quote +// bits, which are the simple +// unescaped quoted bits. We also update the prev_iter_inside_quote value to +// tell the next iteration +// whether we finished the final iteration inside a quote pair; if so, this +// inverts our behavior of +// whether we're inside quotes for the next iteration. +// Note that we don't do any error checking to see if we have backslash +// sequences outside quotes; these +// backslash sequences (of any length) will be detected elsewhere. +// We need to compile that code for multiple architectures. However, target attributes can be used +// only once by function definition. Huge macro seemed better than huge code duplication. +// uint64_t FIND_QUOTE_MASK_AND_BITS(architecture T, simd_input in, uint64_t odd_ends, +// uint64_t &prev_iter_inside_quote, uint64_t "e_bits, uint64_t &error_mask) +#define FIND_QUOTE_MASK_AND_BITS(T, in, odd_ends, prev_iter_inside_quote, quote_bits, error_mask) { \ + quote_bits = cmp_mask_against_input(in, '"'); \ + quote_bits = quote_bits & ~odd_ends; \ + uint64_t quote_mask = compute_quote_mask(quote_bits); \ + quote_mask ^= prev_iter_inside_quote; \ + /* All Unicode characters may be placed within the */ \ + /* quotation marks, except for the characters that MUST be escaped: */ \ + /* quotation mark, reverse solidus, and the control characters (U+0000 */ \ + /*through U+001F). */ \ + /* https://tools.ietf.org/html/rfc8259 */ \ + uint64_t unescaped = unsigned_lteq_against_input(in, 0x1F); \ + error_mask |= quote_mask & unescaped; \ + /* right shift of a signed value expected to be well-defined and standard */ \ + /* compliant as of C++20, */ \ + /* John Regher from Utah U. says this is fine code */ \ + prev_iter_inside_quote = \ + static_cast(static_cast(quote_mask) >> 63); \ + return quote_mask; \ +} \ + + + +// We need to compile that code for multiple architectures. However, target attributes can be used +// only once by function definition. Huge macro seemed better than huge code duplication. +// FIND_STRUCTURAL_BITS(architecture T, const uint8_t *buf, size_t len, ParsedJson &pj) +#define FIND_STRUCTURAL_BITS(T, buf, len, pj) { \ + if (len > pj.bytecapacity) { \ + std::cerr << "Your ParsedJson object only supports documents up to " \ + << pj.bytecapacity << " bytes but you are trying to process " << len \ + << " bytes" << std::endl; \ + return simdjson::CAPACITY; \ + } \ + uint32_t *base_ptr = pj.structural_indexes; \ + uint32_t base = 0; \ + utf8_checking_state state; \ + \ + /* we have padded the input out to 64 byte multiple with the remainder being */ \ + /* zeros */ \ + \ + /* persistent state across loop */ \ + /* does the last iteration end with an odd-length sequence of backslashes? */ \ + /* either 0 or 1, but a 64-bit value */ \ + uint64_t prev_iter_ends_odd_backslash = 0ULL; \ + /* does the previous iteration end inside a double-quote pair? */ \ + uint64_t prev_iter_inside_quote = 0ULL; /* either all zeros or all ones */ \ + /* does the previous iteration end on something that is a predecessor of a */ \ + /* pseudo-structural character - i.e. whitespace or a structural character */ \ + /* effectively the very first char is considered to follow "whitespace" for */ \ + /* the */ \ + /* purposes of pseudo-structural character detection so we initialize to 1 */ \ + uint64_t prev_iter_ends_pseudo_pred = 1ULL; \ + \ + /* structurals are persistent state across loop as we flatten them on the */ \ + /* subsequent iteration into our array pointed to be base_ptr. */ \ + /* This is harmless on the first iteration as structurals==0 */ \ + /* and is done for performance reasons; we can hide some of the latency of the */ \ + /* expensive carryless multiply in the previous step with this work */ \ + uint64_t structurals = 0; \ + \ + size_t lenminus64 = len < 64 ? 0 : len - 64; \ + size_t idx = 0; \ + uint64_t error_mask = 0; /* for unescaped characters within strings (ASCII code points < 0x20) */ \ + \ + for (; idx < lenminus64; idx += 64) { \ + \ + simd_input in = fill_input(buf+idx); \ + check_utf8(in, state); \ + /* detect odd sequences of backslashes */ \ + uint64_t odd_ends = find_odd_backslash_sequences( \ + in, prev_iter_ends_odd_backslash); \ + \ + /* detect insides of quote pairs ("quote_mask") and also our quote_bits */ \ + /* themselves */ \ + uint64_t quote_bits; \ + uint64_t quote_mask = find_quote_mask_and_bits( \ + in, odd_ends, prev_iter_inside_quote, quote_bits, error_mask); \ + \ + /* take the previous iterations structural bits, not our current iteration, */ \ + /* and flatten */ \ + flatten_bits(base_ptr, base, idx, structurals); \ + \ + uint64_t whitespace; \ + find_whitespace_and_structurals(in, whitespace, structurals); \ + \ + /* fixup structurals to reflect quotes and add pseudo-structural characters */ \ + structurals = finalize_structurals(structurals, whitespace, quote_mask, \ + quote_bits, prev_iter_ends_pseudo_pred); \ + } \ + \ + /*////////////// */ \ + /*/ we use a giant copy-paste which is ugly. */ \ + /*/ but otherwise the string needs to be properly padded or else we */ \ + /*/ risk invalidating the UTF-8 checks. */ \ + /*////////// */ \ + if (idx < len) { \ + uint8_t tmpbuf[64]; \ + memset(tmpbuf, 0x20, 64); \ + memcpy(tmpbuf, buf + idx, len - idx); \ + simd_input in = fill_input(tmpbuf); \ + check_utf8(in, state); \ + \ + /* detect odd sequences of backslashes */ \ + uint64_t odd_ends = find_odd_backslash_sequences( \ + in, prev_iter_ends_odd_backslash); \ + \ + /* detect insides of quote pairs ("quote_mask") and also our quote_bits */ \ + /* themselves */ \ + uint64_t quote_bits; \ + uint64_t quote_mask = find_quote_mask_and_bits( \ + in, odd_ends, prev_iter_inside_quote, quote_bits, error_mask); \ + \ + /* take the previous iterations structural bits, not our current iteration, */ \ + /* and flatten */ \ + flatten_bits(base_ptr, base, idx, structurals); \ + \ + uint64_t whitespace; \ + find_whitespace_and_structurals(in, whitespace, structurals); \ + \ + /* fixup structurals to reflect quotes and add pseudo-structural characters */ \ + structurals = finalize_structurals(structurals, whitespace, quote_mask, \ + quote_bits, prev_iter_ends_pseudo_pred); \ + idx += 64; \ + } \ + \ + /* is last string quote closed? */ \ + if (prev_iter_inside_quote) { \ + return simdjson::UNCLOSED_STRING; \ + } \ + \ + /* finally, flatten out the remaining structurals from the last iteration */ \ + flatten_bits(base_ptr, base, idx, structurals); \ + \ + pj.n_structural_indexes = base; \ + /* a valid JSON file cannot have zero structural indexes - we should have */ \ + /* found something */ \ + if (pj.n_structural_indexes == 0u) { \ + return simdjson::EMPTY; \ + } \ + if (base_ptr[pj.n_structural_indexes - 1] > len) { \ + return simdjson::UNEXPECTED_ERROR; \ + } \ + if (len != base_ptr[pj.n_structural_indexes - 1]) { \ + /* the string might not be NULL terminated, but we add a virtual NULL ending */ \ + /* character. */ \ + base_ptr[pj.n_structural_indexes++] = len; \ + } \ + /* make it safe to dereference one beyond this array */ \ + base_ptr[pj.n_structural_indexes] = 0; \ + if (error_mask) { \ + return simdjson::UNESCAPED_CHARS; \ + } \ + return check_utf8_errors(state); \ +} + + +#endif // SIMDJSON_STAGE1_FIND_MARKS_MACROS_H \ No newline at end of file diff --git a/include/simdjson/stage1_find_marks_westmere.h b/include/simdjson/stage1_find_marks_westmere.h new file mode 100644 index 000000000..376bce363 --- /dev/null +++ b/include/simdjson/stage1_find_marks_westmere.h @@ -0,0 +1,181 @@ +#ifndef SIMDJSON_STAGE1_FIND_MARKS_WESTMERE_H +#define SIMDJSON_STAGE1_FIND_MARKS_WESTMERE_H + +#include "simdjson/stage1_find_marks.h" +#include "simdjson/stage1_find_marks_macros.h" +#include "simdjson/stage1_find_marks_flatten.h" +#include "simdjson/simdutf8check_westmere.h" + +#ifdef IS_X86_64 + +TARGET_WESTMERE +namespace simdjson { +template<> +struct simd_input { + __m128i v0; + __m128i v1; + __m128i v2; + __m128i v3; +}; + +template<> really_inline +simd_input fill_input(const uint8_t * ptr) { + struct simd_input in; + in.v0 = _mm_loadu_si128(reinterpret_cast(ptr + 0)); + in.v1 = _mm_loadu_si128(reinterpret_cast(ptr + 16)); + in.v2 = _mm_loadu_si128(reinterpret_cast(ptr + 32)); + in.v3 = _mm_loadu_si128(reinterpret_cast(ptr + 48)); + return in; +} + +template<> really_inline +uint64_t compute_quote_mask(uint64_t quote_bits) { + return _mm_cvtsi128_si64(_mm_clmulepi64_si128( + _mm_set_epi64x(0ULL, quote_bits), _mm_set1_epi8(0xFF), 0)); +} + +template<> +struct utf8_checking_state +{ + __m128i has_error = _mm_setzero_si128(); + processed_utf_bytes previous { + _mm_setzero_si128(), // rawbytes + _mm_setzero_si128(), // high_nibbles + _mm_setzero_si128() // carried_continuations + }; +}; + +template<> really_inline +void check_utf8(simd_input in, + utf8_checking_state& state) { + __m128i highbit = _mm_set1_epi8(0x80); + if ((_mm_testz_si128(_mm_or_si128(in.v0, in.v1), highbit)) == 1) { + // it is ascii, we just check continuation + state.has_error = _mm_or_si128( + _mm_cmpgt_epi8( + state.previous.carried_continuations, + _mm_setr_epi8(9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 1)), + state.has_error); + } else { + // it is not ascii so we have to do heavy work + state.previous = checkUTF8Bytes(in.v0, &(state.previous), &(state.has_error)); + state.previous = checkUTF8Bytes(in.v1, &(state.previous), &(state.has_error)); + } + + if ((_mm_testz_si128(_mm_or_si128(in.v2, in.v3), highbit)) == 1) { + // it is ascii, we just check continuation + state.has_error = _mm_or_si128( + _mm_cmpgt_epi8( + state.previous.carried_continuations, + _mm_setr_epi8(9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 1)), + state.has_error); + } else { + // it is not ascii so we have to do heavy work + state.previous = checkUTF8Bytes(in.v2, &(state.previous), &(state.has_error)); + state.previous = checkUTF8Bytes(in.v3, &(state.previous), &(state.has_error)); + } +} + +template<> really_inline +errorValues check_utf8_errors(utf8_checking_state& state) { + return _mm_testz_si128(state.has_error, state.has_error) == 0 ? simdjson::UTF8_ERROR : simdjson::SUCCESS; +} + +template<> really_inline +uint64_t cmp_mask_against_input(simd_input in, uint8_t m) { + const __m128i mask = _mm_set1_epi8(m); + __m128i cmp_res_0 = _mm_cmpeq_epi8(in.v0, mask); + uint64_t res_0 = _mm_movemask_epi8(cmp_res_0); + __m128i cmp_res_1 = _mm_cmpeq_epi8(in.v1, mask); + uint64_t res_1 = _mm_movemask_epi8(cmp_res_1); + __m128i cmp_res_2 = _mm_cmpeq_epi8(in.v2, mask); + uint64_t res_2 = _mm_movemask_epi8(cmp_res_2); + __m128i cmp_res_3 = _mm_cmpeq_epi8(in.v3, mask); + uint64_t res_3 = _mm_movemask_epi8(cmp_res_3); + return res_0 | (res_1 << 16) | (res_2 << 32) | (res_3 << 48); +} + +template<> really_inline +uint64_t unsigned_lteq_against_input(simd_input in, uint8_t m) { + const __m128i maxval = _mm_set1_epi8(m); + __m128i cmp_res_0 = _mm_cmpeq_epi8(_mm_max_epu8(maxval,in.v0),maxval); + uint64_t res_0 = _mm_movemask_epi8(cmp_res_0); + __m128i cmp_res_1 = _mm_cmpeq_epi8(_mm_max_epu8(maxval,in.v1),maxval); + uint64_t res_1 = _mm_movemask_epi8(cmp_res_1); + __m128i cmp_res_2 = _mm_cmpeq_epi8(_mm_max_epu8(maxval,in.v2),maxval); + uint64_t res_2 = _mm_movemask_epi8(cmp_res_2); + __m128i cmp_res_3 = _mm_cmpeq_epi8(_mm_max_epu8(maxval,in.v3),maxval); + uint64_t res_3 = _mm_movemask_epi8(cmp_res_3); + return res_0 | (res_1 << 16) | (res_2 << 32) | (res_3 << 48); +} + +template<> really_inline +uint64_t find_odd_backslash_sequences(simd_input in, uint64_t &prev_iter_ends_odd_backslash) { + FIND_ODD_BACKSLASH_SEQUENCES(architecture::westmere, in, prev_iter_ends_odd_backslash); +} + +template<> really_inline +uint64_t find_quote_mask_and_bits(simd_input in, uint64_t odd_ends, + uint64_t &prev_iter_inside_quote, uint64_t "e_bits, uint64_t &error_mask) { + FIND_QUOTE_MASK_AND_BITS(architecture::westmere, in, odd_ends, prev_iter_inside_quote, quote_bits, error_mask) +} + +template<> really_inline +void find_whitespace_and_structurals(simd_input in, + uint64_t &whitespace, uint64_t &structurals) { + const __m128i structural_table = _mm_setr_epi8(44, 125, 0, 0, 0xc0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 123); + const __m128i white_table = _mm_setr_epi8( + 32, 100, 100, 100, 17, 100, 113, 2, 100, 9, 10, 112, 100, 13, 100, 100); + const __m128i struct_offset = _mm_set1_epi8(0xd4); + const __m128i struct_mask = _mm_set1_epi8(32); + + __m128i white0 = _mm_cmpeq_epi8(in.v0, + _mm_shuffle_epi8(white_table, in.v0)); + __m128i white1 = _mm_cmpeq_epi8(in.v1, + _mm_shuffle_epi8(white_table, in.v1)); + __m128i white2 = _mm_cmpeq_epi8(in.v2, + _mm_shuffle_epi8(white_table, in.v2)); + __m128i white3 = _mm_cmpeq_epi8(in.v3, + _mm_shuffle_epi8(white_table, in.v3)); + uint64_t ws_res_0 = _mm_movemask_epi8(white0); + uint64_t ws_res_1 = _mm_movemask_epi8(white1); + uint64_t ws_res_2 = _mm_movemask_epi8(white2); + uint64_t ws_res_3 = _mm_movemask_epi8(white3); + + whitespace = (ws_res_0 | (ws_res_1 << 16) | (ws_res_2 << 32) | (ws_res_3 << 48)); + + __m128i struct1_r1 = _mm_add_epi8(struct_offset, in.v0); + __m128i struct2_r1 = _mm_add_epi8(struct_offset, in.v1); + __m128i struct3_r1 = _mm_add_epi8(struct_offset, in.v2); + __m128i struct4_r1 = _mm_add_epi8(struct_offset, in.v3); + + __m128i struct1_r2 = _mm_or_si128(in.v0, struct_mask); + __m128i struct2_r2 = _mm_or_si128(in.v1, struct_mask); + __m128i struct3_r2 = _mm_or_si128(in.v2, struct_mask); + __m128i struct4_r2 = _mm_or_si128(in.v3, struct_mask); + + __m128i struct1_r3 = _mm_shuffle_epi8(structural_table, struct1_r1); + __m128i struct2_r3 = _mm_shuffle_epi8(structural_table, struct2_r1); + __m128i struct3_r3 = _mm_shuffle_epi8(structural_table, struct3_r1); + __m128i struct4_r3 = _mm_shuffle_epi8(structural_table, struct4_r1); + + __m128i struct1 = _mm_cmpeq_epi8(struct1_r2, struct1_r3); + __m128i struct2 = _mm_cmpeq_epi8(struct2_r2, struct2_r3); + __m128i struct3 = _mm_cmpeq_epi8(struct3_r2, struct3_r3); + __m128i struct4 = _mm_cmpeq_epi8(struct4_r2, struct4_r3); + + uint64_t structural_res_0 = _mm_movemask_epi8(struct1); + uint64_t structural_res_1 = _mm_movemask_epi8(struct2); + uint64_t structural_res_2 = _mm_movemask_epi8(struct3); + uint64_t structural_res_3 = _mm_movemask_epi8(struct4); + + structurals = (structural_res_0 | (structural_res_1 << 16) | (structural_res_2 << 32) | (structural_res_3 << 48)); +} + + +} // namespace simdjson +UNTARGET_REGION + + +#endif // IS_X86_64 +#endif // SIMDJSON_STAGE1_FIND_MARKS_WESTMERE_H \ No newline at end of file diff --git a/include/simdjson/stage2_build_tape.h b/include/simdjson/stage2_build_tape.h index 6f7b1b6da..36990f647 100644 --- a/include/simdjson/stage2_build_tape.h +++ b/include/simdjson/stage2_build_tape.h @@ -68,519 +68,15 @@ really_inline bool is_valid_null_atom(const uint8_t *loc) { return error == 0; } - -/************ - * The JSON is parsed to a tape, see the accompanying tape.md file - * for documentation. - ***********/ -template +template WARN_UNUSED ALLOW_SAME_PAGE_BUFFER_OVERRUN_QUALIFIER LENIENT_MEM_SANITIZER -int unified_machine(const uint8_t *buf, size_t len, ParsedJson &pj) { -#ifndef ALLOW_SAME_PAGE_BUFFER_OVERRUN - memset((uint8_t*)buf + len, 0, SIMDJSON_PADDING); // to please valgrind -#endif - uint32_t i = 0; // index of the structural character (0,1,2,3...) - uint32_t idx; // location of the structural character in the input (buf) - uint8_t c; // used to track the (structural) character we are looking at, updated - // by UPDATE_CHAR macro - uint32_t depth = 0; // could have an arbitrary starting depth - pj.init(); // sets isvalid to false - if(pj.bytecapacity < len) { - pj.errorcode = simdjson::CAPACITY; - return pj.errorcode; - } -// this macro reads the next structural character, updating idx, i and c. -#define UPDATE_CHAR() \ - { \ - idx = pj.structural_indexes[i++]; \ - c = buf[idx]; \ - } +int unified_machine(const uint8_t *buf, size_t len, ParsedJson &pj); - - ////////////////////////////// START STATE ///////////////////////////// -#ifdef SIMDJSON_USE_COMPUTED_GOTO - pj.ret_address[depth] = &&start_continue; -#else - pj.ret_address[depth] = 's'; -#endif - pj.containing_scope_offset[depth] = pj.get_current_loc(); - pj.write_tape(0, 'r'); // r for root, 0 is going to get overwritten - // the root is used, if nothing else, to capture the size of the tape - depth++; // everything starts at depth = 1, depth = 0 is just for the root, the root may contain an object, an array or something else. - if (depth >= pj.depthcapacity) { - goto fail; - } - - UPDATE_CHAR(); - switch (c) { - case '{': - pj.containing_scope_offset[depth] = pj.get_current_loc(); -#ifdef SIMDJSON_USE_COMPUTED_GOTO - pj.ret_address[depth] = &&start_continue; -#else - pj.ret_address[depth] = 's'; -#endif - depth++; - if (depth >= pj.depthcapacity) { - goto fail; - } - pj.write_tape(0, c); // strangely, moving this to object_begin slows things down - goto object_begin; - case '[': - pj.containing_scope_offset[depth] = pj.get_current_loc(); -#ifdef SIMDJSON_USE_COMPUTED_GOTO - pj.ret_address[depth] = &&start_continue; -#else - pj.ret_address[depth] = 's'; -#endif - depth++; - if (depth >= pj.depthcapacity) { - goto fail; - } - pj.write_tape(0, c); - goto array_begin; -#define SIMDJSON_ALLOWANYTHINGINROOT - // A JSON text is a serialized value. Note that certain previous - // specifications of JSON constrained a JSON text to be an object or an - // array. Implementations that generate only objects or arrays where a - // JSON text is called for will be interoperable in the sense that all - // implementations will accept these as conforming JSON texts. - // https://tools.ietf.org/html/rfc8259 -#ifdef SIMDJSON_ALLOWANYTHINGINROOT - case '"': { - if (!parse_string(buf, len, pj, depth, idx)) { - goto fail; - } - break; - } - case 't': { - // we need to make a copy to make sure that the string is space terminated. - // this only applies to the JSON document made solely of the true value. - // this will almost never be called in practice - char * copy = static_cast(malloc(len + SIMDJSON_PADDING)); - if(copy == nullptr) { - goto fail; - } - memcpy(copy, buf, len); - copy[len] = ' '; - if (!is_valid_true_atom(reinterpret_cast(copy) + idx)) { - free(copy); - goto fail; - } - free(copy); - pj.write_tape(0, c); - break; - } - case 'f': { - // we need to make a copy to make sure that the string is space terminated. - // this only applies to the JSON document made solely of the false value. - // this will almost never be called in practice - char * copy = static_cast(malloc(len + SIMDJSON_PADDING)); - if(copy == nullptr) { - goto fail; - } - memcpy(copy, buf, len); - copy[len] = ' '; - if (!is_valid_false_atom(reinterpret_cast(copy) + idx)) { - free(copy); - goto fail; - } - free(copy); - pj.write_tape(0, c); - break; - } - case 'n': { - // we need to make a copy to make sure that the string is space terminated. - // this only applies to the JSON document made solely of the null value. - // this will almost never be called in practice - char * copy = static_cast(malloc(len + SIMDJSON_PADDING)); - if(copy == nullptr) { - goto fail; - } - memcpy(copy, buf, len); - copy[len] = ' '; - if (!is_valid_null_atom(reinterpret_cast(copy) + idx)) { - free(copy); - goto fail; - } - free(copy); - pj.write_tape(0, c); - break; - } - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': { - // we need to make a copy to make sure that the string is space terminated. - // this is done only for JSON documents made of a sole number - // this will almost never be called in practice. We terminate with a space - // because we do not want to allow NULLs in the middle of a number (whereas a - // space in the middle of a number would be identified in stage 1). - char * copy = static_cast(malloc(len + SIMDJSON_PADDING)); - if(copy == nullptr) { - goto fail; - } - memcpy(copy, buf, len); - copy[len] = ' '; - if (!parse_number(reinterpret_cast(copy), pj, idx, false)) { - free(copy); - goto fail; - } - free(copy); - break; - } - case '-': { - // we need to make a copy to make sure that the string is NULL terminated. - // this is done only for JSON documents made of a sole number - // this will almost never be called in practice - char * copy = static_cast(malloc(len + SIMDJSON_PADDING)); - if(copy == nullptr) { - goto fail; - } - memcpy(copy, buf, len); - copy[len] = '\0'; - if (!parse_number(reinterpret_cast(copy), pj, idx, true)) { - free(copy); - goto fail; - } - free(copy); - break; - } -#endif // ALLOWANYTHINGINROOT - default: - goto fail; - } -start_continue: - // the string might not be NULL terminated. - if(i + 1 == pj.n_structural_indexes) { - goto succeed; - } else { - goto fail; - } - ////////////////////////////// OBJECT STATES ///////////////////////////// - -object_begin: - UPDATE_CHAR(); - switch (c) { - case '"': { - if (!parse_string(buf, len, pj, depth, idx)) { - goto fail; - } - goto object_key_state; - } - case '}': - goto scope_end; // could also go to object_continue - default: - goto fail; - } - -object_key_state: - UPDATE_CHAR(); - if (c != ':') { - goto fail; - } - UPDATE_CHAR(); - switch (c) { - case '"': { - if (!parse_string(buf, len, pj, depth, idx)) { - goto fail; - } - break; - } - case 't': - if (!is_valid_true_atom(buf + idx)) { - goto fail; - } - pj.write_tape(0, c); - break; - case 'f': - if (!is_valid_false_atom(buf + idx)) { - goto fail; - } - pj.write_tape(0, c); - break; - case 'n': - if (!is_valid_null_atom(buf + idx)) { - goto fail; - } - pj.write_tape(0, c); - break; - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': { - if (!parse_number(buf, pj, idx, false)) { - goto fail; - } - break; - } - case '-': { - if (!parse_number(buf, pj, idx, true)) { - goto fail; - } - break; - } - case '{': { - pj.containing_scope_offset[depth] = pj.get_current_loc(); - pj.write_tape(0, c); // here the compilers knows what c is so this gets optimized - // we have not yet encountered } so we need to come back for it -#ifdef SIMDJSON_USE_COMPUTED_GOTO - pj.ret_address[depth] = &&object_continue; -#else - pj.ret_address[depth] = 'o'; -#endif - // we found an object inside an object, so we need to increment the depth - depth++; - if (depth >= pj.depthcapacity) { - goto fail; - } - - goto object_begin; - } - case '[': { - pj.containing_scope_offset[depth] = pj.get_current_loc(); - pj.write_tape(0, c); // here the compilers knows what c is so this gets optimized - // we have not yet encountered } so we need to come back for it -#ifdef SIMDJSON_USE_COMPUTED_GOTO - pj.ret_address[depth] = &&object_continue; -#else - pj.ret_address[depth] = 'o'; -#endif - // we found an array inside an object, so we need to increment the depth - depth++; - if (depth >= pj.depthcapacity) { - goto fail; - } - goto array_begin; - } - default: - goto fail; - } - -object_continue: - UPDATE_CHAR(); - switch (c) { - case ',': - UPDATE_CHAR(); - if (c != '"') { - goto fail; - } else { - if (!parse_string(buf, len, pj, depth, idx)) { - goto fail; - } - goto object_key_state; - } - case '}': - goto scope_end; - default: - goto fail; - } - - ////////////////////////////// COMMON STATE ///////////////////////////// - -scope_end: - // write our tape location to the header scope - depth--; - pj.write_tape(pj.containing_scope_offset[depth], c); - pj.annotate_previousloc(pj.containing_scope_offset[depth], - pj.get_current_loc()); - // goto saved_state -#ifdef SIMDJSON_USE_COMPUTED_GOTO - goto *pj.ret_address[depth]; -#else - if(pj.ret_address[depth] == 'a') { - goto array_continue; - } else if (pj.ret_address[depth] == 'o') { - goto object_continue; - } else goto start_continue; -#endif - - ////////////////////////////// ARRAY STATES ///////////////////////////// -array_begin: - UPDATE_CHAR(); - if (c == ']') { - goto scope_end; // could also go to array_continue - } - -main_array_switch: - // we call update char on all paths in, so we can peek at c on the - // on paths that can accept a close square brace (post-, and at start) - switch (c) { - case '"': { - if (!parse_string(buf, len, pj, depth, idx)) { - goto fail; - } - break; - } - case 't': - if (!is_valid_true_atom(buf + idx)) { - goto fail; - } - pj.write_tape(0, c); - break; - case 'f': - if (!is_valid_false_atom(buf + idx)) { - goto fail; - } - pj.write_tape(0, c); - break; - case 'n': - if (!is_valid_null_atom(buf + idx)) { - goto fail; - } - pj.write_tape(0, c); - break; // goto array_continue; - - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': { - if (!parse_number(buf, pj, idx, false)) { - goto fail; - } - break; // goto array_continue; - } - case '-': { - if (!parse_number(buf, pj, idx, true)) { - goto fail; - } - break; // goto array_continue; - } - case '{': { - // we have not yet encountered ] so we need to come back for it - pj.containing_scope_offset[depth] = pj.get_current_loc(); - pj.write_tape(0, c); // here the compilers knows what c is so this gets optimized -#ifdef SIMDJSON_USE_COMPUTED_GOTO - pj.ret_address[depth] = &&array_continue; -#else - pj.ret_address[depth] = 'a'; -#endif - // we found an object inside an array, so we need to increment the depth - depth++; - if (depth >= pj.depthcapacity) { - goto fail; - } - - goto object_begin; - } - case '[': { - // we have not yet encountered ] so we need to come back for it - pj.containing_scope_offset[depth] = pj.get_current_loc(); - pj.write_tape(0, c); // here the compilers knows what c is so this gets optimized -#ifdef SIMDJSON_USE_COMPUTED_GOTO - pj.ret_address[depth] = &&array_continue; -#else - pj.ret_address[depth] = 'a'; -#endif - // we found an array inside an array, so we need to increment the depth - depth++; - if (depth >= pj.depthcapacity) { - goto fail; - } - goto array_begin; - } - default: - goto fail; - } - -array_continue: - UPDATE_CHAR(); - switch (c) { - case ',': - UPDATE_CHAR(); - goto main_array_switch; - case ']': - goto scope_end; - default: - goto fail; - } - - ////////////////////////////// FINAL STATES ///////////////////////////// - -succeed: - depth --; - if(depth != 0) { - fprintf(stderr, "internal bug\n"); - abort(); - } - if(pj.containing_scope_offset[depth] != 0) { - fprintf(stderr, "internal bug\n"); - abort(); - } - pj.annotate_previousloc(pj.containing_scope_offset[depth], - pj.get_current_loc()); - pj.write_tape(pj.containing_scope_offset[depth], 'r'); // r is root - - pj.isvalid = true; - pj.errorcode = simdjson::SUCCESS; - return pj.errorcode; -fail: - // we do not need the next line because this is done by pj.init(), pessimistically. - // pj.isvalid = false; - // At this point in the code, we have all the time in the world. - // Note that we know exactly where we are in the document so we could, - // without any overhead on the processing code, report a specific location. - // We could even trigger special code paths to assess what happened carefully, - // all without any added cost. - if (depth >= pj.depthcapacity) { - pj.errorcode = simdjson::DEPTH_ERROR; - return pj.errorcode; - } - switch(c) { - case '"': - pj.errorcode = simdjson::STRING_ERROR; - return pj.errorcode; - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': - case '-': - pj.errorcode = simdjson::NUMBER_ERROR; - return pj.errorcode; - case 't': - pj.errorcode = simdjson::T_ATOM_ERROR; - return pj.errorcode; - case 'n': - pj.errorcode = simdjson::N_ATOM_ERROR; - return pj.errorcode; - case 'f': - pj.errorcode = simdjson::F_ATOM_ERROR; - return pj.errorcode; - default: - break; - } - pj.errorcode = simdjson::TAPE_ERROR; - return pj.errorcode; -} - -template +template int unified_machine(const char *buf, size_t len, ParsedJson &pj) { return unified_machine(reinterpret_cast(buf), len, pj); } + } #endif diff --git a/include/simdjson/stringparsing.h b/include/simdjson/stringparsing.h index b0332ca6a..ca097eb1b 100644 --- a/include/simdjson/stringparsing.h +++ b/include/simdjson/stringparsing.h @@ -84,169 +84,22 @@ struct parse_string_helper { }; // Finds where the backslashes and quotes are located. -template +template parse_string_helper find_bs_bits_and_quote_bits(const uint8_t *src, uint8_t *dst); -#ifdef __AVX2__ -template<> really_inline -parse_string_helper find_bs_bits_and_quote_bits (const uint8_t *src, uint8_t *dst) { - // this can read up to 31 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(sizeof(__m256i) - 1 <= SIMDJSON_PADDING); - __m256i v = _mm256_loadu_si256(reinterpret_cast(src)); - // store to dest unconditionally - we can overwrite the bits we don't like - // later - _mm256_storeu_si256(reinterpret_cast<__m256i *>(dst), v); - auto quote_mask = _mm256_cmpeq_epi8(v, _mm256_set1_epi8('"')); - return { - static_cast(_mm256_movemask_epi8(_mm256_cmpeq_epi8(v, _mm256_set1_epi8('\\')))), // bs_bits - static_cast(_mm256_movemask_epi8(quote_mask)) // quote_bits - }; -} -#endif -#if defined(__SSE4_2__) || (defined(_MSC_VER) && defined(_M_AMD64)) -template<> really_inline -parse_string_helper find_bs_bits_and_quote_bits (const uint8_t *src, uint8_t *dst) { - // this can read up to 31 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - __m128i v = _mm_loadu_si128(reinterpret_cast(src)); - // store to dest unconditionally - we can overwrite the bits we don't like - // later - _mm_storeu_si128(reinterpret_cast<__m128i *>(dst), v); - auto quote_mask = _mm_cmpeq_epi8(v, _mm_set1_epi8('"')); - return { - static_cast(_mm_movemask_epi8(_mm_cmpeq_epi8(v, _mm_set1_epi8('\\')))), // bs_bits - static_cast(_mm_movemask_epi8(quote_mask)) // quote_bits - }; -} -#endif -#ifdef __ARM_NEON -template<> really_inline -parse_string_helper find_bs_bits_and_quote_bits (const uint8_t *src, uint8_t *dst) { - // this can read up to 31 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(2 * sizeof(uint8x16_t) - 1 <= SIMDJSON_PADDING); - uint8x16_t v0 = vld1q_u8(src); - uint8x16_t v1 = vld1q_u8(src+16); - vst1q_u8(dst, v0); - vst1q_u8(dst+16, v1); - - uint8x16_t bs_mask = vmovq_n_u8('\\'); - uint8x16_t qt_mask = vmovq_n_u8('"'); - const uint8x16_t bitmask = { 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80, - 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80}; - uint8x16_t cmp_bs_0 = vceqq_u8(v0, bs_mask); - uint8x16_t cmp_bs_1 = vceqq_u8(v1, bs_mask); - uint8x16_t cmp_qt_0 = vceqq_u8(v0, qt_mask); - uint8x16_t cmp_qt_1 = vceqq_u8(v1, qt_mask); - - cmp_bs_0 = vandq_u8(cmp_bs_0, bitmask); - cmp_bs_1 = vandq_u8(cmp_bs_1, bitmask); - cmp_qt_0 = vandq_u8(cmp_qt_0, bitmask); - cmp_qt_1 = vandq_u8(cmp_qt_1, bitmask); - - uint8x16_t sum0 = vpaddq_u8(cmp_bs_0, cmp_bs_1); - uint8x16_t sum1 = vpaddq_u8(cmp_qt_0, cmp_qt_1); - sum0 = vpaddq_u8(sum0, sum1); - sum0 = vpaddq_u8(sum0, sum0); - return { - vgetq_lane_u32(vreinterpretq_u32_u8(sum0), 0), // bs_bits - vgetq_lane_u32(vreinterpretq_u32_u8(sum0), 1) // quote_bits - }; -} -#endif - -template +template WARN_UNUSED ALLOW_SAME_PAGE_BUFFER_OVERRUN_QUALIFIER LENIENT_MEM_SANITIZER really_inline bool parse_string(UNUSED const uint8_t *buf, UNUSED size_t len, - ParsedJson &pj, UNUSED const uint32_t depth, UNUSED uint32_t offset) { -#ifdef SIMDJSON_SKIPSTRINGPARSING // for performance analysis, it is sometimes useful to skip parsing - pj.write_tape(0, '"');// don't bother with the string parsing at all - return true; // always succeeds -#else - pj.write_tape(pj.current_string_buf_loc - pj.string_buf, '"'); - const uint8_t *src = &buf[offset + 1]; // we know that buf at offset is a " - uint8_t *dst = pj.current_string_buf_loc + sizeof(uint32_t); - const uint8_t *const start_of_string = dst; - while (1) { - parse_string_helper helper = find_bs_bits_and_quote_bits(src, dst); - if(((helper.bs_bits - 1) & helper.quote_bits) != 0 ) { - // we encountered quotes first. Move dst to point to quotes and exit + ParsedJson &pj, UNUSED const uint32_t depth, UNUSED uint32_t offset); - // find out where the quote is... - uint32_t quote_dist = trailingzeroes(helper.quote_bits); - // NULL termination is still handy if you expect all your strings to be NULL terminated? - // It comes at a small cost - dst[quote_dist] = 0; - - uint32_t str_length = (dst - start_of_string) + quote_dist; - memcpy(pj.current_string_buf_loc,&str_length, sizeof(uint32_t)); - /////////////////////// - // Above, check for overflow in case someone has a crazy string (>=4GB?) - // But only add the overflow check when the document itself exceeds 4GB - // Currently unneeded because we refuse to parse docs larger or equal to 4GB. - //////////////////////// - - - // we advance the point, accounting for the fact that we have a NULL termination - pj.current_string_buf_loc = dst + quote_dist + 1; - -#ifdef JSON_TEST_STRINGS // for unit testing - foundString(buf + offset,start_of_string,pj.current_string_buf_loc - 1); -#endif // JSON_TEST_STRINGS - return true; - } - if(((helper.quote_bits - 1) & helper.bs_bits ) != 0 ) { - // find out where the backspace is - uint32_t bs_dist = trailingzeroes(helper.bs_bits); - uint8_t escape_char = src[bs_dist + 1]; - // we encountered backslash first. Handle backslash - if (escape_char == 'u') { - // move src/dst up to the start; they will be further adjusted - // within the unicode codepoint handling code. - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst)) { -#ifdef JSON_TEST_STRINGS // for unit testing - foundBadString(buf + offset); -#endif // JSON_TEST_STRINGS - return false; - } - } else { - // simple 1:1 conversion. Will eat bs_dist+2 characters in input and - // write bs_dist+1 characters to output - // note this may reach beyond the part of the buffer we've actually - // seen. I think this is ok - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { -#ifdef JSON_TEST_STRINGS // for unit testing - foundBadString(buf + offset); -#endif // JSON_TEST_STRINGS - return false; // bogus escape value is an error - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - // they are the same. Since they can't co-occur, it means we encountered - // neither. - if constexpr(T == instruction_set::sse4_2) { - src += 16; - dst += 16; - } else { - src += 32; - dst += 32; - } - } - } - // can't be reached - return true; -#endif // SIMDJSON_SKIPSTRINGPARSING -} } +/// Now include the specializations: +#include "simdjson/stringparsing_arm64.h" +#include "simdjson/stringparsing_haswell.h" +#include "simdjson/stringparsing_westmere.h" + #endif diff --git a/include/simdjson/stringparsing_arm64.h b/include/simdjson/stringparsing_arm64.h new file mode 100644 index 000000000..11eb6b88c --- /dev/null +++ b/include/simdjson/stringparsing_arm64.h @@ -0,0 +1,51 @@ +#ifndef SIMDJSON_STRINGPARSING_ARM64_H +#define SIMDJSON_STRINGPARSING_ARM64_H + +#include "simdjson/stringparsing.h" +#include "simdjson/stringparsing_macros.h" + +#ifdef IS_ARM64 +namespace simdjson { +template<> really_inline +parse_string_helper find_bs_bits_and_quote_bits (const uint8_t *src, uint8_t *dst) { + // this can read up to 31 bytes beyond the buffer size, but we require + // SIMDJSON_PADDING of padding + static_assert(2 * sizeof(uint8x16_t) - 1 <= SIMDJSON_PADDING); + uint8x16_t v0 = vld1q_u8(src); + uint8x16_t v1 = vld1q_u8(src+16); + vst1q_u8(dst, v0); + vst1q_u8(dst+16, v1); + + uint8x16_t bs_mask = vmovq_n_u8('\\'); + uint8x16_t qt_mask = vmovq_n_u8('"'); + const uint8x16_t bitmask = { 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80, + 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80}; + uint8x16_t cmp_bs_0 = vceqq_u8(v0, bs_mask); + uint8x16_t cmp_bs_1 = vceqq_u8(v1, bs_mask); + uint8x16_t cmp_qt_0 = vceqq_u8(v0, qt_mask); + uint8x16_t cmp_qt_1 = vceqq_u8(v1, qt_mask); + + cmp_bs_0 = vandq_u8(cmp_bs_0, bitmask); + cmp_bs_1 = vandq_u8(cmp_bs_1, bitmask); + cmp_qt_0 = vandq_u8(cmp_qt_0, bitmask); + cmp_qt_1 = vandq_u8(cmp_qt_1, bitmask); + + uint8x16_t sum0 = vpaddq_u8(cmp_bs_0, cmp_bs_1); + uint8x16_t sum1 = vpaddq_u8(cmp_qt_0, cmp_qt_1); + sum0 = vpaddq_u8(sum0, sum1); + sum0 = vpaddq_u8(sum0, sum0); + return { + vgetq_lane_u32(vreinterpretq_u32_u8(sum0), 0), // bs_bits + vgetq_lane_u32(vreinterpretq_u32_u8(sum0), 1) // quote_bits + }; +} + +template<> +WARN_UNUSED ALLOW_SAME_PAGE_BUFFER_OVERRUN_QUALIFIER LENIENT_MEM_SANITIZER really_inline +bool parse_string(UNUSED const uint8_t *buf, UNUSED size_t len, + ParsedJson &pj, UNUSED const uint32_t depth, UNUSED uint32_t offset) { + PARSE_STRING(architecture::arm64, buf, len, pj, depth, offset); +} +} +#endif +#endif diff --git a/include/simdjson/stringparsing_haswell.h b/include/simdjson/stringparsing_haswell.h new file mode 100644 index 000000000..d49daacf2 --- /dev/null +++ b/include/simdjson/stringparsing_haswell.h @@ -0,0 +1,38 @@ +#ifndef SIMDJSON_STRINGPARSING_HASWELL_H +#define SIMDJSON_STRINGPARSING_HASWELL_H + +#include "simdjson/stringparsing.h" +#include "simdjson/stringparsing_macros.h" + + +#ifdef IS_X86_64 +TARGET_HASWELL +namespace simdjson { +template<> really_inline +parse_string_helper find_bs_bits_and_quote_bits (const uint8_t *src, uint8_t *dst) { + // this can read up to 31 bytes beyond the buffer size, but we require + // SIMDJSON_PADDING of padding + static_assert(sizeof(__m256i) - 1 <= SIMDJSON_PADDING); + __m256i v = _mm256_loadu_si256(reinterpret_cast(src)); + // store to dest unconditionally - we can overwrite the bits we don't like + // later + _mm256_storeu_si256(reinterpret_cast<__m256i *>(dst), v); + auto quote_mask = _mm256_cmpeq_epi8(v, _mm256_set1_epi8('"')); + return { + static_cast(_mm256_movemask_epi8(_mm256_cmpeq_epi8(v, _mm256_set1_epi8('\\')))), // bs_bits + static_cast(_mm256_movemask_epi8(quote_mask)) // quote_bits + }; +} + +template<> +WARN_UNUSED ALLOW_SAME_PAGE_BUFFER_OVERRUN_QUALIFIER LENIENT_MEM_SANITIZER really_inline +bool parse_string(UNUSED const uint8_t *buf, UNUSED size_t len, + ParsedJson &pj, UNUSED const uint32_t depth, UNUSED uint32_t offset) { + PARSE_STRING(architecture::haswell, buf, len, pj, depth, offset); +} + +} +UNTARGET_REGION +#endif + +#endif \ No newline at end of file diff --git a/include/simdjson/stringparsing_macros.h b/include/simdjson/stringparsing_macros.h new file mode 100644 index 000000000..ae6c2e3e8 --- /dev/null +++ b/include/simdjson/stringparsing_macros.h @@ -0,0 +1,80 @@ +#ifndef SIMDJSON_STRINGPARSING_MACROS_H +#define SIMDJSON_STRINGPARSING_MACROS_H + +// We need to compile that code for multiple architectures. However, target attributes can be used +// only once by function definition. Huge macro seemed better than huge code duplication. +// bool PARSE_STRING(architecture T, UNUSED const uint8_t *buf, UNUSED size_t len, +// ParsedJson &pj, UNUSED const uint32_t depth, UNUSED uint32_t offset) +#define PARSE_STRING(T, buf, len, pj, depth, offset) { \ + pj.write_tape(pj.current_string_buf_loc - pj.string_buf, '"'); \ + const uint8_t *src = &buf[offset + 1]; /* we know that buf at offset is a " */ \ + uint8_t *dst = pj.current_string_buf_loc + sizeof(uint32_t); \ + const uint8_t *const start_of_string = dst; \ + while (1) { \ + parse_string_helper helper = find_bs_bits_and_quote_bits(src, dst); \ + if(((helper.bs_bits - 1) & helper.quote_bits) != 0 ) { \ + /* we encountered quotes first. Move dst to point to quotes and exit */ \ + \ + /* find out where the quote is... */ \ + uint32_t quote_dist = trailingzeroes(helper.quote_bits); \ + \ + /* NULL termination is still handy if you expect all your strings to be NULL terminated? */ \ + /* It comes at a small cost */ \ + dst[quote_dist] = 0; \ + \ + uint32_t str_length = (dst - start_of_string) + quote_dist; \ + memcpy(pj.current_string_buf_loc,&str_length, sizeof(uint32_t)); \ + /*///////////////////// */ \ + /* Above, check for overflow in case someone has a crazy string (>=4GB?) */ \ + /* But only add the overflow check when the document itself exceeds 4GB */ \ + /* Currently unneeded because we refuse to parse docs larger or equal to 4GB. */ \ + /*////////////////////// */ \ + \ + \ + /* we advance the point, accounting for the fact that we have a NULL termination */ \ + pj.current_string_buf_loc = dst + quote_dist + 1; \ + return true; \ + } \ + if(((helper.quote_bits - 1) & helper.bs_bits ) != 0 ) { \ + /* find out where the backspace is */ \ + uint32_t bs_dist = trailingzeroes(helper.bs_bits); \ + uint8_t escape_char = src[bs_dist + 1]; \ + /* we encountered backslash first. Handle backslash */ \ + if (escape_char == 'u') { \ + /* move src/dst up to the start; they will be further adjusted */ \ + /* within the unicode codepoint handling code. */ \ + src += bs_dist; \ + dst += bs_dist; \ + if (!handle_unicode_codepoint(&src, &dst)) { \ + return false; \ + } \ + } else { \ + /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and */ \ + /* write bs_dist+1 characters to output */ \ + /* note this may reach beyond the part of the buffer we've actually */ \ + /* seen. I think this is ok */ \ + uint8_t escape_result = escape_map[escape_char]; \ + if (escape_result == 0u) { \ + return false; /* bogus escape value is an error */ \ + } \ + dst[bs_dist] = escape_result; \ + src += bs_dist + 2; \ + dst += bs_dist + 1; \ + } \ + } else { \ + /* they are the same. Since they can't co-occur, it means we encountered */ \ + /* neither. */ \ + if constexpr(T == architecture::westmere) { \ + src += 16; \ + dst += 16; \ + } else { \ + src += 32; \ + dst += 32; \ + } \ + } \ + } \ + /* can't be reached */ \ + return true; \ +} + +#endif \ No newline at end of file diff --git a/include/simdjson/stringparsing_westmere.h b/include/simdjson/stringparsing_westmere.h new file mode 100644 index 000000000..b7bb69ad3 --- /dev/null +++ b/include/simdjson/stringparsing_westmere.h @@ -0,0 +1,36 @@ +#ifndef SIMDJSON_STRINGPARSING_WESTMERE_H +#define SIMDJSON_STRINGPARSING_WESTMERE_H + +#include "simdjson/stringparsing.h" +#include "simdjson/stringparsing_macros.h" + + +#ifdef IS_X86_64 +TARGET_WESTMERE +namespace simdjson { +template<> really_inline +parse_string_helper find_bs_bits_and_quote_bits (const uint8_t *src, uint8_t *dst) { + // this can read up to 31 bytes beyond the buffer size, but we require + // SIMDJSON_PADDING of padding + __m128i v = _mm_loadu_si128(reinterpret_cast(src)); + // store to dest unconditionally - we can overwrite the bits we don't like + // later + _mm_storeu_si128(reinterpret_cast<__m128i *>(dst), v); + auto quote_mask = _mm_cmpeq_epi8(v, _mm_set1_epi8('"')); + return { + static_cast(_mm_movemask_epi8(_mm_cmpeq_epi8(v, _mm_set1_epi8('\\')))), // bs_bits + static_cast(_mm_movemask_epi8(quote_mask)) // quote_bits + }; +} + +template<> +WARN_UNUSED ALLOW_SAME_PAGE_BUFFER_OVERRUN_QUALIFIER LENIENT_MEM_SANITIZER really_inline +bool parse_string(UNUSED const uint8_t *buf, UNUSED size_t len, + ParsedJson &pj, UNUSED const uint32_t depth, UNUSED uint32_t offset) { + PARSE_STRING(architecture::westmere, buf, len, pj, depth, offset); +} +} +UNTARGET_REGION +#endif + +#endif \ No newline at end of file diff --git a/singleheader/README.md b/singleheader/README.md index 3f2bb8e6e..fea01d320 100644 --- a/singleheader/README.md +++ b/singleheader/README.md @@ -1 +1 @@ -c++ -march=native -O3 -std=c++17 -o amalgamation_demo amalgamation_demo.cpp && ./amalgamation_demo ../jsonexamples/twitter.json +c++ -O3 -std=c++17 -o amalgamation_demo amalgamation_demo.cpp && ./amalgamation_demo ../jsonexamples/twitter.json diff --git a/singleheader/amalgamation_demo.cpp b/singleheader/amalgamation_demo.cpp index 2c31db170..afe83364e 100644 --- a/singleheader/amalgamation_demo.cpp +++ b/singleheader/amalgamation_demo.cpp @@ -1,4 +1,4 @@ -/* auto-generated on Thu Jul 11 09:40:22 EDT 2019. Do not edit! */ +/* auto-generated on Sun 28 Jul 2019 18:08:49 EDT. Do not edit! */ #include #include "simdjson.h" diff --git a/singleheader/simdjson.cpp b/singleheader/simdjson.cpp index 25f4a6e44..2f708d4cd 100644 --- a/singleheader/simdjson.cpp +++ b/singleheader/simdjson.cpp @@ -1,4 +1,4 @@ -/* auto-generated on Thu Jul 11 09:40:22 EDT 2019. Do not edit! */ +/* auto-generated on Sun 28 Jul 2019 18:08:49 EDT. Do not edit! */ #include "simdjson.h" /* used for http://dmalloc.com/ Dmalloc - Debug Malloc Library */ @@ -132,6 +132,30 @@ size_t jsonminify(const unsigned char *bytes, size_t howmany, #include namespace simdjson { + + +// some intrinsics are missing under GCC? +#ifndef __clang__ +#ifndef _MSC_VER +static __m256i inline _mm256_loadu2_m128i(__m128i const *__addr_hi, + __m128i const *__addr_lo) { + __m256i __v256 = _mm256_castsi128_si256(_mm_loadu_si128(__addr_lo)); + return _mm256_insertf128_si256(__v256, _mm_loadu_si128(__addr_hi), 1); +} + +static inline void _mm256_storeu2_m128i(__m128i *__addr_hi, __m128i *__addr_lo, + __m256i __a) { + __m128i __v128; + __v128 = _mm256_castsi256_si128(__a); + _mm_storeu_si128(__addr_lo, __v128); + __v128 = _mm256_extractf128_si256(__a, 1); + _mm_storeu_si128(__addr_hi, __v128); +} +#endif +#endif + + + // a straightforward comparison of a mask against input. static uint64_t cmp_mask_against_input_mini(__m256i input_lo, __m256i input_hi, __m256i mask) { @@ -329,50 +353,41 @@ size_t jsonminify(const uint8_t *buf, size_t len, uint8_t *out) { #endif namespace simdjson { + +architecture find_best_supported_implementation() { + constexpr uint32_t haswell_flags = SIMDExtensions::AVX2 | SIMDExtensions::PCLMULQDQ + | SIMDExtensions::BMI1 | SIMDExtensions::BMI2; + constexpr uint32_t westmere_flags = SIMDExtensions::SSE42 | SIMDExtensions::PCLMULQDQ; + + uint32_t supports = detect_supported_architectures(); + // Order from best to worst (within architecture) + if ((haswell_flags & supports) == haswell_flags) return architecture::haswell; + if ((westmere_flags & supports) == westmere_flags) return architecture::westmere; + if (SIMDExtensions::NEON & supports) return architecture::arm64; + + return architecture::none; +} + // Responsible to select the best json_parse implementation int json_parse_dispatch(const uint8_t *buf, size_t len, ParsedJson &pj, bool reallocifneeded) { - // Versions for each implementation -#ifdef __AVX2__ - json_parse_functype* avx_implementation = &json_parse_implementation; -#endif -#if defined(__SSE4_2__) || (defined(_MSC_VER) && defined(_M_AMD64)) - json_parse_functype* sse4_2_implementation = &json_parse_implementation; -#endif -#if defined(__ARM_NEON) || (defined(_MSC_VER) && defined(_M_ARM64)) - json_parse_functype* neon_implementation = &json_parse_implementation; -#endif - - // Determining which implementation is the more suitable - // Should be done at runtime. Does not make any sense on preprocessor. -#ifdef __AVX2__ - instruction_set best_implementation = instruction_set::avx2; -#elif defined (__SSE4_2__) || (defined(_MSC_VER) && defined(_M_AMD64)) - instruction_set best_implementation = instruction_set::sse4_2; -#elif defined (__ARM_NEON) || (defined(_MSC_VER) && defined(_M_ARM64)) - instruction_set best_implementation = instruction_set::neon; -#else - instruction_set best_implementation = instruction_set::none; -#endif - + architecture best_implementation = find_best_supported_implementation(); // Selecting the best implementation switch (best_implementation) { -#ifdef __AVX2__ - case instruction_set::avx2 : - json_parse_ptr = avx_implementation; +#ifdef IS_X86_64 + case architecture::haswell: + json_parse_ptr = &json_parse_implementation; + break; + case architecture::westmere: + json_parse_ptr = &json_parse_implementation; break; #endif -#if defined(__SSE4_2__) || (defined(_MSC_VER) && defined(_M_AMD64)) - case instruction_set::sse4_2 : - json_parse_ptr = sse4_2_implementation; - break; -#endif -#if defined(__ARM_NEON) || (defined(_MSC_VER) && defined(_M_ARM64)) - case instruction_set::neon : - json_parse_ptr = neon_implementation; +#ifdef IS_ARM64 + case architecture::arm64: + json_parse_ptr = &json_parse_implementation; break; #endif default : - std::cerr << "No implemented simd instruction set supported" << std::endl; + std::cerr << "The processor is not supported by simdjson." << std::endl; return simdjson::UNEXPECTED_ERROR; } @@ -395,10 +410,578 @@ ParsedJson build_parsed_json(const uint8_t *buf, size_t len, bool reallocifneede } /* end file src/jsonparser.cpp */ /* begin file src/stage1_find_marks.cpp */ -// File kept in case we want to reuse it soon. (many configuration files to edit) + + +#ifdef IS_X86_64 + +TARGET_HASWELL +namespace simdjson { +template<> +int find_structural_bits(const uint8_t *buf, size_t len, ParsedJson &pj) { + FIND_STRUCTURAL_BITS(architecture::haswell, buf, len, pj); +} + +} +UNTARGET_REGION + +TARGET_WESTMERE +namespace simdjson { +template<> +int find_structural_bits(const uint8_t *buf, size_t len, ParsedJson &pj) { + FIND_STRUCTURAL_BITS(architecture::westmere, buf, len, pj); +} + +} +UNTARGET_REGION + +#endif + + +#ifdef IS_ARM64 +namespace simdjson { +template<> +inline int find_structural_bits(const uint8_t *buf, size_t len, ParsedJson &pj) { + FIND_STRUCTURAL_BITS(architecture::arm64, buf, len, pj); +} + + +} +#endif /* end file src/stage1_find_marks.cpp */ /* begin file src/stage2_build_tape.cpp */ -// File kept in case we want to reuse it soon. (many configuration files to edit) + +namespace simdjson { + +// this macro reads the next structural character, updating idx, i and c. +#define UPDATE_CHAR() \ +{ \ + idx = pj.structural_indexes[i++]; \ + c = buf[idx]; \ +} + +#ifdef SIMDJSON_USE_COMPUTED_GOTO +#define SET_GOTO_ARRAY_CONTINUE() pj.ret_address[depth] = &&array_continue; +#define SET_GOTO_OBJECT_CONTINUE() pj.ret_address[depth] = &&object_continue; +#define SET_GOTO_START_CONTINUE() pj.ret_address[depth] = &&start_continue; +#define GOTO_CONTINUE() goto *pj.ret_address[depth]; +#else +#define SET_GOTO_ARRAY_CONTINUE() pj.ret_address[depth] = 'a'; +#define SET_GOTO_OBJECT_CONTINUE() pj.ret_address[depth] = 'o'; +#define SET_GOTO_START_CONTINUE() pj.ret_address[depth] = 's'; +#define GOTO_CONTINUE() { \ + if(pj.ret_address[depth] == 'a') { \ + goto array_continue; \ + } else if (pj.ret_address[depth] == 'o') { \ + goto object_continue; \ + } else goto { \ + start_continue; \ + } \ +} +#endif + +/************ + * The JSON is parsed to a tape, see the accompanying tape.md file + * for documentation. + ***********/ +// We need to compile that code for multiple architectures. However, target attributes can be used +// only once by function definition. Huge macro seemed better than huge code duplication. +// int UNIFIED_MACHINE(const uint8_t *buf, size_t len, ParsedJson &pj) +#define UNIFIED_MACHINE(T, buf, len, pj) { \ + if (ALLOW_SAME_PAGE_BUFFER_OVERRUN) { \ + memset((uint8_t*)buf + len, 0, SIMDJSON_PADDING); /* to please valgrind */ \ + } \ + uint32_t i = 0; /* index of the structural character (0,1,2,3...) */ \ + uint32_t idx; /* location of the structural character in the input (buf) */ \ + uint8_t c; /* used to track the (structural) character we are looking at, updated */ \ + /* by UPDATE_CHAR macro */ \ + uint32_t depth = 0; /* could have an arbitrary starting depth */ \ + pj.init(); /* sets isvalid to false */ \ + if(pj.bytecapacity < len) { \ + pj.errorcode = simdjson::CAPACITY; \ + return pj.errorcode; \ + } \ + \ + /*//////////////////////////// START STATE ///////////////////////////// */ \ + SET_GOTO_START_CONTINUE() \ + pj.containing_scope_offset[depth] = pj.get_current_loc(); \ + pj.write_tape(0, 'r'); /* r for root, 0 is going to get overwritten */ \ + /* the root is used, if nothing else, to capture the size of the tape */ \ + depth++; /* everything starts at depth = 1, depth = 0 is just for the root, the root may contain an object, an array or something else. */ \ + if (depth >= pj.depthcapacity) { \ + goto fail; \ + } \ + \ + UPDATE_CHAR(); \ + switch (c) { \ + case '{': \ + pj.containing_scope_offset[depth] = pj.get_current_loc(); \ + SET_GOTO_START_CONTINUE(); \ + depth++; \ + if (depth >= pj.depthcapacity) { \ + goto fail; \ + } \ + pj.write_tape(0, c); /* strangely, moving this to object_begin slows things down */ \ + goto object_begin; \ + case '[': \ + pj.containing_scope_offset[depth] = pj.get_current_loc(); \ + SET_GOTO_START_CONTINUE(); \ + depth++; \ + if (depth >= pj.depthcapacity) { \ + goto fail; \ + } \ + pj.write_tape(0, c); \ + goto array_begin; \ +/* #define SIMDJSON_ALLOWANYTHINGINROOT */ \ + /* A JSON text is a serialized value. Note that certain previous */ \ + /* specifications of JSON constrained a JSON text to be an object or an */ \ + /* array. Implementations that generate only objects or arrays where a */ \ + /* JSON text is called for will be interoperable in the sense that all */ \ + /* implementations will accept these as conforming JSON texts. */ \ + /* https://tools.ietf.org/html/rfc8259 */ \ +/* #ifdef SIMDJSON_ALLOWANYTHINGINROOT */ \ + case '"': { \ + if (!parse_string(buf, len, pj, depth, idx)) { \ + goto fail; \ + } \ + break; \ + } \ + case 't': { \ + /* we need to make a copy to make sure that the string is space terminated. */ \ + /* this only applies to the JSON document made solely of the true value. */ \ + /* this will almost never be called in practice */ \ + char * copy = static_cast(malloc(len + SIMDJSON_PADDING)); \ + if(copy == nullptr) { \ + goto fail; \ + } \ + memcpy(copy, buf, len); \ + copy[len] = ' '; \ + if (!is_valid_true_atom(reinterpret_cast(copy) + idx)) { \ + free(copy); \ + goto fail; \ + } \ + free(copy); \ + pj.write_tape(0, c); \ + break; \ + } \ + case 'f': { \ + /* we need to make a copy to make sure that the string is space terminated. */ \ + /* this only applies to the JSON document made solely of the false value. */ \ + /* this will almost never be called in practice */ \ + char * copy = static_cast(malloc(len + SIMDJSON_PADDING)); \ + if(copy == nullptr) { \ + goto fail; \ + } \ + memcpy(copy, buf, len); \ + copy[len] = ' '; \ + if (!is_valid_false_atom(reinterpret_cast(copy) + idx)) { \ + free(copy); \ + goto fail; \ + } \ + free(copy); \ + pj.write_tape(0, c); \ + break; \ + } \ + case 'n': { \ + /* we need to make a copy to make sure that the string is space terminated. */ \ + /* this only applies to the JSON document made solely of the null value. */ \ + /* this will almost never be called in practice */ \ + char * copy = static_cast(malloc(len + SIMDJSON_PADDING)); \ + if(copy == nullptr) { \ + goto fail; \ + } \ + memcpy(copy, buf, len); \ + copy[len] = ' '; \ + if (!is_valid_null_atom(reinterpret_cast(copy) + idx)) { \ + free(copy); \ + goto fail; \ + } \ + free(copy); \ + pj.write_tape(0, c); \ + break; \ + } \ + case '0': \ + case '1': \ + case '2': \ + case '3': \ + case '4': \ + case '5': \ + case '6': \ + case '7': \ + case '8': \ + case '9': { \ + /* we need to make a copy to make sure that the string is space terminated. */ \ + /* this is done only for JSON documents made of a sole number */ \ + /* this will almost never be called in practice. We terminate with a space */ \ + /* because we do not want to allow NULLs in the middle of a number (whereas a */ \ + /* space in the middle of a number would be identified in stage 1). */ \ + char * copy = static_cast(malloc(len + SIMDJSON_PADDING)); \ + if(copy == nullptr) { \ + goto fail; \ + } \ + memcpy(copy, buf, len); \ + copy[len] = ' '; \ + if (!parse_number(reinterpret_cast(copy), pj, idx, false)) { \ + free(copy); \ + goto fail; \ + } \ + free(copy); \ + break; \ + } \ + case '-': { \ + /* we need to make a copy to make sure that the string is NULL terminated. */ \ + /* this is done only for JSON documents made of a sole number */ \ + /* this will almost never be called in practice */ \ + char * copy = static_cast(malloc(len + SIMDJSON_PADDING)); \ + if(copy == nullptr) { \ + goto fail; \ + } \ + memcpy(copy, buf, len); \ + copy[len] = ' '; \ + if (!parse_number(reinterpret_cast(copy), pj, idx, true)) { \ + free(copy); \ + goto fail; \ + } \ + free(copy); \ + break; \ + } \ +/* #endif // ALLOWANYTHINGINROOT */ \ + default: \ + goto fail; \ + } \ +start_continue: \ + /* the string might not be NULL terminated. */ \ + if(i + 1 == pj.n_structural_indexes) { \ + goto succeed; \ + } else { \ + goto fail; \ + } \ + /*//////////////////////////// OBJECT STATES ///////////////////////////// */ \ + \ +object_begin: \ + UPDATE_CHAR(); \ + switch (c) { \ + case '"': { \ + if (!parse_string(buf, len, pj, depth, idx)) { \ + goto fail; \ + } \ + goto object_key_state; \ + } \ + case '}': \ + goto scope_end; /* could also go to object_continue */ \ + default: \ + goto fail; \ + } \ + \ +object_key_state: \ + UPDATE_CHAR(); \ + if (c != ':') { \ + goto fail; \ + } \ + UPDATE_CHAR(); \ + switch (c) { \ + case '"': { \ + if (!parse_string(buf, len, pj, depth, idx)) { \ + goto fail; \ + } \ + break; \ + } \ + case 't': \ + if (!is_valid_true_atom(buf + idx)) { \ + goto fail; \ + } \ + pj.write_tape(0, c); \ + break; \ + case 'f': \ + if (!is_valid_false_atom(buf + idx)) { \ + goto fail; \ + } \ + pj.write_tape(0, c); \ + break; \ + case 'n': \ + if (!is_valid_null_atom(buf + idx)) { \ + goto fail; \ + } \ + pj.write_tape(0, c); \ + break; \ + case '0': \ + case '1': \ + case '2': \ + case '3': \ + case '4': \ + case '5': \ + case '6': \ + case '7': \ + case '8': \ + case '9': { \ + if (!parse_number(buf, pj, idx, false)) { \ + goto fail; \ + } \ + break; \ + } \ + case '-': { \ + if (!parse_number(buf, pj, idx, true)) { \ + goto fail; \ + } \ + break; \ + } \ + case '{': { \ + pj.containing_scope_offset[depth] = pj.get_current_loc(); \ + pj.write_tape(0, c); /* here the compilers knows what c is so this gets optimized */ \ + /* we have not yet encountered } so we need to come back for it */ \ + SET_GOTO_OBJECT_CONTINUE() \ + /* we found an object inside an object, so we need to increment the depth */ \ + depth++; \ + if (depth >= pj.depthcapacity) { \ + goto fail; \ + } \ + \ + goto object_begin; \ + } \ + case '[': { \ + pj.containing_scope_offset[depth] = pj.get_current_loc(); \ + pj.write_tape(0, c); /* here the compilers knows what c is so this gets optimized */ \ + /* we have not yet encountered } so we need to come back for it */ \ + SET_GOTO_OBJECT_CONTINUE() \ + /* we found an array inside an object, so we need to increment the depth */ \ + depth++; \ + if (depth >= pj.depthcapacity) { \ + goto fail; \ + } \ + goto array_begin; \ + } \ + default: \ + goto fail; \ + } \ + \ +object_continue: \ + UPDATE_CHAR(); \ + switch (c) { \ + case ',': \ + UPDATE_CHAR(); \ + if (c != '"') { \ + goto fail; \ + } else { \ + if (!parse_string(buf, len, pj, depth, idx)) { \ + goto fail; \ + } \ + goto object_key_state; \ + } \ + case '}': \ + goto scope_end; \ + default: \ + goto fail; \ + } \ + \ + /*//////////////////////////// COMMON STATE ///////////////////////////// */ \ + \ +scope_end: \ + /* write our tape location to the header scope */ \ + depth--; \ + pj.write_tape(pj.containing_scope_offset[depth], c); \ + pj.annotate_previousloc(pj.containing_scope_offset[depth], \ + pj.get_current_loc()); \ + /* goto saved_state */ \ + GOTO_CONTINUE() \ + \ + /*//////////////////////////// ARRAY STATES ///////////////////////////// */ \ +array_begin: \ + UPDATE_CHAR(); \ + if (c == ']') { \ + goto scope_end; /* could also go to array_continue */ \ + } \ + \ +main_array_switch: \ + /* we call update char on all paths in, so we can peek at c on the */ \ + /* on paths that can accept a close square brace (post-, and at start) */ \ + switch (c) { \ + case '"': { \ + if (!parse_string(buf, len, pj, depth, idx)) { \ + goto fail; \ + } \ + break; \ + } \ + case 't': \ + if (!is_valid_true_atom(buf + idx)) { \ + goto fail; \ + } \ + pj.write_tape(0, c); \ + break; \ + case 'f': \ + if (!is_valid_false_atom(buf + idx)) { \ + goto fail; \ + } \ + pj.write_tape(0, c); \ + break; \ + case 'n': \ + if (!is_valid_null_atom(buf + idx)) { \ + goto fail; \ + } \ + pj.write_tape(0, c); \ + break; /* goto array_continue; */ \ + \ + case '0': \ + case '1': \ + case '2': \ + case '3': \ + case '4': \ + case '5': \ + case '6': \ + case '7': \ + case '8': \ + case '9': { \ + if (!parse_number(buf, pj, idx, false)) { \ + goto fail; \ + } \ + break; /* goto array_continue; */ \ + } \ + case '-': { \ + if (!parse_number(buf, pj, idx, true)) { \ + goto fail; \ + } \ + break; /* goto array_continue; */ \ + } \ + case '{': { \ + /* we have not yet encountered ] so we need to come back for it */ \ + pj.containing_scope_offset[depth] = pj.get_current_loc(); \ + pj.write_tape(0, c); /* here the compilers knows what c is so this gets optimized */ \ + SET_GOTO_ARRAY_CONTINUE() \ + /* we found an object inside an array, so we need to increment the depth */ \ + depth++; \ + if (depth >= pj.depthcapacity) { \ + goto fail; \ + } \ + \ + goto object_begin; \ + } \ + case '[': { \ + /* we have not yet encountered ] so we need to come back for it */ \ + pj.containing_scope_offset[depth] = pj.get_current_loc(); \ + pj.write_tape(0, c); /* here the compilers knows what c is so this gets optimized */ \ + SET_GOTO_ARRAY_CONTINUE() \ + /* we found an array inside an array, so we need to increment the depth */ \ + depth++; \ + if (depth >= pj.depthcapacity) { \ + goto fail; \ + } \ + goto array_begin; \ + } \ + default: \ + goto fail; \ + } \ + \ +array_continue: \ + UPDATE_CHAR(); \ + switch (c) { \ + case ',': \ + UPDATE_CHAR(); \ + goto main_array_switch; \ + case ']': \ + goto scope_end; \ + default: \ + goto fail; \ + } \ + \ + /*//////////////////////////// FINAL STATES ///////////////////////////// */ \ + \ +succeed: \ + depth --; \ + if(depth != 0) { \ + fprintf(stderr, "internal bug\n"); \ + abort(); \ + } \ + if(pj.containing_scope_offset[depth] != 0) { \ + fprintf(stderr, "internal bug\n"); \ + abort(); \ + } \ + pj.annotate_previousloc(pj.containing_scope_offset[depth], \ + pj.get_current_loc()); \ + pj.write_tape(pj.containing_scope_offset[depth], 'r'); /* r is root */ \ + \ + pj.isvalid = true; \ + pj.errorcode = simdjson::SUCCESS; \ + return pj.errorcode; \ +fail: \ + /* we do not need the next line because this is done by pj.init(), pessimistically. */ \ + /* pj.isvalid = false; */ \ + /* At this point in the code, we have all the time in the world. */ \ + /* Note that we know exactly where we are in the document so we could, */ \ + /* without any overhead on the processing code, report a specific location. */ \ + /* We could even trigger special code paths to assess what happened carefully, */ \ + /* all without any added cost. */ \ + if (depth >= pj.depthcapacity) { \ + pj.errorcode = simdjson::DEPTH_ERROR; \ + return pj.errorcode; \ + } \ + switch(c) { \ + case '"': \ + pj.errorcode = simdjson::STRING_ERROR; \ + return pj.errorcode; \ + case '0': \ + case '1': \ + case '2': \ + case '3': \ + case '4': \ + case '5': \ + case '6': \ + case '7': \ + case '8': \ + case '9': \ + case '-': \ + pj.errorcode = simdjson::NUMBER_ERROR; \ + return pj.errorcode; \ + case 't': \ + pj.errorcode = simdjson::T_ATOM_ERROR; \ + return pj.errorcode; \ + case 'n': \ + pj.errorcode = simdjson::N_ATOM_ERROR; \ + return pj.errorcode; \ + case 'f': \ + pj.errorcode = simdjson::F_ATOM_ERROR; \ + return pj.errorcode; \ + default: \ + break; \ + } \ + pj.errorcode = simdjson::TAPE_ERROR; \ + return pj.errorcode; \ +} \ + + +} + + +#ifdef IS_X86_64 +TARGET_HASWELL +namespace simdjson { +template<> +WARN_UNUSED ALLOW_SAME_PAGE_BUFFER_OVERRUN_QUALIFIER LENIENT_MEM_SANITIZER +int unified_machine(const uint8_t *buf, size_t len, ParsedJson &pj) { + UNIFIED_MACHINE(architecture::haswell, buf, len, pj); +} +} +UNTARGET_REGION + +TARGET_WESTMERE +namespace simdjson { +template<> +WARN_UNUSED ALLOW_SAME_PAGE_BUFFER_OVERRUN_QUALIFIER LENIENT_MEM_SANITIZER +int unified_machine(const uint8_t *buf, size_t len, ParsedJson &pj) { + UNIFIED_MACHINE(architecture::westmere, buf, len, pj); +} +} +UNTARGET_REGION +#endif // IS_X86_64 + +#ifdef IS_ARM64 +namespace simdjson { +template<> +WARN_UNUSED ALLOW_SAME_PAGE_BUFFER_OVERRUN_QUALIFIER LENIENT_MEM_SANITIZER +int unified_machine(const uint8_t *buf, size_t len, ParsedJson &pj) { + UNIFIED_MACHINE(architecture::arm64, buf, len, pj); +} +} +#endif + + /* end file src/stage2_build_tape.cpp */ /* begin file src/parsedjson.cpp */ diff --git a/singleheader/simdjson.h b/singleheader/simdjson.h index 50f27a9f1..aa5e759bc 100644 --- a/singleheader/simdjson.h +++ b/singleheader/simdjson.h @@ -1,4 +1,4 @@ -/* auto-generated on Thu Jul 11 09:40:22 EDT 2019. Do not edit! */ +/* auto-generated on Sun 28 Jul 2019 18:08:49 EDT. Do not edit! */ /* begin file include/simdjson/simdjson_version.h */ // /include/simdjson/simdjson_version.h automatically generated by release.py, do not change by hand #ifndef SIMDJSON_INCLUDE_SIMDJSON_VERSION @@ -13,62 +13,71 @@ enum { } #endif // SIMDJSON_INCLUDE_SIMDJSON_VERSION /* end file include/simdjson/simdjson_version.h */ -/* begin file include/simdjson/simdjson.h */ -#ifndef SIMDJSON_ERR_H -# define SIMDJSON_ERR_H - -#include - -namespace simdjson { -enum class instruction_set { - avx2, - sse4_2, - neon, - none, -// the 'native' enum class value should point at a good default on the current machine -#ifdef __AVX2__ - native = avx2 -#elif defined(__ARM_NEON) || (defined(_MSC_VER) && defined(_M_ARM64)) - native = neon -#else - // Let us assume that we have an old x64 processor, but one that has SSE (i.e., something - // that came out in the second decade of the XXIst century. - // It would be nicer to check explicitly, but there many not be a good way to do so - // that is cross-platform. - // Under Visual Studio, there is no way to check for SSE4.2 support at compile-time. - native = sse4_2 -#endif -}; - -enum errorValues { - SUCCESS = 0, - CAPACITY, // This ParsedJson can't support a document that big - MEMALLOC, // Error allocating memory, most likely out of memory - TAPE_ERROR, // Something went wrong while writing to the tape (stage 2), this is a generic error - DEPTH_ERROR, // Your document exceeds the user-specified depth limitation - STRING_ERROR, // Problem while parsing a string - T_ATOM_ERROR, // Problem while parsing an atom starting with the letter 't' - F_ATOM_ERROR, // Problem while parsing an atom starting with the letter 'f' - N_ATOM_ERROR, // Problem while parsing an atom starting with the letter 'n' - NUMBER_ERROR, // Problem while parsing a number - UTF8_ERROR, // the input is not valid UTF-8 - UNITIALIZED, // unknown error, or uninitialized document - EMPTY, // no structural document found - UNESCAPED_CHARS, // found unescaped characters in a string. - UNCLOSED_STRING, // missing quote at the end - UNEXPECTED_ERROR // indicative of a bug in simdjson -}; -const std::string& errorMsg(const int); -} -#endif -/* end file include/simdjson/simdjson.h */ /* begin file include/simdjson/portability.h */ #ifndef SIMDJSON_PORTABILITY_H #define SIMDJSON_PORTABILITY_H +#if defined(__x86_64__) || defined(_M_AMD64) +# define IS_X86_64 1 +#endif +#if defined(__aarch64__) || defined(_M_ARM64) +# define IS_ARM64 1 +#endif + + + + +// we are going to use runtime dispatch +#ifdef IS_X86_64 +#if defined(_MSC_VER) +// under visual studio, nothing needs to be done +// under ARM, we don't want these macros +#define TARGET_HASWELL +#define TARGET_WESTMERE +#else +/////// +// under clang OR gcc, we need do do extra work +/////// + +#ifdef __clang__ +// clang does not have GCC push pop +// warning: clang attribute push can't be used within a namespace in clang up til 8.0 so TARGET_REGION and +// UNTARGET_REGION must be *outside* of a namespace. +#define STRINGIFY(a) #a +#define TARGET_REGION(T) _Pragma(STRINGIFY(clang attribute push(__attribute__((target(T))), apply_to=function))) +#define UNTARGET_REGION _Pragma("clang attribute pop") +#undef STINGIFY +#elif defined(__GNUC__) +// GCC is easier +#define TARGET_REGION(T) \ +_Pragma("GCC push_options") \ +_Pragma(STRINGIFY(GCC target(T))) +#define UNTARGET_REGION \ +_Pragma("GCC pop_options") +#endif + +// under GCC and CLANG, we use these two macros +#define TARGET_HASWELL TARGET_REGION("avx2,bmi,pclmul") +#define TARGET_WESTMERE TARGET_REGION("sse4.2,pclmul") + + +#endif // msc_ver +#endif // x86 + + + +#ifdef _MSC_VER +# include +#else +# if IS_X86_64 +# include +# elif IS_ARM64 +# include +# endif +#endif + #ifdef _MSC_VER /* Microsoft C/C++-compatible compiler */ -#include #include #include @@ -105,9 +114,6 @@ static inline int hamming(uint64_t input_num) { #include #include -#if defined(__BMI2__) || defined(__POPCOUNT__) || defined(__AVX2__) || defined(__SSE4_2__) -#include -#endif namespace simdjson { static inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t *result) { return __builtin_uaddll_overflow(value1, value2, (unsigned long long*)result); @@ -166,30 +172,6 @@ static inline char *aligned_malloc_char(size_t alignment, size_t size) { return (char*)aligned_malloc(alignment, size); } -#ifdef __AVX2__ - -#ifndef __clang__ -#ifndef _MSC_VER -static __m256i inline _mm256_loadu2_m128i(__m128i const *__addr_hi, - __m128i const *__addr_lo) { - __m256i __v256 = _mm256_castsi128_si256(_mm_loadu_si128(__addr_lo)); - return _mm256_insertf128_si256(__v256, _mm_loadu_si128(__addr_hi), 1); -} - -static inline void _mm256_storeu2_m128i(__m128i *__addr_hi, __m128i *__addr_lo, - __m256i __a) { - __m128i __v128; - - __v128 = _mm256_castsi256_si128(__a); - _mm_storeu_si128(__addr_lo, __v128); - __v128 = _mm256_extractf128_si256(__a, 1); - _mm_storeu_si128(__addr_hi, __v128); -} -#endif -#endif - -#endif // AVX_2 - static inline void aligned_free(void *memblock) { if(memblock == nullptr) { return; } #ifdef _MSC_VER @@ -207,9 +189,208 @@ static inline void aligned_free_char(char *memblock) { aligned_free((void*)memblock); } } - #endif // SIMDJSON_PORTABILITY_H /* end file include/simdjson/portability.h */ +/* begin file include/simdjson/simddetection.h */ +/* From https://github.com/endorno/pytorch/blob/master/torch/lib/TH/generic/simd/simd.h +Highly modified. + +Copyright (c) 2016- Facebook, Inc (Adam Paszke) +Copyright (c) 2014- Facebook, Inc (Soumith Chintala) +Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert) +Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu) +Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu) +Copyright (c) 2011-2013 NYU (Clement Farabet) +Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, Iain Melvin, Jason Weston) +Copyright (c) 2006 Idiap Research Institute (Samy Bengio) +Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, Samy Bengio, Johnny Mariethoz) + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories America + and IDIAP Research Institute nor the names of its contributors may be + used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef ISADETECTION_H +#define ISADETECTION_H + +#include +#include +#if defined(_MSC_VER) +#include +#elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID) +#include +#endif + +namespace simdjson { +// Can be found on Intel ISA Reference for CPUID +constexpr uint32_t cpuid_avx2_bit = 1 << 5; // Bit 5 of EBX for EAX=0x7 +constexpr uint32_t cpuid_bmi1_bit = 1 << 3; // bit 3 of EBX for EAX=0x7 +constexpr uint32_t cpuid_bmi2_bit = 1 << 8; // bit 8 of EBX for EAX=0x7 +constexpr uint32_t cpuid_sse42_bit = 1 << 20; // bit 20 of ECX for EAX=0x1 +constexpr uint32_t cpuid_pclmulqdq_bit = 1 << 1; // bit 1 of ECX for EAX=0x1 + +enum SIMDExtensions { + DEFAULT = 0x0, + NEON = 0x1, + AVX2 = 0x4, + SSE42 = 0x8, + PCLMULQDQ = 0x10, + BMI1 = 0x20, + BMI2 = 0x40 +}; + +#if defined(__arm__) || defined(__aarch64__) // incl. armel, armhf, arm64 + + #if defined(__NEON__) + +static inline uint32_t detect_supported_architectures() +{ + return SIMDExtensions::NEON; +} + + #else //ARM without NEON + +static inline uint32_t detect_supported_architectures() +{ + return SIMDExtensions::DEFAULT; +} + + #endif + +#else // x86 +static inline void cpuid(uint32_t *eax, uint32_t *ebx, uint32_t *ecx, uint32_t *edx) +{ +#if defined(_MSC_VER) + uint32_t cpuInfo[4]; + __cpuid(cpuInfo, *eax); + *eax = cpuInfo[0]; + *ebx = cpuInfo[1]; + *ecx = cpuInfo[2]; + *edx = cpuInfo[3]; +#elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID) + uint32_t level = *eax; + __get_cpuid (level, eax, ebx, ecx, edx); +#else + uint32_t a = *eax, b, c = *ecx, d; + asm volatile ( "cpuid\n\t" + : "+a"(a), "=b"(b), "+c"(c), "=d"(d) ); + *eax = a; + *ebx = b; + *ecx = c; + *edx = d; +#endif +} + +static inline uint32_t detect_supported_architectures() +{ + uint32_t eax, ebx, ecx, edx; + uint32_t hostSimdExts = 0x0; + + // ECX for EAX=0x1 + eax = 0x7; + ecx = 0x0; + cpuid(&eax, &ebx, &ecx, &edx); + + if (ebx & cpuid_avx2_bit) { + hostSimdExts |= SIMDExtensions::AVX2; + } + + if (ebx & cpuid_bmi1_bit) { + hostSimdExts |= SIMDExtensions::BMI1; + } + + if (ebx & cpuid_bmi2_bit) { + hostSimdExts |= SIMDExtensions::BMI2; + } + + // EBX for EAX=0x7 + eax = 0x1; + cpuid(&eax, &ebx, &ecx, &edx); + + if (ecx & cpuid_sse42_bit) { + hostSimdExts |= SIMDExtensions::SSE42; + } + + if (ecx & cpuid_pclmulqdq_bit) { + hostSimdExts |= SIMDExtensions::PCLMULQDQ; + } + + return hostSimdExts; +} + +#endif // end SIMD extension detection code +} +#endif +/* end file include/simdjson/simddetection.h */ +/* begin file include/simdjson/simdjson.h */ +#ifndef SIMDJSON_ERR_H +#define SIMDJSON_ERR_H + +#include + +namespace simdjson { +// Represents the minimal architecture that would support an implementation +enum class architecture { + westmere, + haswell, + arm64, + none, + // TODO remove 'native' in favor of runtime dispatch? + // the 'native' enum class value should point at a good default on the current machine +#ifdef IS_X86_64 + native = westmere +#elif defined(IS_ARM64) + native = arm64 +#endif +}; + +enum errorValues { + SUCCESS = 0, + CAPACITY, // This ParsedJson can't support a document that big + MEMALLOC, // Error allocating memory, most likely out of memory + TAPE_ERROR, // Something went wrong while writing to the tape (stage 2), this is a generic error + DEPTH_ERROR, // Your document exceeds the user-specified depth limitation + STRING_ERROR, // Problem while parsing a string + T_ATOM_ERROR, // Problem while parsing an atom starting with the letter 't' + F_ATOM_ERROR, // Problem while parsing an atom starting with the letter 'f' + N_ATOM_ERROR, // Problem while parsing an atom starting with the letter 'n' + NUMBER_ERROR, // Problem while parsing a number + UTF8_ERROR, // the input is not valid UTF-8 + UNITIALIZED, // unknown error, or uninitialized document + EMPTY, // no structural document found + UNESCAPED_CHARS, // found unescaped characters in a string. + UNCLOSED_STRING, // missing quote at the end + UNEXPECTED_ERROR // indicative of a bug in simdjson +}; +const std::string& errorMsg(const int); +} +#endif +/* end file include/simdjson/simdjson.h */ /* begin file include/simdjson/common_defs.h */ #ifndef SIMDJSON_COMMON_DEFS_H #define SIMDJSON_COMMON_DEFS_H @@ -261,7 +442,9 @@ static inline void aligned_free_char(char *memblock) { // For non-Visual Studio compilers, we may assume that same-page buffer overrun is fine. // However, it will make it difficult to be "valgrind clean". //#ifndef ALLOW_SAME_PAGE_BUFFER_OVERRUN -//#define ALLOW_SAME_PAGE_BUFFER_OVERRUN +//#define ALLOW_SAME_PAGE_BUFFER_OVERRUN true +//#else +#define ALLOW_SAME_PAGE_BUFFER_OVERRUN false //#endif // The following is likely unnecessarily complex. @@ -920,8 +1103,6 @@ padded_string get_corpus(const std::string& filename); #include - -#ifdef __AVX__ namespace simdjson { static const unsigned char mask128_epi8[] = { 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe, @@ -35903,9 +36084,7 @@ static const unsigned char mask128_epi32[] = { 0xff, 0xff, 0xff, 0xff, }; } // namespace simdjson -#endif //__SSE3__ -#ifdef __AVX2__ namespace simdjson { static const uint32_t mask256_epi32[] = { 0, 1, 2, 3, 4, 5, 6, 7, 1, 2, 3, 4, 5, 6, 7, 7, 0, 2, 3, 4, 5, 6, 7, 7, 2, @@ -35991,20 +36170,18 @@ static const uint32_t mask256_epi32[] = { 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; } -#endif //__AVX2__ - #endif /* end file include/simdjson/simdprune_tables.h */ -/* begin file include/simdjson/simdutf8check.h */ +/* begin file include/simdjson/simdutf8check_haswell.h */ +#ifndef SIMDJSON_SIMDUTF8CHECK_HASWELL_H +#define SIMDJSON_SIMDUTF8CHECK_HASWELL_H -#ifndef SIMDJSON_SIMDUTF8CHECK_H -#define SIMDJSON_SIMDUTF8CHECK_H - -#if defined (__AVX2__) || defined (__SSE4_2__) || (defined(_MSC_VER) && defined(_M_AMD64)) #include #include #include + +#ifdef IS_X86_64 /* * legal utf-8 byte sequence * http://www.unicode.org/versions/Unicode6.0.0/ch03.pdf - page 94 @@ -36024,141 +36201,8 @@ static const uint32_t mask256_epi32[] = { // all byte values must be no larger than 0xF4 +TARGET_HASWELL namespace simdjson { -// all byte values must be no larger than 0xF4 -static inline void checkSmallerThan0xF4(__m128i current_bytes, - __m128i *has_error) { - // unsigned, saturates to 0 below max - *has_error = _mm_or_si128(*has_error, - _mm_subs_epu8(current_bytes, _mm_set1_epi8(0xF4))); -} - -static inline __m128i continuationLengths(__m128i high_nibbles) { - return _mm_shuffle_epi8( - _mm_setr_epi8(1, 1, 1, 1, 1, 1, 1, 1, // 0xxx (ASCII) - 0, 0, 0, 0, // 10xx (continuation) - 2, 2, // 110x - 3, // 1110 - 4), // 1111, next should be 0 (not checked here) - high_nibbles); -} - -static inline __m128i carryContinuations(__m128i initial_lengths, - __m128i previous_carries) { - - __m128i right1 = - _mm_subs_epu8(_mm_alignr_epi8(initial_lengths, previous_carries, 16 - 1), - _mm_set1_epi8(1)); - __m128i sum = _mm_add_epi8(initial_lengths, right1); - - __m128i right2 = _mm_subs_epu8(_mm_alignr_epi8(sum, previous_carries, 16 - 2), - _mm_set1_epi8(2)); - return _mm_add_epi8(sum, right2); -} - -static inline void checkContinuations(__m128i initial_lengths, __m128i carries, - __m128i *has_error) { - - // overlap || underlap - // carry > length && length > 0 || !(carry > length) && !(length > 0) - // (carries > length) == (lengths > 0) - __m128i overunder = - _mm_cmpeq_epi8(_mm_cmpgt_epi8(carries, initial_lengths), - _mm_cmpgt_epi8(initial_lengths, _mm_setzero_si128())); - - *has_error = _mm_or_si128(*has_error, overunder); -} - -// when 0xED is found, next byte must be no larger than 0x9F -// when 0xF4 is found, next byte must be no larger than 0x8F -// next byte must be continuation, ie sign bit is set, so signed < is ok -static inline void checkFirstContinuationMax(__m128i current_bytes, - __m128i off1_current_bytes, - __m128i *has_error) { - __m128i maskED = _mm_cmpeq_epi8(off1_current_bytes, _mm_set1_epi8(0xED)); - __m128i maskF4 = _mm_cmpeq_epi8(off1_current_bytes, _mm_set1_epi8(0xF4)); - - __m128i badfollowED = - _mm_and_si128(_mm_cmpgt_epi8(current_bytes, _mm_set1_epi8(0x9F)), maskED); - __m128i badfollowF4 = - _mm_and_si128(_mm_cmpgt_epi8(current_bytes, _mm_set1_epi8(0x8F)), maskF4); - - *has_error = _mm_or_si128(*has_error, _mm_or_si128(badfollowED, badfollowF4)); -} - -// map off1_hibits => error condition -// hibits off1 cur -// C => < C2 && true -// E => < E1 && < A0 -// F => < F1 && < 90 -// else false && false -static inline void checkOverlong(__m128i current_bytes, - __m128i off1_current_bytes, __m128i hibits, - __m128i previous_hibits, __m128i *has_error) { - __m128i off1_hibits = _mm_alignr_epi8(hibits, previous_hibits, 16 - 1); - __m128i initial_mins = _mm_shuffle_epi8( - _mm_setr_epi8(-128, -128, -128, -128, -128, -128, -128, -128, -128, -128, - -128, -128, // 10xx => false - 0xC2, -128, // 110x - 0xE1, // 1110 - 0xF1), - off1_hibits); - - __m128i initial_under = _mm_cmpgt_epi8(initial_mins, off1_current_bytes); - - __m128i second_mins = _mm_shuffle_epi8( - _mm_setr_epi8(-128, -128, -128, -128, -128, -128, -128, -128, -128, -128, - -128, -128, // 10xx => false - 127, 127, // 110x => true - 0xA0, // 1110 - 0x90), - off1_hibits); - __m128i second_under = _mm_cmpgt_epi8(second_mins, current_bytes); - *has_error = - _mm_or_si128(*has_error, _mm_and_si128(initial_under, second_under)); -} - -struct processed_utf_bytes { - __m128i rawbytes; - __m128i high_nibbles; - __m128i carried_continuations; -}; - -static inline void count_nibbles(__m128i bytes, - struct processed_utf_bytes *answer) { - answer->rawbytes = bytes; - answer->high_nibbles = - _mm_and_si128(_mm_srli_epi16(bytes, 4), _mm_set1_epi8(0x0F)); -} - -// check whether the current bytes are valid UTF-8 -// at the end of the function, previous gets updated -static struct processed_utf_bytes -checkUTF8Bytes(__m128i current_bytes, struct processed_utf_bytes *previous, - __m128i *has_error) { - struct processed_utf_bytes pb; - count_nibbles(current_bytes, &pb); - - checkSmallerThan0xF4(current_bytes, has_error); - - __m128i initial_lengths = continuationLengths(pb.high_nibbles); - - pb.carried_continuations = - carryContinuations(initial_lengths, previous->carried_continuations); - - checkContinuations(initial_lengths, pb.carried_continuations, has_error); - - __m128i off1_current_bytes = - _mm_alignr_epi8(pb.rawbytes, previous->rawbytes, 16 - 1); - checkFirstContinuationMax(current_bytes, off1_current_bytes, has_error); - - checkOverlong(current_bytes, off1_current_bytes, pb.high_nibbles, - previous->high_nibbles, has_error); - return pb; -} - -#ifdef __AVX2__ -/*****************************/ static inline __m256i push_last_byte_of_a_to_b(__m256i a, __m256i b) { return _mm256_alignr_epi8(b, _mm256_permute2x128_si256(a, b, 0x21), 15); } @@ -36319,20 +36363,191 @@ avxcheckUTF8Bytes(__m256i current_bytes, previous->high_nibbles, has_error); return pb; } +}// simdjson +UNTARGET_REGION // haswell -#endif // __AVX2__ -} -#endif // defined (__AVX2__) || defined (__SSE4_2__) || (defined(_MSC_VER) && defined(_M_AMD64)) +#endif // IS_X86_64 #endif -/* end file include/simdjson/simdutf8check.h */ -/* begin file include/simdjson/simdutf8check_neon.h */ +/* end file include/simdjson/simdutf8check_haswell.h */ +/* begin file include/simdjson/simdutf8check_westmere.h */ +#ifndef SIMDJSON_SIMDUTF8CHECK_WESTMERE_H +#define SIMDJSON_SIMDUTF8CHECK_WESTMERE_H + +#include +#include +#include +#ifdef IS_X86_64 + +/* + * legal utf-8 byte sequence + * http://www.unicode.org/versions/Unicode6.0.0/ch03.pdf - page 94 + * + * Code Points 1st 2s 3s 4s + * U+0000..U+007F 00..7F + * U+0080..U+07FF C2..DF 80..BF + * U+0800..U+0FFF E0 A0..BF 80..BF + * U+1000..U+CFFF E1..EC 80..BF 80..BF + * U+D000..U+D7FF ED 80..9F 80..BF + * U+E000..U+FFFF EE..EF 80..BF 80..BF + * U+10000..U+3FFFF F0 90..BF 80..BF 80..BF + * U+40000..U+FFFFF F1..F3 80..BF 80..BF 80..BF + * U+100000..U+10FFFF F4 80..8F 80..BF 80..BF + * + */ + +// all byte values must be no larger than 0xF4 + +/********** sse code **********/ +TARGET_WESTMERE + +namespace simdjson{ +// all byte values must be no larger than 0xF4 +static inline void checkSmallerThan0xF4(__m128i current_bytes, + __m128i *has_error) { + // unsigned, saturates to 0 below max + *has_error = _mm_or_si128(*has_error, + _mm_subs_epu8(current_bytes, _mm_set1_epi8(0xF4))); +} + +static inline __m128i continuationLengths(__m128i high_nibbles) { + return _mm_shuffle_epi8( + _mm_setr_epi8(1, 1, 1, 1, 1, 1, 1, 1, // 0xxx (ASCII) + 0, 0, 0, 0, // 10xx (continuation) + 2, 2, // 110x + 3, // 1110 + 4), // 1111, next should be 0 (not checked here) + high_nibbles); +} + +static inline __m128i carryContinuations(__m128i initial_lengths, + __m128i previous_carries) { + + __m128i right1 = + _mm_subs_epu8(_mm_alignr_epi8(initial_lengths, previous_carries, 16 - 1), + _mm_set1_epi8(1)); + __m128i sum = _mm_add_epi8(initial_lengths, right1); + + __m128i right2 = _mm_subs_epu8(_mm_alignr_epi8(sum, previous_carries, 16 - 2), + _mm_set1_epi8(2)); + return _mm_add_epi8(sum, right2); +} + +static inline void checkContinuations(__m128i initial_lengths, __m128i carries, + __m128i *has_error) { + + // overlap || underlap + // carry > length && length > 0 || !(carry > length) && !(length > 0) + // (carries > length) == (lengths > 0) + __m128i overunder = + _mm_cmpeq_epi8(_mm_cmpgt_epi8(carries, initial_lengths), + _mm_cmpgt_epi8(initial_lengths, _mm_setzero_si128())); + + *has_error = _mm_or_si128(*has_error, overunder); +} + +// when 0xED is found, next byte must be no larger than 0x9F +// when 0xF4 is found, next byte must be no larger than 0x8F +// next byte must be continuation, ie sign bit is set, so signed < is ok +static inline void checkFirstContinuationMax(__m128i current_bytes, + __m128i off1_current_bytes, + __m128i *has_error) { + __m128i maskED = _mm_cmpeq_epi8(off1_current_bytes, _mm_set1_epi8(0xED)); + __m128i maskF4 = _mm_cmpeq_epi8(off1_current_bytes, _mm_set1_epi8(0xF4)); + + __m128i badfollowED = + _mm_and_si128(_mm_cmpgt_epi8(current_bytes, _mm_set1_epi8(0x9F)), maskED); + __m128i badfollowF4 = + _mm_and_si128(_mm_cmpgt_epi8(current_bytes, _mm_set1_epi8(0x8F)), maskF4); + + *has_error = _mm_or_si128(*has_error, _mm_or_si128(badfollowED, badfollowF4)); +} + +// map off1_hibits => error condition +// hibits off1 cur +// C => < C2 && true +// E => < E1 && < A0 +// F => < F1 && < 90 +// else false && false +static inline void checkOverlong(__m128i current_bytes, + __m128i off1_current_bytes, __m128i hibits, + __m128i previous_hibits, __m128i *has_error) { + __m128i off1_hibits = _mm_alignr_epi8(hibits, previous_hibits, 16 - 1); + __m128i initial_mins = _mm_shuffle_epi8( + _mm_setr_epi8(-128, -128, -128, -128, -128, -128, -128, -128, -128, -128, + -128, -128, // 10xx => false + 0xC2, -128, // 110x + 0xE1, // 1110 + 0xF1), + off1_hibits); + + __m128i initial_under = _mm_cmpgt_epi8(initial_mins, off1_current_bytes); + + __m128i second_mins = _mm_shuffle_epi8( + _mm_setr_epi8(-128, -128, -128, -128, -128, -128, -128, -128, -128, -128, + -128, -128, // 10xx => false + 127, 127, // 110x => true + 0xA0, // 1110 + 0x90), + off1_hibits); + __m128i second_under = _mm_cmpgt_epi8(second_mins, current_bytes); + *has_error = + _mm_or_si128(*has_error, _mm_and_si128(initial_under, second_under)); +} + +struct processed_utf_bytes { + __m128i rawbytes; + __m128i high_nibbles; + __m128i carried_continuations; +}; + +static inline void count_nibbles(__m128i bytes, + struct processed_utf_bytes *answer) { + answer->rawbytes = bytes; + answer->high_nibbles = + _mm_and_si128(_mm_srli_epi16(bytes, 4), _mm_set1_epi8(0x0F)); +} + +// check whether the current bytes are valid UTF-8 +// at the end of the function, previous gets updated +static struct processed_utf_bytes +checkUTF8Bytes(__m128i current_bytes, struct processed_utf_bytes *previous, + __m128i *has_error) { + struct processed_utf_bytes pb; + count_nibbles(current_bytes, &pb); + + checkSmallerThan0xF4(current_bytes, has_error); + + __m128i initial_lengths = continuationLengths(pb.high_nibbles); + + pb.carried_continuations = + carryContinuations(initial_lengths, previous->carried_continuations); + + checkContinuations(initial_lengths, pb.carried_continuations, has_error); + + __m128i off1_current_bytes = + _mm_alignr_epi8(pb.rawbytes, previous->rawbytes, 16 - 1); + checkFirstContinuationMax(current_bytes, off1_current_bytes, has_error); + + checkOverlong(current_bytes, off1_current_bytes, pb.high_nibbles, + previous->high_nibbles, has_error); + return pb; +} +}//simdjson +UNTARGET_REGION // westmere + + +#endif // IS_X86_64 + +#endif +/* end file include/simdjson/simdutf8check_westmere.h */ +/* begin file include/simdjson/simdutf8check_arm64.h */ // From https://github.com/cyb70289/utf8/blob/master/lemire-neon.c // Adapted from https://github.com/lemire/fastvalidate-utf-8 -#ifndef SIMDJSON_SIMDUTF8CHECK_NEON_H -#define SIMDJSON_SIMDUTF8CHECK_NEON_H +#ifndef SIMDJSON_SIMDUTF8CHECK_ARM64_H +#define SIMDJSON_SIMDUTF8CHECK_ARM64_H #if defined(_ARM_NEON) || defined(__aarch64__) || (defined(_MSC_VER) && defined(_M_ARM64)) @@ -36359,6 +36574,7 @@ avxcheckUTF8Bytes(__m256i current_bytes, * U+100000..U+10FFFF F4 80..8F 80..BF 80..BF * */ +namespace simdjson { // all byte values must be no larger than 0xF4 static inline void checkSmallerThan0xF4(int8x16_t current_bytes, @@ -36497,10 +36713,10 @@ checkUTF8Bytes(int8x16_t current_bytes, struct processed_utf_bytes *previous, previous->high_nibbles, has_error); return pb; } - +}// simdjson #endif #endif -/* end file include/simdjson/simdutf8check_neon.h */ +/* end file include/simdjson/simdutf8check_arm64.h */ /* begin file include/simdjson/jsonminifier.h */ #ifndef SIMDJSON_JSONMINIFIER_H #define SIMDJSON_JSONMINIFIER_H @@ -37040,114 +37256,12 @@ bool ParsedJson::iterator::next() { #include -#if defined (__AVX2__) -#elif defined (__SSE4_2__) || (defined(_MSC_VER) && defined(_M_AMD64)) -#elif defined(__ARM_NEON) || (defined(_MSC_VER) && defined(_M_ARM64)) -#include -#else -#warning It appears that neither ARM NEON nor AVX2 nor SSE are detected. -#endif // (__AVX2__) - -#ifndef SIMDJSON_SKIPUTF8VALIDATION -#define SIMDJSON_UTF8VALIDATE -#endif - -// It seems that many parsers do UTF-8 validation. -// RapidJSON does not do it by default, but a flag -// allows it. -#ifdef SIMDJSON_UTF8VALIDATE -#if defined (__AVX2__) -#elif defined (__SSE4_2__) || (defined(_MSC_VER) && defined(_M_AMD64)) -#elif defined(__ARM_NEON) || (defined(_MSC_VER) && defined(_M_ARM64)) -#endif // (__AVX2__) -#endif // SIMDJSON_UTF8VALIDATE - -//#define TRANSPOSE - namespace simdjson { -template + +template struct simd_input; -#ifdef __AVX2__ -template<> -struct simd_input -{ - __m256i lo; - __m256i hi; -}; -#endif - -#if defined(__SSE4_2__) || (defined(_MSC_VER) && defined(_M_AMD64)) -template<> -struct simd_input -{ - __m128i v0; - __m128i v1; - __m128i v2; - __m128i v3; -}; -#endif - -#if defined(__ARM_NEON) || (defined(_MSC_VER) && defined(_M_ARM64)) -template<> struct simd_input -{ -#ifndef TRANSPOSE - uint8x16_t i0; - uint8x16_t i1; - uint8x16_t i2; - uint8x16_t i3; -#else - uint8x16x4_t i; -#endif -}; -#endif - -#if defined(__ARM_NEON) || (defined(_MSC_VER) && defined(_M_ARM64)) -really_inline -uint16_t neonmovemask(uint8x16_t input) { - const uint8x16_t bitmask = { 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80, - 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80}; - uint8x16_t minput = vandq_u8(input, bitmask); - uint8x16_t tmp = vpaddq_u8(minput, minput); - tmp = vpaddq_u8(tmp, tmp); - tmp = vpaddq_u8(tmp, tmp); - return vgetq_lane_u16(vreinterpretq_u16_u8(tmp), 0); -} - -really_inline -uint64_t neonmovemask_bulk(uint8x16_t p0, uint8x16_t p1, uint8x16_t p2, uint8x16_t p3) { -#ifndef TRANSPOSE - const uint8x16_t bitmask = { 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80, - 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80}; - uint8x16_t t0 = vandq_u8(p0, bitmask); - uint8x16_t t1 = vandq_u8(p1, bitmask); - uint8x16_t t2 = vandq_u8(p2, bitmask); - uint8x16_t t3 = vandq_u8(p3, bitmask); - uint8x16_t sum0 = vpaddq_u8(t0, t1); - uint8x16_t sum1 = vpaddq_u8(t2, t3); - sum0 = vpaddq_u8(sum0, sum1); - sum0 = vpaddq_u8(sum0, sum0); - return vgetq_lane_u64(vreinterpretq_u64_u8(sum0), 0); -#else - const uint8x16_t bitmask1 = { 0x01, 0x10, 0x01, 0x10, 0x01, 0x10, 0x01, 0x10, - 0x01, 0x10, 0x01, 0x10, 0x01, 0x10, 0x01, 0x10}; - const uint8x16_t bitmask2 = { 0x02, 0x20, 0x02, 0x20, 0x02, 0x20, 0x02, 0x20, - 0x02, 0x20, 0x02, 0x20, 0x02, 0x20, 0x02, 0x20}; - const uint8x16_t bitmask3 = { 0x04, 0x40, 0x04, 0x40, 0x04, 0x40, 0x04, 0x40, - 0x04, 0x40, 0x04, 0x40, 0x04, 0x40, 0x04, 0x40}; - const uint8x16_t bitmask4 = { 0x08, 0x80, 0x08, 0x80, 0x08, 0x80, 0x08, 0x80, - 0x08, 0x80, 0x08, 0x80, 0x08, 0x80, 0x08, 0x80}; - uint8x16_t t0 = vandq_u8(p0, bitmask1); - uint8x16_t t1 = vbslq_u8(bitmask2, p1, t0); - uint8x16_t t2 = vbslq_u8(bitmask3, p2, t1); - uint8x16_t tmp = vbslq_u8(bitmask4, p3, t2); - uint8x16_t sum = vpaddq_u8(tmp, tmp); - return vgetq_lane_u64(vreinterpretq_u64_u8(sum), 0); -#endif -} -#endif - -template +template uint64_t compute_quote_mask(uint64_t quote_bits); namespace { @@ -37163,428 +37277,40 @@ namespace { } } -// In practice, if you have NEON or __PCLMUL__, you would -// always want to use them, but it might be useful, for research -// purposes, to disable it willingly, that's what SIMDJSON_AVOID_CLMUL -// does. -// Also: we don't know of an instance where AVX2 is supported but -// where clmul is not supported, so check for both, to be sure. -#ifdef SIMDJSON_AVOID_CLMUL -template really_inline -uint64_t compute_quote_mask(uint64_t quote_bits) { - return portable_compute_quote_mask(quote_bits); -} -#else -template -uint64_t compute_quote_mask(uint64_t quote_bits); - -#ifdef __AVX2__ -template<> really_inline -uint64_t compute_quote_mask(uint64_t quote_bits) { - // There should be no such thing with a processing supporting avx2 - // but not clmul. - uint64_t quote_mask = _mm_cvtsi128_si64(_mm_clmulepi64_si128( - _mm_set_epi64x(0ULL, quote_bits), _mm_set1_epi8(0xFF), 0)); - return quote_mask; -} -#endif - -#if defined(__SSE4_2__) || (defined(_MSC_VER) && defined(_M_AMD64)) -template<> really_inline -uint64_t compute_quote_mask(uint64_t quote_bits) { - // CLMUL is supported on some SSE42 hardware such as Sandy Bridge, - // but not on others. -#ifdef __PCLMUL__ - return _mm_cvtsi128_si64(_mm_clmulepi64_si128( - _mm_set_epi64x(0ULL, quote_bits), _mm_set1_epi8(0xFF), 0)); -#else - return portable_compute_quote_mask(quote_bits); -#endif -} -#endif - -#if defined(__ARM_NEON) || (defined(_MSC_VER) && defined(_M_ARM64)) -template<> really_inline -uint64_t compute_quote_mask(uint64_t quote_bits) { -#ifdef __ARM_FEATURE_CRYPTO // some ARM processors lack this extension - return vmull_p64( -1ULL, quote_bits); -#else - return portable_compute_quote_mask(quote_bits); -#endif -} -#endif -#endif // SIMDJSON_AVOID_CLMUL - -#ifdef SIMDJSON_UTF8VALIDATE // Holds the state required to perform check_utf8(). -template +template struct utf8_checking_state; -#ifdef __AVX2__ -template<> -struct utf8_checking_state -{ - __m256i has_error = _mm256_setzero_si256(); - avx_processed_utf_bytes previous { - _mm256_setzero_si256(), // rawbytes - _mm256_setzero_si256(), // high_nibbles - _mm256_setzero_si256() // carried_continuations - }; -}; -#endif -#if defined(__SSE4_2__) || (defined(_MSC_VER) && defined(_M_AMD64)) -template<> -struct utf8_checking_state -{ - __m128i has_error = _mm_setzero_si128(); - processed_utf_bytes previous { - _mm_setzero_si128(), // rawbytes - _mm_setzero_si128(), // high_nibbles - _mm_setzero_si128() // carried_continuations - }; -}; -#endif - -#if defined(__ARM_NEON) || (defined(_MSC_VER) && defined(_M_ARM64)) -template<> -struct utf8_checking_state -{ - int8x16_t has_error {}; - processed_utf_bytes previous {}; -}; -#endif - -#if defined(__ARM_NEON) || (defined(_MSC_VER) && defined(_M_ARM64)) -// Checks that all bytes are ascii -really_inline -bool check_ascii_neon(simd_input in) { - // checking if the most significant bit is always equal to 0. - uint8x16_t highbit = vdupq_n_u8(0x80); - uint8x16_t t0 = vorrq_u8(in.i0, in.i1); - uint8x16_t t1 = vorrq_u8(in.i2, in.i3); - uint8x16_t t3 = vorrq_u8(t0, t1); - uint8x16_t t4 = vandq_u8(t3, highbit); - uint64x2_t v64 = vreinterpretq_u64_u8(t4); - uint32x2_t v32 = vqmovn_u64(v64); - uint64x1_t result = vreinterpret_u64_u32(v32); - return vget_lane_u64(result, 0) == 0; -} -#endif - -template +template void check_utf8(simd_input in, utf8_checking_state& state); -#ifdef __AVX2__ -template<> really_inline -void check_utf8(simd_input in, - utf8_checking_state& state) { - __m256i highbit = _mm256_set1_epi8(0x80); - if ((_mm256_testz_si256(_mm256_or_si256(in.lo, in.hi), highbit)) == 1) { - // it is ascii, we just check continuation - state.has_error = _mm256_or_si256( - _mm256_cmpgt_epi8( - state.previous.carried_continuations, - _mm256_setr_epi8(9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 1)), - state.has_error); - } else { - // it is not ascii so we have to do heavy work - state.previous = avxcheckUTF8Bytes(in.lo, &(state.previous), &(state.has_error)); - state.previous = avxcheckUTF8Bytes(in.hi, &(state.previous), &(state.has_error)); - } -} -#endif //__AVX2__ - -#if defined(__SSE4_2__) || (defined(_MSC_VER) && defined(_M_AMD64)) -template<> really_inline -void check_utf8(simd_input in, - utf8_checking_state& state) { - __m128i highbit = _mm_set1_epi8(0x80); - if ((_mm_testz_si128(_mm_or_si128(in.v0, in.v1), highbit)) == 1) { - // it is ascii, we just check continuation - state.has_error = _mm_or_si128( - _mm_cmpgt_epi8( - state.previous.carried_continuations, - _mm_setr_epi8(9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 1)), - state.has_error); - } else { - // it is not ascii so we have to do heavy work - state.previous = checkUTF8Bytes(in.v0, &(state.previous), &(state.has_error)); - state.previous = checkUTF8Bytes(in.v1, &(state.previous), &(state.has_error)); - } - - if ((_mm_testz_si128(_mm_or_si128(in.v2, in.v3), highbit)) == 1) { - // it is ascii, we just check continuation - state.has_error = _mm_or_si128( - _mm_cmpgt_epi8( - state.previous.carried_continuations, - _mm_setr_epi8(9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 1)), - state.has_error); - } else { - // it is not ascii so we have to do heavy work - state.previous = checkUTF8Bytes(in.v2, &(state.previous), &(state.has_error)); - state.previous = checkUTF8Bytes(in.v3, &(state.previous), &(state.has_error)); - } -} -#endif // __SSE4_2 - -#if defined(__ARM_NEON) || (defined(_MSC_VER) && defined(_M_ARM64)) -template<> really_inline -void check_utf8(simd_input in, - utf8_checking_state& state) { - if (check_ascii_neon(in)) { - // All bytes are ascii. Therefore the byte that was just before must be ascii too. - // We only check the byte that was just before simd_input. Nines are arbitrary values. - int8_t _verror[] = {9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 1}; - state.has_error = - vorrq_s8(vreinterpretq_s8_u8(vcgtq_s8(state.previous.carried_continuations, - vld1q_s8(_verror))), - state.has_error); - } else { - // it is not ascii so we have to do heavy work - state.previous = checkUTF8Bytes(vreinterpretq_s8_u8(in.i0), &(state.previous), &(state.has_error)); - state.previous = checkUTF8Bytes(vreinterpretq_s8_u8(in.i1), &(state.previous), &(state.has_error)); - state.previous = checkUTF8Bytes(vreinterpretq_s8_u8(in.i2), &(state.previous), &(state.has_error)); - state.previous = checkUTF8Bytes(vreinterpretq_s8_u8(in.i3), &(state.previous), &(state.has_error)); - } -} -#endif // __ARM_NEON - // Checks if the utf8 validation has found any error. -template +template errorValues check_utf8_errors(utf8_checking_state& state); -#ifdef __AVX2__ -template<> really_inline -errorValues check_utf8_errors(utf8_checking_state& state) { - return _mm256_testz_si256(state.has_error, state.has_error) == 0 ? simdjson::UTF8_ERROR : simdjson::SUCCESS; -} -#endif - -#if defined(__SSE4_2__) || (defined(_MSC_VER) && defined(_M_AMD64)) -template<> really_inline -errorValues check_utf8_errors(utf8_checking_state& state) { - return _mm_testz_si128(state.has_error, state.has_error) == 0 ? simdjson::UTF8_ERROR : simdjson::SUCCESS; -} -#endif - -#if defined(__ARM_NEON) || (defined(_MSC_VER) && defined(_M_ARM64)) -template<> really_inline -errorValues check_utf8_errors(utf8_checking_state& state) { - uint64x2_t v64 = vreinterpretq_u64_s8(state.has_error); - uint32x2_t v32 = vqmovn_u64(v64); - uint64x1_t result = vreinterpret_u64_u32(v32); - return vget_lane_u64(result, 0) != 0 ? simdjson::UTF8_ERROR : simdjson::SUCCESS; -} -#endif -#endif // SIMDJSON_UTF8VALIDATE - -template -simd_input fill_input(const uint8_t * ptr); - -#ifdef __AVX2__ -template<> really_inline -simd_input fill_input(const uint8_t * ptr) { - struct simd_input in; - in.lo = _mm256_loadu_si256(reinterpret_cast(ptr + 0)); - in.hi = _mm256_loadu_si256(reinterpret_cast(ptr + 32)); - return in; -} -#endif - -#if defined(__SSE4_2__) || (defined(_MSC_VER) && defined(_M_AMD64)) -template<> really_inline -simd_input fill_input(const uint8_t * ptr) { - struct simd_input in; - in.v0 = _mm_loadu_si128(reinterpret_cast(ptr + 0)); - in.v1 = _mm_loadu_si128(reinterpret_cast(ptr + 16)); - in.v2 = _mm_loadu_si128(reinterpret_cast(ptr + 32)); - in.v3 = _mm_loadu_si128(reinterpret_cast(ptr + 48)); - return in; -} -#endif - -#if defined(__ARM_NEON) || (defined(_MSC_VER) && defined(_M_ARM64)) -template<> really_inline -simd_input fill_input(const uint8_t * ptr) { - struct simd_input in; -#ifndef TRANSPOSE - in.i0 = vld1q_u8(ptr + 0); - in.i1 = vld1q_u8(ptr + 16); - in.i2 = vld1q_u8(ptr + 32); - in.i3 = vld1q_u8(ptr + 48); -#else - in.i = vld4q_u8(ptr); -#endif - return in; -} -#endif - -// a straightforward comparison of a mask against input. 5 uops; would be -// cheaper in AVX512. -template +// a straightforward comparison of a mask against input. +template uint64_t cmp_mask_against_input(simd_input in, uint8_t m); -#ifdef __AVX2__ -template<> really_inline -uint64_t cmp_mask_against_input(simd_input in, uint8_t m) { - const __m256i mask = _mm256_set1_epi8(m); - __m256i cmp_res_0 = _mm256_cmpeq_epi8(in.lo, mask); - uint64_t res_0 = static_cast(_mm256_movemask_epi8(cmp_res_0)); - __m256i cmp_res_1 = _mm256_cmpeq_epi8(in.hi, mask); - uint64_t res_1 = _mm256_movemask_epi8(cmp_res_1); - return res_0 | (res_1 << 32); -} -#endif + +template +simd_input fill_input(const uint8_t * ptr); -#if defined(__SSE4_2__) || (defined(_MSC_VER) && defined(_M_AMD64)) -template<> really_inline -uint64_t cmp_mask_against_input(simd_input in, uint8_t m) { - const __m128i mask = _mm_set1_epi8(m); - __m128i cmp_res_0 = _mm_cmpeq_epi8(in.v0, mask); - uint64_t res_0 = _mm_movemask_epi8(cmp_res_0); - __m128i cmp_res_1 = _mm_cmpeq_epi8(in.v1, mask); - uint64_t res_1 = _mm_movemask_epi8(cmp_res_1); - __m128i cmp_res_2 = _mm_cmpeq_epi8(in.v2, mask); - uint64_t res_2 = _mm_movemask_epi8(cmp_res_2); - __m128i cmp_res_3 = _mm_cmpeq_epi8(in.v3, mask); - uint64_t res_3 = _mm_movemask_epi8(cmp_res_3); - return res_0 | (res_1 << 16) | (res_2 << 32) | (res_3 << 48); -} -#endif - -#if defined(__ARM_NEON) || (defined(_MSC_VER) && defined(_M_ARM64)) -template<> really_inline -uint64_t cmp_mask_against_input(simd_input in, uint8_t m) { - const uint8x16_t mask = vmovq_n_u8(m); - uint8x16_t cmp_res_0 = vceqq_u8(in.i0, mask); - uint8x16_t cmp_res_1 = vceqq_u8(in.i1, mask); - uint8x16_t cmp_res_2 = vceqq_u8(in.i2, mask); - uint8x16_t cmp_res_3 = vceqq_u8(in.i3, mask); - return neonmovemask_bulk(cmp_res_0, cmp_res_1, cmp_res_2, cmp_res_3); -} -#endif // find all values less than or equal than the content of maxval (using unsigned arithmetic) -template +template uint64_t unsigned_lteq_against_input(simd_input in, uint8_t m); -#ifdef __AVX2__ -template<> really_inline -uint64_t unsigned_lteq_against_input(simd_input in, uint8_t m) { - const __m256i maxval = _mm256_set1_epi8(m); - __m256i cmp_res_0 = _mm256_cmpeq_epi8(_mm256_max_epu8(maxval,in.lo),maxval); - uint64_t res_0 = static_cast(_mm256_movemask_epi8(cmp_res_0)); - __m256i cmp_res_1 = _mm256_cmpeq_epi8(_mm256_max_epu8(maxval,in.hi),maxval); - uint64_t res_1 = _mm256_movemask_epi8(cmp_res_1); - return res_0 | (res_1 << 32); -} -#endif -#if defined(__SSE4_2__) || (defined(_MSC_VER) && defined(_M_AMD64)) -template<> really_inline -uint64_t unsigned_lteq_against_input(simd_input in, uint8_t m) { - const __m128i maxval = _mm_set1_epi8(m); - __m128i cmp_res_0 = _mm_cmpeq_epi8(_mm_max_epu8(maxval,in.v0),maxval); - uint64_t res_0 = _mm_movemask_epi8(cmp_res_0); - __m128i cmp_res_1 = _mm_cmpeq_epi8(_mm_max_epu8(maxval,in.v1),maxval); - uint64_t res_1 = _mm_movemask_epi8(cmp_res_1); - __m128i cmp_res_2 = _mm_cmpeq_epi8(_mm_max_epu8(maxval,in.v2),maxval); - uint64_t res_2 = _mm_movemask_epi8(cmp_res_2); - __m128i cmp_res_3 = _mm_cmpeq_epi8(_mm_max_epu8(maxval,in.v3),maxval); - uint64_t res_3 = _mm_movemask_epi8(cmp_res_3); - return res_0 | (res_1 << 16) | (res_2 << 32) | (res_3 << 48); -} -#endif +template really_inline +uint64_t find_odd_backslash_sequences(simd_input in, uint64_t &prev_iter_ends_odd_backslash); -#if defined(__ARM_NEON) || (defined(_MSC_VER) && defined(_M_ARM64)) -template<> really_inline -uint64_t unsigned_lteq_against_input(simd_input in, uint8_t m) { - const uint8x16_t mask = vmovq_n_u8(m); - uint8x16_t cmp_res_0 = vcleq_u8(in.i0, mask); - uint8x16_t cmp_res_1 = vcleq_u8(in.i1, mask); - uint8x16_t cmp_res_2 = vcleq_u8(in.i2, mask); - uint8x16_t cmp_res_3 = vcleq_u8(in.i3, mask); - return neonmovemask_bulk(cmp_res_0, cmp_res_1, cmp_res_2, cmp_res_3); -} -#endif -// return a bitvector indicating where we have characters that end an odd-length -// sequence of backslashes (and thus change the behavior of the next character -// to follow). A even-length sequence of backslashes, and, for that matter, the -// largest even-length prefix of our odd-length sequence of backslashes, simply -// modify the behavior of the backslashes themselves. -// We also update the prev_iter_ends_odd_backslash reference parameter to -// indicate whether we end an iteration on an odd-length sequence of -// backslashes, which modifies our subsequent search for odd-length -// sequences of backslashes in an obvious way. -template really_inline -uint64_t find_odd_backslash_sequences(simd_input in, uint64_t &prev_iter_ends_odd_backslash) { - const uint64_t even_bits = 0x5555555555555555ULL; - const uint64_t odd_bits = ~even_bits; - uint64_t bs_bits = cmp_mask_against_input(in, '\\'); - uint64_t start_edges = bs_bits & ~(bs_bits << 1); - // flip lowest if we have an odd-length run at the end of the prior - // iteration - uint64_t even_start_mask = even_bits ^ prev_iter_ends_odd_backslash; - uint64_t even_starts = start_edges & even_start_mask; - uint64_t odd_starts = start_edges & ~even_start_mask; - uint64_t even_carries = bs_bits + even_starts; - - uint64_t odd_carries; - // must record the carry-out of our odd-carries out of bit 63; this - // indicates whether the sense of any edge going to the next iteration - // should be flipped - bool iter_ends_odd_backslash = - add_overflow(bs_bits, odd_starts, &odd_carries); - - odd_carries |= - prev_iter_ends_odd_backslash; // push in bit zero as a potential end - // if we had an odd-numbered run at the - // end of the previous iteration - prev_iter_ends_odd_backslash = iter_ends_odd_backslash ? 0x1ULL : 0x0ULL; - uint64_t even_carry_ends = even_carries & ~bs_bits; - uint64_t odd_carry_ends = odd_carries & ~bs_bits; - uint64_t even_start_odd_end = even_carry_ends & odd_bits; - uint64_t odd_start_even_end = odd_carry_ends & even_bits; - uint64_t odd_ends = even_start_odd_end | odd_start_even_end; - return odd_ends; -} - -// return both the quote mask (which is a half-open mask that covers the first -// quote -// in an unescaped quote pair and everything in the quote pair) and the quote -// bits, which are the simple -// unescaped quoted bits. We also update the prev_iter_inside_quote value to -// tell the next iteration -// whether we finished the final iteration inside a quote pair; if so, this -// inverts our behavior of -// whether we're inside quotes for the next iteration. -// Note that we don't do any error checking to see if we have backslash -// sequences outside quotes; these -// backslash sequences (of any length) will be detected elsewhere. -template really_inline +template really_inline uint64_t find_quote_mask_and_bits(simd_input in, uint64_t odd_ends, - uint64_t &prev_iter_inside_quote, uint64_t "e_bits, uint64_t &error_mask) { - quote_bits = cmp_mask_against_input(in, '"'); - quote_bits = quote_bits & ~odd_ends; - uint64_t quote_mask = compute_quote_mask(quote_bits); - quote_mask ^= prev_iter_inside_quote; - // All Unicode characters may be placed within the - // quotation marks, except for the characters that MUST be escaped: - // quotation mark, reverse solidus, and the control characters (U+0000 - //through U+001F). - // https://tools.ietf.org/html/rfc8259 - uint64_t unescaped = unsigned_lteq_against_input(in, 0x1F); - error_mask |= quote_mask & unescaped; - // right shift of a signed value expected to be well-defined and standard - // compliant as of C++20, - // John Regher from Utah U. says this is fine code - prev_iter_inside_quote = - static_cast(static_cast(quote_mask) >> 63); - return quote_mask; -} + uint64_t &prev_iter_inside_quote, uint64_t "e_bits, uint64_t &error_mask); + // do a 'shufti' to detect structural JSON characters // they are { 0x7b } 0x7d : 0x3a [ 0x5b ] 0x5d , 0x2c @@ -37593,227 +37319,75 @@ uint64_t find_quote_mask_and_bits(simd_input in, uint64_t odd_ends, // we are also interested in the four whitespace characters // space 0x20, linefeed 0x0a, horizontal tab 0x09 and carriage return 0x0d // these go into the next 2 buckets of the comparison (8/16) -template +template void find_whitespace_and_structurals(simd_input in, uint64_t &whitespace, uint64_t &structurals); -#ifdef __AVX2__ -template<> really_inline -void find_whitespace_and_structurals(simd_input in, - uint64_t &whitespace, - uint64_t &structurals) { -#ifdef SIMDJSON_NAIVE_STRUCTURAL - // You should never need this naive approach, but it can be useful - // for research purposes - const __m256i mask_open_brace = _mm256_set1_epi8(0x7b); - __m256i struct_lo = _mm256_cmpeq_epi8(in.lo, mask_open_brace); - __m256i struct_hi = _mm256_cmpeq_epi8(in.hi, mask_open_brace); - const __m256i mask_close_brace = _mm256_set1_epi8(0x7d); - struct_lo = _mm256_or_si256(struct_lo,_mm256_cmpeq_epi8(in.lo, mask_close_brace)); - struct_hi = _mm256_or_si256(struct_hi,_mm256_cmpeq_epi8(in.hi, mask_close_brace)); - const __m256i mask_open_bracket = _mm256_set1_epi8(0x5b); - struct_lo = _mm256_or_si256(struct_lo,_mm256_cmpeq_epi8(in.lo, mask_open_bracket)); - struct_hi = _mm256_or_si256(struct_hi,_mm256_cmpeq_epi8(in.hi, mask_open_bracket)); - const __m256i mask_close_bracket = _mm256_set1_epi8(0x5d); - struct_lo = _mm256_or_si256(struct_lo,_mm256_cmpeq_epi8(in.lo, mask_close_bracket)); - struct_hi = _mm256_or_si256(struct_hi,_mm256_cmpeq_epi8(in.hi, mask_close_bracket)); - const __m256i mask_column = _mm256_set1_epi8(0x3a); - struct_lo = _mm256_or_si256(struct_lo,_mm256_cmpeq_epi8(in.lo, mask_column)); - struct_hi = _mm256_or_si256(struct_hi,_mm256_cmpeq_epi8(in.hi, mask_column)); - const __m256i mask_comma = _mm256_set1_epi8(0x2c); - struct_lo = _mm256_or_si256(struct_lo,_mm256_cmpeq_epi8(in.lo, mask_comma)); - struct_hi = _mm256_or_si256(struct_hi,_mm256_cmpeq_epi8(in.hi, mask_comma)); - uint64_t structural_res_0 = static_cast(_mm256_movemask_epi8(struct_lo)); - uint64_t structural_res_1 = _mm256_movemask_epi8(struct_hi); - structurals = (structural_res_0 | (structural_res_1 << 32)); +// return a updated structural bit vector with quoted contents cleared out and +// pseudo-structural characters added to the mask +// updates prev_iter_ends_pseudo_pred which tells us whether the previous +// iteration ended on a whitespace or a structural character (which means that +// the next iteration +// will have a pseudo-structural character at its start) +really_inline uint64_t finalize_structurals( + uint64_t structurals, uint64_t whitespace, uint64_t quote_mask, + uint64_t quote_bits, uint64_t &prev_iter_ends_pseudo_pred) { + // mask off anything inside quotes + structurals &= ~quote_mask; + // add the real quote bits back into our bitmask as well, so we can + // quickly traverse the strings we've spent all this trouble gathering + structurals |= quote_bits; + // Now, establish "pseudo-structural characters". These are non-whitespace + // characters that are (a) outside quotes and (b) have a predecessor that's + // either whitespace or a structural character. This means that subsequent + // passes will get a chance to encounter the first character of every string + // of non-whitespace and, if we're parsing an atom like true/false/null or a + // number we can stop at the first whitespace or structural character + // following it. - const __m256i mask_space = _mm256_set1_epi8(0x20); - __m256i space_lo = _mm256_cmpeq_epi8(in.lo, mask_space); - __m256i space_hi = _mm256_cmpeq_epi8(in.hi, mask_space); - const __m256i mask_linefeed = _mm256_set1_epi8(0x0a); - space_lo = _mm256_or_si256(space_lo,_mm256_cmpeq_epi8(in.lo, mask_linefeed)); - space_hi = _mm256_or_si256(space_hi,_mm256_cmpeq_epi8(in.hi, mask_linefeed)); - const __m256i mask_tab = _mm256_set1_epi8(0x09); - space_lo = _mm256_or_si256(space_lo,_mm256_cmpeq_epi8(in.lo, mask_tab)); - space_hi = _mm256_or_si256(space_hi,_mm256_cmpeq_epi8(in.hi, mask_tab)); - const __m256i mask_carriage = _mm256_set1_epi8(0x0d); - space_lo = _mm256_or_si256(space_lo,_mm256_cmpeq_epi8(in.lo, mask_carriage)); - space_hi = _mm256_or_si256(space_hi,_mm256_cmpeq_epi8(in.hi, mask_carriage)); + // a qualified predecessor is something that can happen 1 position before an + // pseudo-structural character + uint64_t pseudo_pred = structurals | whitespace; - uint64_t ws_res_0 = static_cast(_mm256_movemask_epi8(space_lo)); - uint64_t ws_res_1 = _mm256_movemask_epi8(space_hi); - whitespace = (ws_res_0 | (ws_res_1 << 32)); - // end of naive approach + uint64_t shifted_pseudo_pred = + (pseudo_pred << 1) | prev_iter_ends_pseudo_pred; + prev_iter_ends_pseudo_pred = pseudo_pred >> 63; + uint64_t pseudo_structurals = + shifted_pseudo_pred & (~whitespace) & (~quote_mask); + structurals |= pseudo_structurals; -#else // SIMDJSON_NAIVE_STRUCTURAL - const __m256i low_nibble_mask = _mm256_setr_epi8( - 16, 0, 0, 0, 0, 0, 0, 0, 0, 8, 12, 1, 2, 9, 0, 0, - 16, 0, 0, 0, 0, 0, 0, 0, 0, 8, 12, 1, 2, 9, 0, 0); - const __m256i high_nibble_mask = _mm256_setr_epi8( - 8, 0, 18, 4, 0, 1, 0, 1, 0, 0, 0, 3, 2, 1, 0, 0, - 8, 0, 18, 4, 0, 1, 0, 1, 0, 0, 0, 3, 2, 1, 0, 0); - - __m256i structural_shufti_mask = _mm256_set1_epi8(0x7); - __m256i whitespace_shufti_mask = _mm256_set1_epi8(0x18); - - __m256i v_lo = _mm256_and_si256( - _mm256_shuffle_epi8(low_nibble_mask, in.lo), - _mm256_shuffle_epi8(high_nibble_mask, - _mm256_and_si256(_mm256_srli_epi32(in.lo, 4), - _mm256_set1_epi8(0x7f)))); - - __m256i v_hi = _mm256_and_si256( - _mm256_shuffle_epi8(low_nibble_mask, in.hi), - _mm256_shuffle_epi8(high_nibble_mask, - _mm256_and_si256(_mm256_srli_epi32(in.hi, 4), - _mm256_set1_epi8(0x7f)))); - __m256i tmp_lo = _mm256_cmpeq_epi8( - _mm256_and_si256(v_lo, structural_shufti_mask), _mm256_set1_epi8(0)); - __m256i tmp_hi = _mm256_cmpeq_epi8( - _mm256_and_si256(v_hi, structural_shufti_mask), _mm256_set1_epi8(0)); - - uint64_t structural_res_0 = - static_cast(_mm256_movemask_epi8(tmp_lo)); - uint64_t structural_res_1 = _mm256_movemask_epi8(tmp_hi); - structurals = ~(structural_res_0 | (structural_res_1 << 32)); - - __m256i tmp_ws_lo = _mm256_cmpeq_epi8( - _mm256_and_si256(v_lo, whitespace_shufti_mask), _mm256_set1_epi8(0)); - __m256i tmp_ws_hi = _mm256_cmpeq_epi8( - _mm256_and_si256(v_hi, whitespace_shufti_mask), _mm256_set1_epi8(0)); - - uint64_t ws_res_0 = static_cast(_mm256_movemask_epi8(tmp_ws_lo)); - uint64_t ws_res_1 = _mm256_movemask_epi8(tmp_ws_hi); - whitespace = ~(ws_res_0 | (ws_res_1 << 32)); -#endif // SIMDJSON_NAIVE_STRUCTURAL + // now, we've used our close quotes all we need to. So let's switch them off + // they will be off in the quote mask and on in quote bits. + structurals &= ~(quote_bits & ~quote_mask); + return structurals; } -#endif // __AVX2__ -#if defined(__SSE4_2__) || (defined(_MSC_VER) && defined(_M_AMD64)) -template<> really_inline -void find_whitespace_and_structurals(simd_input in, - uint64_t &whitespace, - uint64_t &structurals) { - const __m128i low_nibble_mask = _mm_setr_epi8( - 16, 0, 0, 0, 0, 0, 0, 0, 0, 8, 12, 1, 2, 9, 0, 0); - const __m128i high_nibble_mask = _mm_setr_epi8( - 8, 0, 18, 4, 0, 1, 0, 1, 0, 0, 0, 3, 2, 1, 0, 0); +template +int find_structural_bits(const uint8_t *buf, size_t len, simdjson::ParsedJson &pj); - __m128i structural_shufti_mask = _mm_set1_epi8(0x7); - __m128i whitespace_shufti_mask = _mm_set1_epi8(0x18); - - __m128i v_0 = _mm_and_si128( - _mm_shuffle_epi8(low_nibble_mask, in.v0), - _mm_shuffle_epi8(high_nibble_mask, - _mm_and_si128(_mm_srli_epi32(in.v0, 4), - _mm_set1_epi8(0x7f)))); - - __m128i v_1 = _mm_and_si128( - _mm_shuffle_epi8(low_nibble_mask, in.v1), - _mm_shuffle_epi8(high_nibble_mask, - _mm_and_si128(_mm_srli_epi32(in.v1, 4), - _mm_set1_epi8(0x7f)))); - - __m128i v_2 = _mm_and_si128( - _mm_shuffle_epi8(low_nibble_mask, in.v2), - _mm_shuffle_epi8(high_nibble_mask, - _mm_and_si128(_mm_srli_epi32(in.v2, 4), - _mm_set1_epi8(0x7f)))); - - __m128i v_3 = _mm_and_si128( - _mm_shuffle_epi8(low_nibble_mask, in.v3), - _mm_shuffle_epi8(high_nibble_mask, - _mm_and_si128(_mm_srli_epi32(in.v3, 4), - _mm_set1_epi8(0x7f)))); - - __m128i tmp_v0 = _mm_cmpeq_epi8( - _mm_and_si128(v_0, structural_shufti_mask), _mm_set1_epi8(0)); - __m128i tmp_v1 = _mm_cmpeq_epi8( - _mm_and_si128(v_1, structural_shufti_mask), _mm_set1_epi8(0)); - __m128i tmp_v2 = _mm_cmpeq_epi8( - _mm_and_si128(v_2, structural_shufti_mask), _mm_set1_epi8(0)); - __m128i tmp_v3 = _mm_cmpeq_epi8( - _mm_and_si128(v_3, structural_shufti_mask), _mm_set1_epi8(0)); - - uint64_t structural_res_0 = _mm_movemask_epi8(tmp_v0); - uint64_t structural_res_1 = _mm_movemask_epi8(tmp_v1); - uint64_t structural_res_2 = _mm_movemask_epi8(tmp_v2); - uint64_t structural_res_3 = _mm_movemask_epi8(tmp_v3); - - structurals = ~(structural_res_0 | (structural_res_1 << 16) | (structural_res_2 << 32) | (structural_res_3 << 48)); - - __m128i tmp_ws_v0 = _mm_cmpeq_epi8( - _mm_and_si128(v_0, whitespace_shufti_mask), _mm_set1_epi8(0)); - __m128i tmp_ws_v1 = _mm_cmpeq_epi8( - _mm_and_si128(v_1, whitespace_shufti_mask), _mm_set1_epi8(0)); - __m128i tmp_ws_v2 = _mm_cmpeq_epi8( - _mm_and_si128(v_2, whitespace_shufti_mask), _mm_set1_epi8(0)); - __m128i tmp_ws_v3 = _mm_cmpeq_epi8( - _mm_and_si128(v_3, whitespace_shufti_mask), _mm_set1_epi8(0)); - - uint64_t ws_res_0 = _mm_movemask_epi8(tmp_ws_v0); - uint64_t ws_res_1 = _mm_movemask_epi8(tmp_ws_v1); - uint64_t ws_res_2 = _mm_movemask_epi8(tmp_ws_v2); - uint64_t ws_res_3 = _mm_movemask_epi8(tmp_ws_v3); - - whitespace = ~(ws_res_0 | (ws_res_1 << 16) | (ws_res_2 << 32) | (ws_res_3 << 48)); +template +int find_structural_bits(const char *buf, size_t len, simdjson::ParsedJson &pj) { + return find_structural_bits((const uint8_t*)buf, len, pj); } -#endif // __SSE4_2__ -#if defined(__ARM_NEON) || (defined(_MSC_VER) && defined(_M_ARM64)) -template<> really_inline -void find_whitespace_and_structurals( - simd_input in, - uint64_t &whitespace, - uint64_t &structurals) { - const uint8x16_t low_nibble_mask = (uint8x16_t){ - 16, 0, 0, 0, 0, 0, 0, 0, 0, 8, 12, 1, 2, 9, 0, 0}; - const uint8x16_t high_nibble_mask = (uint8x16_t){ - 8, 0, 18, 4, 0, 1, 0, 1, 0, 0, 0, 3, 2, 1, 0, 0}; - const uint8x16_t structural_shufti_mask = vmovq_n_u8(0x7); - const uint8x16_t whitespace_shufti_mask = vmovq_n_u8(0x18); - const uint8x16_t low_nib_and_mask = vmovq_n_u8(0xf); - uint8x16_t nib_0_lo = vandq_u8(in.i0, low_nib_and_mask); - uint8x16_t nib_0_hi = vshrq_n_u8(in.i0, 4); - uint8x16_t shuf_0_lo = vqtbl1q_u8(low_nibble_mask, nib_0_lo); - uint8x16_t shuf_0_hi = vqtbl1q_u8(high_nibble_mask, nib_0_hi); - uint8x16_t v_0 = vandq_u8(shuf_0_lo, shuf_0_hi); +} // namespace simdjson - uint8x16_t nib_1_lo = vandq_u8(in.i1, low_nib_and_mask); - uint8x16_t nib_1_hi = vshrq_n_u8(in.i1, 4); - uint8x16_t shuf_1_lo = vqtbl1q_u8(low_nibble_mask, nib_1_lo); - uint8x16_t shuf_1_hi = vqtbl1q_u8(high_nibble_mask, nib_1_hi); - uint8x16_t v_1 = vandq_u8(shuf_1_lo, shuf_1_hi); - uint8x16_t nib_2_lo = vandq_u8(in.i2, low_nib_and_mask); - uint8x16_t nib_2_hi = vshrq_n_u8(in.i2, 4); - uint8x16_t shuf_2_lo = vqtbl1q_u8(low_nibble_mask, nib_2_lo); - uint8x16_t shuf_2_hi = vqtbl1q_u8(high_nibble_mask, nib_2_hi); - uint8x16_t v_2 = vandq_u8(shuf_2_lo, shuf_2_hi); - uint8x16_t nib_3_lo = vandq_u8(in.i3, low_nib_and_mask); - uint8x16_t nib_3_hi = vshrq_n_u8(in.i3, 4); - uint8x16_t shuf_3_lo = vqtbl1q_u8(low_nibble_mask, nib_3_lo); - uint8x16_t shuf_3_hi = vqtbl1q_u8(high_nibble_mask, nib_3_hi); - uint8x16_t v_3 = vandq_u8(shuf_3_lo, shuf_3_hi); - uint8x16_t tmp_0 = vtstq_u8(v_0, structural_shufti_mask); - uint8x16_t tmp_1 = vtstq_u8(v_1, structural_shufti_mask); - uint8x16_t tmp_2 = vtstq_u8(v_2, structural_shufti_mask); - uint8x16_t tmp_3 = vtstq_u8(v_3, structural_shufti_mask); - structurals = neonmovemask_bulk(tmp_0, tmp_1, tmp_2, tmp_3); - uint8x16_t tmp_ws_0 = vtstq_u8(v_0, whitespace_shufti_mask); - uint8x16_t tmp_ws_1 = vtstq_u8(v_1, whitespace_shufti_mask); - uint8x16_t tmp_ws_2 = vtstq_u8(v_2, whitespace_shufti_mask); - uint8x16_t tmp_ws_3 = vtstq_u8(v_3, whitespace_shufti_mask); - whitespace = neonmovemask_bulk(tmp_ws_0, tmp_ws_1, tmp_ws_2, tmp_ws_3); -} -#endif // __ARM_NEON +#endif +/* end file include/simdjson/stage1_find_marks.h */ +/* begin file include/simdjson/stage1_find_marks_flatten.h */ +#ifndef SIMDJSON_STAGE1_FIND_MARKS_FLATTEN_H +#define SIMDJSON_STAGE1_FIND_MARKS_FLATTEN_H + +namespace simdjson { + #ifdef SIMDJSON_NAIVE_FLATTEN // useful for benchmarking // // This is just a naive implementation. It should be normally @@ -37898,198 +37472,793 @@ really_inline void flatten_bits(uint32_t *base_ptr, uint32_t &base, base = next_base; } #endif // SIMDJSON_NAIVE_FLATTEN - -// return a updated structural bit vector with quoted contents cleared out and -// pseudo-structural characters added to the mask -// updates prev_iter_ends_pseudo_pred which tells us whether the previous -// iteration ended on a whitespace or a structural character (which means that -// the next iteration -// will have a pseudo-structural character at its start) -really_inline uint64_t finalize_structurals( - uint64_t structurals, uint64_t whitespace, uint64_t quote_mask, - uint64_t quote_bits, uint64_t &prev_iter_ends_pseudo_pred) { - // mask off anything inside quotes - structurals &= ~quote_mask; - // add the real quote bits back into our bitmask as well, so we can - // quickly traverse the strings we've spent all this trouble gathering - structurals |= quote_bits; - // Now, establish "pseudo-structural characters". These are non-whitespace - // characters that are (a) outside quotes and (b) have a predecessor that's - // either whitespace or a structural character. This means that subsequent - // passes will get a chance to encounter the first character of every string - // of non-whitespace and, if we're parsing an atom like true/false/null or a - // number we can stop at the first whitespace or structural character - // following it. - - // a qualified predecessor is something that can happen 1 position before an - // pseudo-structural character - uint64_t pseudo_pred = structurals | whitespace; - - uint64_t shifted_pseudo_pred = - (pseudo_pred << 1) | prev_iter_ends_pseudo_pred; - prev_iter_ends_pseudo_pred = pseudo_pred >> 63; - uint64_t pseudo_structurals = - shifted_pseudo_pred & (~whitespace) & (~quote_mask); - structurals |= pseudo_structurals; - - // now, we've used our close quotes all we need to. So let's switch them off - // they will be off in the quote mask and on in quote bits. - structurals &= ~(quote_bits & ~quote_mask); - return structurals; } -template -WARN_UNUSED -/*never_inline*/ int find_structural_bits(const uint8_t *buf, size_t len, - ParsedJson &pj) { - if (len > pj.bytecapacity) { - std::cerr << "Your ParsedJson object only supports documents up to " - << pj.bytecapacity << " bytes but you are trying to process " << len - << " bytes" << std::endl; - return simdjson::CAPACITY; - } - uint32_t *base_ptr = pj.structural_indexes; - uint32_t base = 0; -#ifdef SIMDJSON_UTF8VALIDATE - utf8_checking_state state; -#endif +#endif // SIMDJSON_STAGE1_FIND_MARKS_FLATTEN_H +/* end file include/simdjson/stage1_find_marks_flatten.h */ +/* begin file include/simdjson/stage1_find_marks_macros.h */ +#ifndef SIMDJSON_STAGE1_FIND_MARKS_MACROS_H +#define SIMDJSON_STAGE1_FIND_MARKS_MACROS_H - // we have padded the input out to 64 byte multiple with the remainder being - // zeros +// return a bitvector indicating where we have characters that end an odd-length +// sequence of backslashes (and thus change the behavior of the next character +// to follow). A even-length sequence of backslashes, and, for that matter, the +// largest even-length prefix of our odd-length sequence of backslashes, simply +// modify the behavior of the backslashes themselves. +// We also update the prev_iter_ends_odd_backslash reference parameter to +// indicate whether we end an iteration on an odd-length sequence of +// backslashes, which modifies our subsequent search for odd-length +// sequences of backslashes in an obvious way. +// We need to compile that code for multiple architectures. However, target attributes can be used +// only once by function definition. Huge macro seemed better than huge code duplication. +// uint64_t FIND_ODD_BACKSLASH_SEQUENCES(architecture T, simd_input in, uint64_t &prev_iter_ends_odd_backslash) +#define FIND_ODD_BACKSLASH_SEQUENCES(T, in, prev_iter_ends_odd_backslash) { \ + const uint64_t even_bits = 0x5555555555555555ULL; \ + const uint64_t odd_bits = ~even_bits; \ + uint64_t bs_bits = cmp_mask_against_input(in, '\\'); \ + uint64_t start_edges = bs_bits & ~(bs_bits << 1); \ + /* flip lowest if we have an odd-length run at the end of the prior */ \ + /* iteration */ \ + uint64_t even_start_mask = even_bits ^ prev_iter_ends_odd_backslash; \ + uint64_t even_starts = start_edges & even_start_mask; \ + uint64_t odd_starts = start_edges & ~even_start_mask; \ + uint64_t even_carries = bs_bits + even_starts; \ + \ + uint64_t odd_carries; \ + /* must record the carry-out of our odd-carries out of bit 63; this */ \ + /* indicates whether the sense of any edge going to the next iteration */ \ + /* should be flipped */ \ + bool iter_ends_odd_backslash = \ + add_overflow(bs_bits, odd_starts, &odd_carries); \ + \ + odd_carries |= \ + prev_iter_ends_odd_backslash; /* push in bit zero as a potential end */ \ + /* if we had an odd-numbered run at the */ \ + /* end of the previous iteration */ \ + prev_iter_ends_odd_backslash = iter_ends_odd_backslash ? 0x1ULL : 0x0ULL; \ + uint64_t even_carry_ends = even_carries & ~bs_bits; \ + uint64_t odd_carry_ends = odd_carries & ~bs_bits; \ + uint64_t even_start_odd_end = even_carry_ends & odd_bits; \ + uint64_t odd_start_even_end = odd_carry_ends & even_bits; \ + uint64_t odd_ends = even_start_odd_end | odd_start_even_end; \ + return odd_ends; \ +} - // persistent state across loop - // does the last iteration end with an odd-length sequence of backslashes? - // either 0 or 1, but a 64-bit value - uint64_t prev_iter_ends_odd_backslash = 0ULL; - // does the previous iteration end inside a double-quote pair? - uint64_t prev_iter_inside_quote = 0ULL; // either all zeros or all ones - // does the previous iteration end on something that is a predecessor of a - // pseudo-structural character - i.e. whitespace or a structural character - // effectively the very first char is considered to follow "whitespace" for - // the - // purposes of pseudo-structural character detection so we initialize to 1 - uint64_t prev_iter_ends_pseudo_pred = 1ULL; - // structurals are persistent state across loop as we flatten them on the - // subsequent iteration into our array pointed to be base_ptr. - // This is harmless on the first iteration as structurals==0 - // and is done for performance reasons; we can hide some of the latency of the - // expensive carryless multiply in the previous step with this work - uint64_t structurals = 0; +// return both the quote mask (which is a half-open mask that covers the first +// quote +// in an unescaped quote pair and everything in the quote pair) and the quote +// bits, which are the simple +// unescaped quoted bits. We also update the prev_iter_inside_quote value to +// tell the next iteration +// whether we finished the final iteration inside a quote pair; if so, this +// inverts our behavior of +// whether we're inside quotes for the next iteration. +// Note that we don't do any error checking to see if we have backslash +// sequences outside quotes; these +// backslash sequences (of any length) will be detected elsewhere. +// We need to compile that code for multiple architectures. However, target attributes can be used +// only once by function definition. Huge macro seemed better than huge code duplication. +// uint64_t FIND_QUOTE_MASK_AND_BITS(architecture T, simd_input in, uint64_t odd_ends, +// uint64_t &prev_iter_inside_quote, uint64_t "e_bits, uint64_t &error_mask) +#define FIND_QUOTE_MASK_AND_BITS(T, in, odd_ends, prev_iter_inside_quote, quote_bits, error_mask) { \ + quote_bits = cmp_mask_against_input(in, '"'); \ + quote_bits = quote_bits & ~odd_ends; \ + uint64_t quote_mask = compute_quote_mask(quote_bits); \ + quote_mask ^= prev_iter_inside_quote; \ + /* All Unicode characters may be placed within the */ \ + /* quotation marks, except for the characters that MUST be escaped: */ \ + /* quotation mark, reverse solidus, and the control characters (U+0000 */ \ + /*through U+001F). */ \ + /* https://tools.ietf.org/html/rfc8259 */ \ + uint64_t unescaped = unsigned_lteq_against_input(in, 0x1F); \ + error_mask |= quote_mask & unescaped; \ + /* right shift of a signed value expected to be well-defined and standard */ \ + /* compliant as of C++20, */ \ + /* John Regher from Utah U. says this is fine code */ \ + prev_iter_inside_quote = \ + static_cast(static_cast(quote_mask) >> 63); \ + return quote_mask; \ +} \ - size_t lenminus64 = len < 64 ? 0 : len - 64; - size_t idx = 0; - uint64_t error_mask = 0; // for unescaped characters within strings (ASCII code points < 0x20) - for (; idx < lenminus64; idx += 64) { -#ifndef _MSC_VER - __builtin_prefetch(buf + idx + 128); -#endif - simd_input in = fill_input(buf+idx); -#ifdef SIMDJSON_UTF8VALIDATE - check_utf8(in, state); -#endif - // detect odd sequences of backslashes - uint64_t odd_ends = find_odd_backslash_sequences( - in, prev_iter_ends_odd_backslash); - // detect insides of quote pairs ("quote_mask") and also our quote_bits - // themselves - uint64_t quote_bits; - uint64_t quote_mask = find_quote_mask_and_bits( - in, odd_ends, prev_iter_inside_quote, quote_bits, error_mask); +// We need to compile that code for multiple architectures. However, target attributes can be used +// only once by function definition. Huge macro seemed better than huge code duplication. +// FIND_STRUCTURAL_BITS(architecture T, const uint8_t *buf, size_t len, ParsedJson &pj) +#define FIND_STRUCTURAL_BITS(T, buf, len, pj) { \ + if (len > pj.bytecapacity) { \ + std::cerr << "Your ParsedJson object only supports documents up to " \ + << pj.bytecapacity << " bytes but you are trying to process " << len \ + << " bytes" << std::endl; \ + return simdjson::CAPACITY; \ + } \ + uint32_t *base_ptr = pj.structural_indexes; \ + uint32_t base = 0; \ + utf8_checking_state state; \ + \ + /* we have padded the input out to 64 byte multiple with the remainder being */ \ + /* zeros */ \ + \ + /* persistent state across loop */ \ + /* does the last iteration end with an odd-length sequence of backslashes? */ \ + /* either 0 or 1, but a 64-bit value */ \ + uint64_t prev_iter_ends_odd_backslash = 0ULL; \ + /* does the previous iteration end inside a double-quote pair? */ \ + uint64_t prev_iter_inside_quote = 0ULL; /* either all zeros or all ones */ \ + /* does the previous iteration end on something that is a predecessor of a */ \ + /* pseudo-structural character - i.e. whitespace or a structural character */ \ + /* effectively the very first char is considered to follow "whitespace" for */ \ + /* the */ \ + /* purposes of pseudo-structural character detection so we initialize to 1 */ \ + uint64_t prev_iter_ends_pseudo_pred = 1ULL; \ + \ + /* structurals are persistent state across loop as we flatten them on the */ \ + /* subsequent iteration into our array pointed to be base_ptr. */ \ + /* This is harmless on the first iteration as structurals==0 */ \ + /* and is done for performance reasons; we can hide some of the latency of the */ \ + /* expensive carryless multiply in the previous step with this work */ \ + uint64_t structurals = 0; \ + \ + size_t lenminus64 = len < 64 ? 0 : len - 64; \ + size_t idx = 0; \ + uint64_t error_mask = 0; /* for unescaped characters within strings (ASCII code points < 0x20) */ \ + \ + for (; idx < lenminus64; idx += 64) { \ + \ + simd_input in = fill_input(buf+idx); \ + check_utf8(in, state); \ + /* detect odd sequences of backslashes */ \ + uint64_t odd_ends = find_odd_backslash_sequences( \ + in, prev_iter_ends_odd_backslash); \ + \ + /* detect insides of quote pairs ("quote_mask") and also our quote_bits */ \ + /* themselves */ \ + uint64_t quote_bits; \ + uint64_t quote_mask = find_quote_mask_and_bits( \ + in, odd_ends, prev_iter_inside_quote, quote_bits, error_mask); \ + \ + /* take the previous iterations structural bits, not our current iteration, */ \ + /* and flatten */ \ + flatten_bits(base_ptr, base, idx, structurals); \ + \ + uint64_t whitespace; \ + find_whitespace_and_structurals(in, whitespace, structurals); \ + \ + /* fixup structurals to reflect quotes and add pseudo-structural characters */ \ + structurals = finalize_structurals(structurals, whitespace, quote_mask, \ + quote_bits, prev_iter_ends_pseudo_pred); \ + } \ + \ + /*////////////// */ \ + /*/ we use a giant copy-paste which is ugly. */ \ + /*/ but otherwise the string needs to be properly padded or else we */ \ + /*/ risk invalidating the UTF-8 checks. */ \ + /*////////// */ \ + if (idx < len) { \ + uint8_t tmpbuf[64]; \ + memset(tmpbuf, 0x20, 64); \ + memcpy(tmpbuf, buf + idx, len - idx); \ + simd_input in = fill_input(tmpbuf); \ + check_utf8(in, state); \ + \ + /* detect odd sequences of backslashes */ \ + uint64_t odd_ends = find_odd_backslash_sequences( \ + in, prev_iter_ends_odd_backslash); \ + \ + /* detect insides of quote pairs ("quote_mask") and also our quote_bits */ \ + /* themselves */ \ + uint64_t quote_bits; \ + uint64_t quote_mask = find_quote_mask_and_bits( \ + in, odd_ends, prev_iter_inside_quote, quote_bits, error_mask); \ + \ + /* take the previous iterations structural bits, not our current iteration, */ \ + /* and flatten */ \ + flatten_bits(base_ptr, base, idx, structurals); \ + \ + uint64_t whitespace; \ + find_whitespace_and_structurals(in, whitespace, structurals); \ + \ + /* fixup structurals to reflect quotes and add pseudo-structural characters */ \ + structurals = finalize_structurals(structurals, whitespace, quote_mask, \ + quote_bits, prev_iter_ends_pseudo_pred); \ + idx += 64; \ + } \ + \ + /* is last string quote closed? */ \ + if (prev_iter_inside_quote) { \ + return simdjson::UNCLOSED_STRING; \ + } \ + \ + /* finally, flatten out the remaining structurals from the last iteration */ \ + flatten_bits(base_ptr, base, idx, structurals); \ + \ + pj.n_structural_indexes = base; \ + /* a valid JSON file cannot have zero structural indexes - we should have */ \ + /* found something */ \ + if (pj.n_structural_indexes == 0u) { \ + return simdjson::EMPTY; \ + } \ + if (base_ptr[pj.n_structural_indexes - 1] > len) { \ + return simdjson::UNEXPECTED_ERROR; \ + } \ + if (len != base_ptr[pj.n_structural_indexes - 1]) { \ + /* the string might not be NULL terminated, but we add a virtual NULL ending */ \ + /* character. */ \ + base_ptr[pj.n_structural_indexes++] = len; \ + } \ + /* make it safe to dereference one beyond this array */ \ + base_ptr[pj.n_structural_indexes] = 0; \ + if (error_mask) { \ + return simdjson::UNESCAPED_CHARS; \ + } \ + return check_utf8_errors(state); \ +} - // take the previous iterations structural bits, not our current iteration, - // and flatten - flatten_bits(base_ptr, base, idx, structurals); - uint64_t whitespace; - find_whitespace_and_structurals(in, whitespace, structurals); +#endif // SIMDJSON_STAGE1_FIND_MARKS_MACROS_H +/* end file include/simdjson/stage1_find_marks_macros.h */ +/* begin file include/simdjson/stage1_find_marks_westmere.h */ +#ifndef SIMDJSON_STAGE1_FIND_MARKS_WESTMERE_H +#define SIMDJSON_STAGE1_FIND_MARKS_WESTMERE_H - // fixup structurals to reflect quotes and add pseudo-structural characters - structurals = finalize_structurals(structurals, whitespace, quote_mask, - quote_bits, prev_iter_ends_pseudo_pred); + +#ifdef IS_X86_64 + +TARGET_WESTMERE +namespace simdjson { +template<> +struct simd_input { + __m128i v0; + __m128i v1; + __m128i v2; + __m128i v3; +}; + +template<> really_inline +simd_input fill_input(const uint8_t * ptr) { + struct simd_input in; + in.v0 = _mm_loadu_si128(reinterpret_cast(ptr + 0)); + in.v1 = _mm_loadu_si128(reinterpret_cast(ptr + 16)); + in.v2 = _mm_loadu_si128(reinterpret_cast(ptr + 32)); + in.v3 = _mm_loadu_si128(reinterpret_cast(ptr + 48)); + return in; +} + +template<> really_inline +uint64_t compute_quote_mask(uint64_t quote_bits) { + return _mm_cvtsi128_si64(_mm_clmulepi64_si128( + _mm_set_epi64x(0ULL, quote_bits), _mm_set1_epi8(0xFF), 0)); +} + +template<> +struct utf8_checking_state +{ + __m128i has_error = _mm_setzero_si128(); + processed_utf_bytes previous { + _mm_setzero_si128(), // rawbytes + _mm_setzero_si128(), // high_nibbles + _mm_setzero_si128() // carried_continuations + }; +}; + +template<> really_inline +void check_utf8(simd_input in, + utf8_checking_state& state) { + __m128i highbit = _mm_set1_epi8(0x80); + if ((_mm_testz_si128(_mm_or_si128(in.v0, in.v1), highbit)) == 1) { + // it is ascii, we just check continuation + state.has_error = _mm_or_si128( + _mm_cmpgt_epi8( + state.previous.carried_continuations, + _mm_setr_epi8(9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 1)), + state.has_error); + } else { + // it is not ascii so we have to do heavy work + state.previous = checkUTF8Bytes(in.v0, &(state.previous), &(state.has_error)); + state.previous = checkUTF8Bytes(in.v1, &(state.previous), &(state.has_error)); } - //////////////// - /// we use a giant copy-paste which is ugly. - /// but otherwise the string needs to be properly padded or else we - /// risk invalidating the UTF-8 checks. - //////////// - if (idx < len) { - uint8_t tmpbuf[64]; - memset(tmpbuf, 0x20, 64); - memcpy(tmpbuf, buf + idx, len - idx); - simd_input in = fill_input(tmpbuf); -#ifdef SIMDJSON_UTF8VALIDATE - check_utf8(in, state); -#endif - - // detect odd sequences of backslashes - uint64_t odd_ends = find_odd_backslash_sequences( - in, prev_iter_ends_odd_backslash); - - // detect insides of quote pairs ("quote_mask") and also our quote_bits - // themselves - uint64_t quote_bits; - uint64_t quote_mask = find_quote_mask_and_bits( - in, odd_ends, prev_iter_inside_quote, quote_bits, error_mask); - - // take the previous iterations structural bits, not our current iteration, - // and flatten - flatten_bits(base_ptr, base, idx, structurals); - - uint64_t whitespace; - find_whitespace_and_structurals(in, whitespace, structurals); - - // fixup structurals to reflect quotes and add pseudo-structural characters - structurals = finalize_structurals(structurals, whitespace, quote_mask, - quote_bits, prev_iter_ends_pseudo_pred); - idx += 64; + if ((_mm_testz_si128(_mm_or_si128(in.v2, in.v3), highbit)) == 1) { + // it is ascii, we just check continuation + state.has_error = _mm_or_si128( + _mm_cmpgt_epi8( + state.previous.carried_continuations, + _mm_setr_epi8(9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 1)), + state.has_error); + } else { + // it is not ascii so we have to do heavy work + state.previous = checkUTF8Bytes(in.v2, &(state.previous), &(state.has_error)); + state.previous = checkUTF8Bytes(in.v3, &(state.previous), &(state.has_error)); } +} - // is last string quote closed? - if (prev_iter_inside_quote) { - return simdjson::UNCLOSED_STRING; - } +template<> really_inline +errorValues check_utf8_errors(utf8_checking_state& state) { + return _mm_testz_si128(state.has_error, state.has_error) == 0 ? simdjson::UTF8_ERROR : simdjson::SUCCESS; +} - // finally, flatten out the remaining structurals from the last iteration - flatten_bits(base_ptr, base, idx, structurals); +template<> really_inline +uint64_t cmp_mask_against_input(simd_input in, uint8_t m) { + const __m128i mask = _mm_set1_epi8(m); + __m128i cmp_res_0 = _mm_cmpeq_epi8(in.v0, mask); + uint64_t res_0 = _mm_movemask_epi8(cmp_res_0); + __m128i cmp_res_1 = _mm_cmpeq_epi8(in.v1, mask); + uint64_t res_1 = _mm_movemask_epi8(cmp_res_1); + __m128i cmp_res_2 = _mm_cmpeq_epi8(in.v2, mask); + uint64_t res_2 = _mm_movemask_epi8(cmp_res_2); + __m128i cmp_res_3 = _mm_cmpeq_epi8(in.v3, mask); + uint64_t res_3 = _mm_movemask_epi8(cmp_res_3); + return res_0 | (res_1 << 16) | (res_2 << 32) | (res_3 << 48); +} - pj.n_structural_indexes = base; - // a valid JSON file cannot have zero structural indexes - we should have - // found something - if (pj.n_structural_indexes == 0u) { - return simdjson::EMPTY; +template<> really_inline +uint64_t unsigned_lteq_against_input(simd_input in, uint8_t m) { + const __m128i maxval = _mm_set1_epi8(m); + __m128i cmp_res_0 = _mm_cmpeq_epi8(_mm_max_epu8(maxval,in.v0),maxval); + uint64_t res_0 = _mm_movemask_epi8(cmp_res_0); + __m128i cmp_res_1 = _mm_cmpeq_epi8(_mm_max_epu8(maxval,in.v1),maxval); + uint64_t res_1 = _mm_movemask_epi8(cmp_res_1); + __m128i cmp_res_2 = _mm_cmpeq_epi8(_mm_max_epu8(maxval,in.v2),maxval); + uint64_t res_2 = _mm_movemask_epi8(cmp_res_2); + __m128i cmp_res_3 = _mm_cmpeq_epi8(_mm_max_epu8(maxval,in.v3),maxval); + uint64_t res_3 = _mm_movemask_epi8(cmp_res_3); + return res_0 | (res_1 << 16) | (res_2 << 32) | (res_3 << 48); +} + +template<> really_inline +uint64_t find_odd_backslash_sequences(simd_input in, uint64_t &prev_iter_ends_odd_backslash) { + FIND_ODD_BACKSLASH_SEQUENCES(architecture::westmere, in, prev_iter_ends_odd_backslash); +} + +template<> really_inline +uint64_t find_quote_mask_and_bits(simd_input in, uint64_t odd_ends, + uint64_t &prev_iter_inside_quote, uint64_t "e_bits, uint64_t &error_mask) { + FIND_QUOTE_MASK_AND_BITS(architecture::westmere, in, odd_ends, prev_iter_inside_quote, quote_bits, error_mask) +} + +template<> really_inline +void find_whitespace_and_structurals(simd_input in, + uint64_t &whitespace, uint64_t &structurals) { + const __m128i structural_table = _mm_setr_epi8(44, 125, 0, 0, 0xc0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 123); + const __m128i white_table = _mm_setr_epi8( + 32, 100, 100, 100, 17, 100, 113, 2, 100, 9, 10, 112, 100, 13, 100, 100); + const __m128i struct_offset = _mm_set1_epi8(0xd4); + const __m128i struct_mask = _mm_set1_epi8(32); + + __m128i white0 = _mm_cmpeq_epi8(in.v0, + _mm_shuffle_epi8(white_table, in.v0)); + __m128i white1 = _mm_cmpeq_epi8(in.v1, + _mm_shuffle_epi8(white_table, in.v1)); + __m128i white2 = _mm_cmpeq_epi8(in.v2, + _mm_shuffle_epi8(white_table, in.v2)); + __m128i white3 = _mm_cmpeq_epi8(in.v3, + _mm_shuffle_epi8(white_table, in.v3)); + uint64_t ws_res_0 = _mm_movemask_epi8(white0); + uint64_t ws_res_1 = _mm_movemask_epi8(white1); + uint64_t ws_res_2 = _mm_movemask_epi8(white2); + uint64_t ws_res_3 = _mm_movemask_epi8(white3); + + whitespace = (ws_res_0 | (ws_res_1 << 16) | (ws_res_2 << 32) | (ws_res_3 << 48)); + + __m128i struct1_r1 = _mm_add_epi8(struct_offset, in.v0); + __m128i struct2_r1 = _mm_add_epi8(struct_offset, in.v1); + __m128i struct3_r1 = _mm_add_epi8(struct_offset, in.v2); + __m128i struct4_r1 = _mm_add_epi8(struct_offset, in.v3); + + __m128i struct1_r2 = _mm_or_si128(in.v0, struct_mask); + __m128i struct2_r2 = _mm_or_si128(in.v1, struct_mask); + __m128i struct3_r2 = _mm_or_si128(in.v2, struct_mask); + __m128i struct4_r2 = _mm_or_si128(in.v3, struct_mask); + + __m128i struct1_r3 = _mm_shuffle_epi8(structural_table, struct1_r1); + __m128i struct2_r3 = _mm_shuffle_epi8(structural_table, struct2_r1); + __m128i struct3_r3 = _mm_shuffle_epi8(structural_table, struct3_r1); + __m128i struct4_r3 = _mm_shuffle_epi8(structural_table, struct4_r1); + + __m128i struct1 = _mm_cmpeq_epi8(struct1_r2, struct1_r3); + __m128i struct2 = _mm_cmpeq_epi8(struct2_r2, struct2_r3); + __m128i struct3 = _mm_cmpeq_epi8(struct3_r2, struct3_r3); + __m128i struct4 = _mm_cmpeq_epi8(struct4_r2, struct4_r3); + + uint64_t structural_res_0 = _mm_movemask_epi8(struct1); + uint64_t structural_res_1 = _mm_movemask_epi8(struct2); + uint64_t structural_res_2 = _mm_movemask_epi8(struct3); + uint64_t structural_res_3 = _mm_movemask_epi8(struct4); + + structurals = (structural_res_0 | (structural_res_1 << 16) | (structural_res_2 << 32) | (structural_res_3 << 48)); +} + + +} // namespace simdjson +UNTARGET_REGION + + +#endif // IS_X86_64 +#endif // SIMDJSON_STAGE1_FIND_MARKS_WESTMERE_H +/* end file include/simdjson/stage1_find_marks_westmere.h */ +/* begin file include/simdjson/stage1_find_marks_haswell.h */ +#ifndef SIMDJSON_STAGE1_FIND_MARKS_HASWELL_H +#define SIMDJSON_STAGE1_FIND_MARKS_HASWELL_H + + +#ifdef IS_X86_64 + +TARGET_HASWELL +namespace simdjson { +template<> +struct simd_input { + __m256i lo; + __m256i hi; +}; + +template<> really_inline +simd_input fill_input(const uint8_t * ptr) { + struct simd_input in; + in.lo = _mm256_loadu_si256(reinterpret_cast(ptr + 0)); + in.hi = _mm256_loadu_si256(reinterpret_cast(ptr + 32)); + return in; +} + +template<> really_inline +uint64_t compute_quote_mask(uint64_t quote_bits) { + // There should be no such thing with a processing supporting avx2 + // but not clmul. + uint64_t quote_mask = _mm_cvtsi128_si64(_mm_clmulepi64_si128( + _mm_set_epi64x(0ULL, quote_bits), _mm_set1_epi8(0xFF), 0)); + return quote_mask; +} + +template<> +struct utf8_checking_state { + __m256i has_error; + avx_processed_utf_bytes previous; + utf8_checking_state() { + has_error = _mm256_setzero_si256(); + previous.rawbytes = _mm256_setzero_si256(); + previous.high_nibbles = _mm256_setzero_si256(); + previous.carried_continuations =_mm256_setzero_si256(); } - if (base_ptr[pj.n_structural_indexes - 1] > len) { - return simdjson::UNEXPECTED_ERROR; +}; + + +template<> really_inline +void check_utf8(simd_input in, + utf8_checking_state& state) { + __m256i highbit = _mm256_set1_epi8(0x80); + if ((_mm256_testz_si256(_mm256_or_si256(in.lo, in.hi), highbit)) == 1) { + // it is ascii, we just check continuation + state.has_error = _mm256_or_si256( + _mm256_cmpgt_epi8( + state.previous.carried_continuations, + _mm256_setr_epi8(9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 1)), + state.has_error); + } else { + // it is not ascii so we have to do heavy work + state.previous = avxcheckUTF8Bytes(in.lo, &(state.previous), &(state.has_error)); + state.previous = avxcheckUTF8Bytes(in.hi, &(state.previous), &(state.has_error)); } - if (len != base_ptr[pj.n_structural_indexes - 1]) { - // the string might not be NULL terminated, but we add a virtual NULL ending - // character. - base_ptr[pj.n_structural_indexes++] = len; - } - // make it safe to dereference one beyond this array - base_ptr[pj.n_structural_indexes] = 0; - if (error_mask) { - return simdjson::UNESCAPED_CHARS; - } -#ifdef SIMDJSON_UTF8VALIDATE - return check_utf8_errors(state); +} + +template<> really_inline +errorValues check_utf8_errors(utf8_checking_state& state) { + return _mm256_testz_si256(state.has_error, state.has_error) == 0 ? simdjson::UTF8_ERROR : simdjson::SUCCESS; +} + +template<> really_inline +uint64_t cmp_mask_against_input(simd_input in, uint8_t m) { + const __m256i mask = _mm256_set1_epi8(m); + __m256i cmp_res_0 = _mm256_cmpeq_epi8(in.lo, mask); + uint64_t res_0 = static_cast(_mm256_movemask_epi8(cmp_res_0)); + __m256i cmp_res_1 = _mm256_cmpeq_epi8(in.hi, mask); + uint64_t res_1 = _mm256_movemask_epi8(cmp_res_1); + return res_0 | (res_1 << 32); +} + +template<> really_inline +uint64_t unsigned_lteq_against_input(simd_input in, uint8_t m) { + const __m256i maxval = _mm256_set1_epi8(m); + __m256i cmp_res_0 = _mm256_cmpeq_epi8(_mm256_max_epu8(maxval,in.lo),maxval); + uint64_t res_0 = static_cast(_mm256_movemask_epi8(cmp_res_0)); + __m256i cmp_res_1 = _mm256_cmpeq_epi8(_mm256_max_epu8(maxval,in.hi),maxval); + uint64_t res_1 = _mm256_movemask_epi8(cmp_res_1); + return res_0 | (res_1 << 32); +} + +template<> really_inline +uint64_t find_odd_backslash_sequences(simd_input in, uint64_t &prev_iter_ends_odd_backslash) { + FIND_ODD_BACKSLASH_SEQUENCES(architecture::haswell, in, prev_iter_ends_odd_backslash); +} + +template<> really_inline +uint64_t find_quote_mask_and_bits(simd_input in, uint64_t odd_ends, + uint64_t &prev_iter_inside_quote, uint64_t "e_bits, uint64_t &error_mask) { + FIND_QUOTE_MASK_AND_BITS(architecture::haswell, in, odd_ends, prev_iter_inside_quote, quote_bits, error_mask) +} + +template<> really_inline +void find_whitespace_and_structurals(simd_input in, + uint64_t &whitespace, + uint64_t &structurals) { +#ifdef SIMDJSON_NAIVE_STRUCTURAL + // You should never need this naive approach, but it can be useful + // for research purposes + const __m256i mask_open_brace = _mm256_set1_epi8(0x7b); + __m256i struct_lo = _mm256_cmpeq_epi8(in.lo, mask_open_brace); + __m256i struct_hi = _mm256_cmpeq_epi8(in.hi, mask_open_brace); + const __m256i mask_close_brace = _mm256_set1_epi8(0x7d); + struct_lo = _mm256_or_si256(struct_lo,_mm256_cmpeq_epi8(in.lo, mask_close_brace)); + struct_hi = _mm256_or_si256(struct_hi,_mm256_cmpeq_epi8(in.hi, mask_close_brace)); + const __m256i mask_open_bracket = _mm256_set1_epi8(0x5b); + struct_lo = _mm256_or_si256(struct_lo,_mm256_cmpeq_epi8(in.lo, mask_open_bracket)); + struct_hi = _mm256_or_si256(struct_hi,_mm256_cmpeq_epi8(in.hi, mask_open_bracket)); + const __m256i mask_close_bracket = _mm256_set1_epi8(0x5d); + struct_lo = _mm256_or_si256(struct_lo,_mm256_cmpeq_epi8(in.lo, mask_close_bracket)); + struct_hi = _mm256_or_si256(struct_hi,_mm256_cmpeq_epi8(in.hi, mask_close_bracket)); + const __m256i mask_column = _mm256_set1_epi8(0x3a); + struct_lo = _mm256_or_si256(struct_lo,_mm256_cmpeq_epi8(in.lo, mask_column)); + struct_hi = _mm256_or_si256(struct_hi,_mm256_cmpeq_epi8(in.hi, mask_column)); + const __m256i mask_comma = _mm256_set1_epi8(0x2c); + struct_lo = _mm256_or_si256(struct_lo,_mm256_cmpeq_epi8(in.lo, mask_comma)); + struct_hi = _mm256_or_si256(struct_hi,_mm256_cmpeq_epi8(in.hi, mask_comma)); + uint64_t structural_res_0 = static_cast(_mm256_movemask_epi8(struct_lo)); + uint64_t structural_res_1 = _mm256_movemask_epi8(struct_hi); + structurals = (structural_res_0 | (structural_res_1 << 32)); + + const __m256i mask_space = _mm256_set1_epi8(0x20); + __m256i space_lo = _mm256_cmpeq_epi8(in.lo, mask_space); + __m256i space_hi = _mm256_cmpeq_epi8(in.hi, mask_space); + const __m256i mask_linefeed = _mm256_set1_epi8(0x0a); + space_lo = _mm256_or_si256(space_lo,_mm256_cmpeq_epi8(in.lo, mask_linefeed)); + space_hi = _mm256_or_si256(space_hi,_mm256_cmpeq_epi8(in.hi, mask_linefeed)); + const __m256i mask_tab = _mm256_set1_epi8(0x09); + space_lo = _mm256_or_si256(space_lo,_mm256_cmpeq_epi8(in.lo, mask_tab)); + space_hi = _mm256_or_si256(space_hi,_mm256_cmpeq_epi8(in.hi, mask_tab)); + const __m256i mask_carriage = _mm256_set1_epi8(0x0d); + space_lo = _mm256_or_si256(space_lo,_mm256_cmpeq_epi8(in.lo, mask_carriage)); + space_hi = _mm256_or_si256(space_hi,_mm256_cmpeq_epi8(in.hi, mask_carriage)); + + uint64_t ws_res_0 = static_cast(_mm256_movemask_epi8(space_lo)); + uint64_t ws_res_1 = _mm256_movemask_epi8(space_hi); + whitespace = (ws_res_0 | (ws_res_1 << 32)); + // end of naive approach + +#else // SIMDJSON_NAIVE_STRUCTURAL + const __m256i structural_table = _mm256_setr_epi8( + 44, 125, 0, 0, 0xc0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 123, + 44, 125, 0, 0, 0xc0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 123); + const __m256i white_table = _mm256_setr_epi8( + 32, 100, 100, 100, 17, 100, 113, 2, 100, 9, 10, 112, 100, 13, 100, 100, + 32, 100, 100, 100, 17, 100, 113, 2, 100, 9, 10, 112, 100, 13, 100, 100); + const __m256i struct_offset = _mm256_set1_epi8(0xd4); + const __m256i struct_mask = _mm256_set1_epi8(32); + + __m256i lo_white = _mm256_cmpeq_epi8(in.lo, + _mm256_shuffle_epi8(white_table, in.lo)); + __m256i hi_white = _mm256_cmpeq_epi8(in.hi, + _mm256_shuffle_epi8(white_table, in.hi)); + uint64_t ws_res_0 = static_cast(_mm256_movemask_epi8(lo_white)); + uint64_t ws_res_1 = _mm256_movemask_epi8(hi_white); + whitespace = (ws_res_0 | (ws_res_1 << 32)); + __m256i lo_struct_r1 = _mm256_add_epi8(struct_offset, in.lo); + __m256i hi_struct_r1 = _mm256_add_epi8(struct_offset, in.hi); + __m256i lo_struct_r2 = _mm256_or_si256(in.lo, struct_mask); + __m256i hi_struct_r2 = _mm256_or_si256(in.hi, struct_mask); + __m256i lo_struct_r3 = _mm256_shuffle_epi8(structural_table, lo_struct_r1); + __m256i hi_struct_r3 = _mm256_shuffle_epi8(structural_table, hi_struct_r1); + __m256i lo_struct = _mm256_cmpeq_epi8(lo_struct_r2, lo_struct_r3); + __m256i hi_struct = _mm256_cmpeq_epi8(hi_struct_r2, hi_struct_r3); + + uint64_t structural_res_0 = + static_cast(_mm256_movemask_epi8(lo_struct)); + uint64_t structural_res_1 = _mm256_movemask_epi8(hi_struct); + structurals = (structural_res_0 | (structural_res_1 << 32)); +#endif // SIMDJSON_NAIVE_STRUCTURAL +} + +} // namespace simdjson +UNTARGET_REGION + + +#endif // IS_X86_64 +#endif // SIMDJSON_STAGE1_FIND_MARKS_HASWELL_H +/* end file include/simdjson/stage1_find_marks_haswell.h */ +/* begin file include/simdjson/stage1_find_marks_arm64.h */ +#ifndef SIMDJSON_STAGE1_FIND_MARKS_ARM64_H +#define SIMDJSON_STAGE1_FIND_MARKS_ARM64_H + + +#ifdef IS_ARM64 +namespace simdjson { +template<> struct simd_input { + uint8x16_t i0; + uint8x16_t i1; + uint8x16_t i2; + uint8x16_t i3; +}; + +template<> really_inline +simd_input fill_input(const uint8_t * ptr) { + struct simd_input in; + in.i0 = vld1q_u8(ptr + 0); + in.i1 = vld1q_u8(ptr + 16); + in.i2 = vld1q_u8(ptr + 32); + in.i3 = vld1q_u8(ptr + 48); + return in; +} + + +really_inline +uint16_t neonmovemask(uint8x16_t input) { + const uint8x16_t bitmask = { 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80, + 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80}; + uint8x16_t minput = vandq_u8(input, bitmask); + uint8x16_t tmp = vpaddq_u8(minput, minput); + tmp = vpaddq_u8(tmp, tmp); + tmp = vpaddq_u8(tmp, tmp); + return vgetq_lane_u16(vreinterpretq_u16_u8(tmp), 0); +} + +really_inline +uint64_t neonmovemask_bulk(uint8x16_t p0, uint8x16_t p1, uint8x16_t p2, uint8x16_t p3) { + const uint8x16_t bitmask = { 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80, + 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80}; + uint8x16_t t0 = vandq_u8(p0, bitmask); + uint8x16_t t1 = vandq_u8(p1, bitmask); + uint8x16_t t2 = vandq_u8(p2, bitmask); + uint8x16_t t3 = vandq_u8(p3, bitmask); + uint8x16_t sum0 = vpaddq_u8(t0, t1); + uint8x16_t sum1 = vpaddq_u8(t2, t3); + sum0 = vpaddq_u8(sum0, sum1); + sum0 = vpaddq_u8(sum0, sum0); + return vgetq_lane_u64(vreinterpretq_u64_u8(sum0), 0); +} + +template<> really_inline +uint64_t compute_quote_mask(uint64_t quote_bits) { +#ifdef __ARM_FEATURE_CRYPTO // some ARM processors lack this extension + return vmull_p64( -1ULL, quote_bits); #else - return simdjson::SUCCESS; -#endif + return portable_compute_quote_mask(quote_bits); +#endif } -template -WARN_UNUSED -int find_structural_bits(const char *buf, size_t len, ParsedJson &pj) { - return find_structural_bits(reinterpret_cast(buf), len, pj); +template<> +struct utf8_checking_state +{ + int8x16_t has_error {}; + processed_utf_bytes previous {}; +}; + +// Checks that all bytes are ascii +really_inline +bool check_ascii_neon(simd_input in) { + // checking if the most significant bit is always equal to 0. + uint8x16_t highbit = vdupq_n_u8(0x80); + uint8x16_t t0 = vorrq_u8(in.i0, in.i1); + uint8x16_t t1 = vorrq_u8(in.i2, in.i3); + uint8x16_t t3 = vorrq_u8(t0, t1); + uint8x16_t t4 = vandq_u8(t3, highbit); + uint64x2_t v64 = vreinterpretq_u64_u8(t4); + uint32x2_t v32 = vqmovn_u64(v64); + uint64x1_t result = vreinterpret_u64_u32(v32); + return vget_lane_u64(result, 0) == 0; } + +template<> really_inline +void check_utf8(simd_input in, + utf8_checking_state& state) { + if (check_ascii_neon(in)) { + // All bytes are ascii. Therefore the byte that was just before must be ascii too. + // We only check the byte that was just before simd_input. Nines are arbitrary values. + const int8x16_t verror = (int8x16_t){9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 1}; + state.has_error = + vorrq_s8(vreinterpretq_s8_u8(vcgtq_s8(state.previous.carried_continuations, + verror)), + state.has_error); + } else { + // it is not ascii so we have to do heavy work + state.previous = checkUTF8Bytes(vreinterpretq_s8_u8(in.i0), &(state.previous), &(state.has_error)); + state.previous = checkUTF8Bytes(vreinterpretq_s8_u8(in.i1), &(state.previous), &(state.has_error)); + state.previous = checkUTF8Bytes(vreinterpretq_s8_u8(in.i2), &(state.previous), &(state.has_error)); + state.previous = checkUTF8Bytes(vreinterpretq_s8_u8(in.i3), &(state.previous), &(state.has_error)); + } } -#endif -/* end file include/simdjson/stage1_find_marks.h */ + +template<> really_inline +errorValues check_utf8_errors(utf8_checking_state& state) { + uint64x2_t v64 = vreinterpretq_u64_s8(state.has_error); + uint32x2_t v32 = vqmovn_u64(v64); + uint64x1_t result = vreinterpret_u64_u32(v32); + return vget_lane_u64(result, 0) != 0 ? simdjson::UTF8_ERROR : simdjson::SUCCESS; +} + +template<> really_inline +uint64_t cmp_mask_against_input(simd_input in, uint8_t m) { + const uint8x16_t mask = vmovq_n_u8(m); + uint8x16_t cmp_res_0 = vceqq_u8(in.i0, mask); + uint8x16_t cmp_res_1 = vceqq_u8(in.i1, mask); + uint8x16_t cmp_res_2 = vceqq_u8(in.i2, mask); + uint8x16_t cmp_res_3 = vceqq_u8(in.i3, mask); + return neonmovemask_bulk(cmp_res_0, cmp_res_1, cmp_res_2, cmp_res_3); +} + +template<> really_inline +uint64_t unsigned_lteq_against_input(simd_input in, uint8_t m) { + const uint8x16_t mask = vmovq_n_u8(m); + uint8x16_t cmp_res_0 = vcleq_u8(in.i0, mask); + uint8x16_t cmp_res_1 = vcleq_u8(in.i1, mask); + uint8x16_t cmp_res_2 = vcleq_u8(in.i2, mask); + uint8x16_t cmp_res_3 = vcleq_u8(in.i3, mask); + return neonmovemask_bulk(cmp_res_0, cmp_res_1, cmp_res_2, cmp_res_3); +} + +template<> really_inline +uint64_t find_odd_backslash_sequences(simd_input in, uint64_t &prev_iter_ends_odd_backslash) { + FIND_ODD_BACKSLASH_SEQUENCES(architecture::arm64, in, prev_iter_ends_odd_backslash); +} + +template<> really_inline +uint64_t find_quote_mask_and_bits(simd_input in, uint64_t odd_ends, + uint64_t &prev_iter_inside_quote, uint64_t "e_bits, uint64_t &error_mask) { + FIND_QUOTE_MASK_AND_BITS(architecture::arm64, in, odd_ends, prev_iter_inside_quote, quote_bits, error_mask) +} + +template<> really_inline +void find_whitespace_and_structurals( + simd_input in, + uint64_t &whitespace, + uint64_t &structurals) { + const uint8x16_t low_nibble_mask = (uint8x16_t){ + 16, 0, 0, 0, 0, 0, 0, 0, 0, 8, 12, 1, 2, 9, 0, 0}; + const uint8x16_t high_nibble_mask = (uint8x16_t){ + 8, 0, 18, 4, 0, 1, 0, 1, 0, 0, 0, 3, 2, 1, 0, 0}; + const uint8x16_t structural_shufti_mask = vmovq_n_u8(0x7); + const uint8x16_t whitespace_shufti_mask = vmovq_n_u8(0x18); + const uint8x16_t low_nib_and_mask = vmovq_n_u8(0xf); + + uint8x16_t nib_0_lo = vandq_u8(in.i0, low_nib_and_mask); + uint8x16_t nib_0_hi = vshrq_n_u8(in.i0, 4); + uint8x16_t shuf_0_lo = vqtbl1q_u8(low_nibble_mask, nib_0_lo); + uint8x16_t shuf_0_hi = vqtbl1q_u8(high_nibble_mask, nib_0_hi); + uint8x16_t v_0 = vandq_u8(shuf_0_lo, shuf_0_hi); + + uint8x16_t nib_1_lo = vandq_u8(in.i1, low_nib_and_mask); + uint8x16_t nib_1_hi = vshrq_n_u8(in.i1, 4); + uint8x16_t shuf_1_lo = vqtbl1q_u8(low_nibble_mask, nib_1_lo); + uint8x16_t shuf_1_hi = vqtbl1q_u8(high_nibble_mask, nib_1_hi); + uint8x16_t v_1 = vandq_u8(shuf_1_lo, shuf_1_hi); + + uint8x16_t nib_2_lo = vandq_u8(in.i2, low_nib_and_mask); + uint8x16_t nib_2_hi = vshrq_n_u8(in.i2, 4); + uint8x16_t shuf_2_lo = vqtbl1q_u8(low_nibble_mask, nib_2_lo); + uint8x16_t shuf_2_hi = vqtbl1q_u8(high_nibble_mask, nib_2_hi); + uint8x16_t v_2 = vandq_u8(shuf_2_lo, shuf_2_hi); + + uint8x16_t nib_3_lo = vandq_u8(in.i3, low_nib_and_mask); + uint8x16_t nib_3_hi = vshrq_n_u8(in.i3, 4); + uint8x16_t shuf_3_lo = vqtbl1q_u8(low_nibble_mask, nib_3_lo); + uint8x16_t shuf_3_hi = vqtbl1q_u8(high_nibble_mask, nib_3_hi); + uint8x16_t v_3 = vandq_u8(shuf_3_lo, shuf_3_hi); + + uint8x16_t tmp_0 = vtstq_u8(v_0, structural_shufti_mask); + uint8x16_t tmp_1 = vtstq_u8(v_1, structural_shufti_mask); + uint8x16_t tmp_2 = vtstq_u8(v_2, structural_shufti_mask); + uint8x16_t tmp_3 = vtstq_u8(v_3, structural_shufti_mask); + structurals = neonmovemask_bulk(tmp_0, tmp_1, tmp_2, tmp_3); + + uint8x16_t tmp_ws_0 = vtstq_u8(v_0, whitespace_shufti_mask); + uint8x16_t tmp_ws_1 = vtstq_u8(v_1, whitespace_shufti_mask); + uint8x16_t tmp_ws_2 = vtstq_u8(v_2, whitespace_shufti_mask); + uint8x16_t tmp_ws_3 = vtstq_u8(v_3, whitespace_shufti_mask); + whitespace = neonmovemask_bulk(tmp_ws_0, tmp_ws_1, tmp_ws_2, tmp_ws_3); +} +}// simdjson namespace + +#endif // IS_ARM64 +#endif // SIMDJSON_STAGE1_FIND_MARKS_ARM64_H +/* end file include/simdjson/stage1_find_marks_arm64.h */ /* begin file include/simdjson/stringparsing.h */ #ifndef SIMDJSON_STRINGPARSING_H #define SIMDJSON_STRINGPARSING_H @@ -38174,12 +38343,152 @@ struct parse_string_helper { }; // Finds where the backslashes and quotes are located. -template +template parse_string_helper find_bs_bits_and_quote_bits(const uint8_t *src, uint8_t *dst); -#ifdef __AVX2__ + + +template +WARN_UNUSED ALLOW_SAME_PAGE_BUFFER_OVERRUN_QUALIFIER LENIENT_MEM_SANITIZER really_inline +bool parse_string(UNUSED const uint8_t *buf, UNUSED size_t len, + ParsedJson &pj, UNUSED const uint32_t depth, UNUSED uint32_t offset); + + +} + +/// Now include the specializations: + +#endif +/* end file include/simdjson/stringparsing.h */ +/* begin file include/simdjson/stringparsing_macros.h */ +#ifndef SIMDJSON_STRINGPARSING_MACROS_H +#define SIMDJSON_STRINGPARSING_MACROS_H + +// We need to compile that code for multiple architectures. However, target attributes can be used +// only once by function definition. Huge macro seemed better than huge code duplication. +// bool PARSE_STRING(architecture T, UNUSED const uint8_t *buf, UNUSED size_t len, +// ParsedJson &pj, UNUSED const uint32_t depth, UNUSED uint32_t offset) +#define PARSE_STRING(T, buf, len, pj, depth, offset) { \ + pj.write_tape(pj.current_string_buf_loc - pj.string_buf, '"'); \ + const uint8_t *src = &buf[offset + 1]; /* we know that buf at offset is a " */ \ + uint8_t *dst = pj.current_string_buf_loc + sizeof(uint32_t); \ + const uint8_t *const start_of_string = dst; \ + while (1) { \ + parse_string_helper helper = find_bs_bits_and_quote_bits(src, dst); \ + if(((helper.bs_bits - 1) & helper.quote_bits) != 0 ) { \ + /* we encountered quotes first. Move dst to point to quotes and exit */ \ + \ + /* find out where the quote is... */ \ + uint32_t quote_dist = trailingzeroes(helper.quote_bits); \ + \ + /* NULL termination is still handy if you expect all your strings to be NULL terminated? */ \ + /* It comes at a small cost */ \ + dst[quote_dist] = 0; \ + \ + uint32_t str_length = (dst - start_of_string) + quote_dist; \ + memcpy(pj.current_string_buf_loc,&str_length, sizeof(uint32_t)); \ + /*///////////////////// */ \ + /* Above, check for overflow in case someone has a crazy string (>=4GB?) */ \ + /* But only add the overflow check when the document itself exceeds 4GB */ \ + /* Currently unneeded because we refuse to parse docs larger or equal to 4GB. */ \ + /*////////////////////// */ \ + \ + \ + /* we advance the point, accounting for the fact that we have a NULL termination */ \ + pj.current_string_buf_loc = dst + quote_dist + 1; \ + return true; \ + } \ + if(((helper.quote_bits - 1) & helper.bs_bits ) != 0 ) { \ + /* find out where the backspace is */ \ + uint32_t bs_dist = trailingzeroes(helper.bs_bits); \ + uint8_t escape_char = src[bs_dist + 1]; \ + /* we encountered backslash first. Handle backslash */ \ + if (escape_char == 'u') { \ + /* move src/dst up to the start; they will be further adjusted */ \ + /* within the unicode codepoint handling code. */ \ + src += bs_dist; \ + dst += bs_dist; \ + if (!handle_unicode_codepoint(&src, &dst)) { \ + return false; \ + } \ + } else { \ + /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and */ \ + /* write bs_dist+1 characters to output */ \ + /* note this may reach beyond the part of the buffer we've actually */ \ + /* seen. I think this is ok */ \ + uint8_t escape_result = escape_map[escape_char]; \ + if (escape_result == 0u) { \ + return false; /* bogus escape value is an error */ \ + } \ + dst[bs_dist] = escape_result; \ + src += bs_dist + 2; \ + dst += bs_dist + 1; \ + } \ + } else { \ + /* they are the same. Since they can't co-occur, it means we encountered */ \ + /* neither. */ \ + if constexpr(T == architecture::westmere) { \ + src += 16; \ + dst += 16; \ + } else { \ + src += 32; \ + dst += 32; \ + } \ + } \ + } \ + /* can't be reached */ \ + return true; \ +} + +#endif +/* end file include/simdjson/stringparsing_macros.h */ +/* begin file include/simdjson/stringparsing_westmere.h */ +#ifndef SIMDJSON_STRINGPARSING_WESTMERE_H +#define SIMDJSON_STRINGPARSING_WESTMERE_H + + + +#ifdef IS_X86_64 +TARGET_WESTMERE +namespace simdjson { template<> really_inline -parse_string_helper find_bs_bits_and_quote_bits (const uint8_t *src, uint8_t *dst) { +parse_string_helper find_bs_bits_and_quote_bits (const uint8_t *src, uint8_t *dst) { + // this can read up to 31 bytes beyond the buffer size, but we require + // SIMDJSON_PADDING of padding + __m128i v = _mm_loadu_si128(reinterpret_cast(src)); + // store to dest unconditionally - we can overwrite the bits we don't like + // later + _mm_storeu_si128(reinterpret_cast<__m128i *>(dst), v); + auto quote_mask = _mm_cmpeq_epi8(v, _mm_set1_epi8('"')); + return { + static_cast(_mm_movemask_epi8(_mm_cmpeq_epi8(v, _mm_set1_epi8('\\')))), // bs_bits + static_cast(_mm_movemask_epi8(quote_mask)) // quote_bits + }; +} + +template<> +WARN_UNUSED ALLOW_SAME_PAGE_BUFFER_OVERRUN_QUALIFIER LENIENT_MEM_SANITIZER really_inline +bool parse_string(UNUSED const uint8_t *buf, UNUSED size_t len, + ParsedJson &pj, UNUSED const uint32_t depth, UNUSED uint32_t offset) { + PARSE_STRING(architecture::westmere, buf, len, pj, depth, offset); +} +} +UNTARGET_REGION +#endif + +#endif +/* end file include/simdjson/stringparsing_westmere.h */ +/* begin file include/simdjson/stringparsing_haswell.h */ +#ifndef SIMDJSON_STRINGPARSING_HASWELL_H +#define SIMDJSON_STRINGPARSING_HASWELL_H + + + +#ifdef IS_X86_64 +TARGET_HASWELL +namespace simdjson { +template<> really_inline +parse_string_helper find_bs_bits_and_quote_bits (const uint8_t *src, uint8_t *dst) { // this can read up to 31 bytes beyond the buffer size, but we require // SIMDJSON_PADDING of padding static_assert(sizeof(__m256i) - 1 <= SIMDJSON_PADDING); @@ -38193,28 +38502,29 @@ parse_string_helper find_bs_bits_and_quote_bits (const ui static_cast(_mm256_movemask_epi8(quote_mask)) // quote_bits }; } -#endif -#if defined(__SSE4_2__) || (defined(_MSC_VER) && defined(_M_AMD64)) -template<> really_inline -parse_string_helper find_bs_bits_and_quote_bits (const uint8_t *src, uint8_t *dst) { - // this can read up to 31 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - __m128i v = _mm_loadu_si128(reinterpret_cast(src)); - // store to dest unconditionally - we can overwrite the bits we don't like - // later - _mm_storeu_si128(reinterpret_cast<__m128i *>(dst), v); - auto quote_mask = _mm_cmpeq_epi8(v, _mm_set1_epi8('"')); - return { - static_cast(_mm_movemask_epi8(_mm_cmpeq_epi8(v, _mm_set1_epi8('\\')))), // bs_bits - static_cast(_mm_movemask_epi8(quote_mask)) // quote_bits - }; +template<> +WARN_UNUSED ALLOW_SAME_PAGE_BUFFER_OVERRUN_QUALIFIER LENIENT_MEM_SANITIZER really_inline +bool parse_string(UNUSED const uint8_t *buf, UNUSED size_t len, + ParsedJson &pj, UNUSED const uint32_t depth, UNUSED uint32_t offset) { + PARSE_STRING(architecture::haswell, buf, len, pj, depth, offset); } + +} +UNTARGET_REGION #endif -#ifdef __ARM_NEON +#endif +/* end file include/simdjson/stringparsing_haswell.h */ +/* begin file include/simdjson/stringparsing_arm64.h */ +#ifndef SIMDJSON_STRINGPARSING_ARM64_H +#define SIMDJSON_STRINGPARSING_ARM64_H + + +#ifdef IS_ARM64 +namespace simdjson { template<> really_inline -parse_string_helper find_bs_bits_and_quote_bits (const uint8_t *src, uint8_t *dst) { +parse_string_helper find_bs_bits_and_quote_bits (const uint8_t *src, uint8_t *dst) { // this can read up to 31 bytes beyond the buffer size, but we require // SIMDJSON_PADDING of padding static_assert(2 * sizeof(uint8x16_t) - 1 <= SIMDJSON_PADDING); @@ -38246,101 +38556,17 @@ parse_string_helper find_bs_bits_and_quote_bits (const ui vgetq_lane_u32(vreinterpretq_u32_u8(sum0), 1) // quote_bits }; } -#endif -template -WARN_UNUSED ALLOW_SAME_PAGE_BUFFER_OVERRUN_QUALIFIER LENIENT_MEM_SANITIZER really_inline -bool parse_string(UNUSED const uint8_t *buf, UNUSED size_t len, +template<> +WARN_UNUSED ALLOW_SAME_PAGE_BUFFER_OVERRUN_QUALIFIER LENIENT_MEM_SANITIZER really_inline +bool parse_string(UNUSED const uint8_t *buf, UNUSED size_t len, ParsedJson &pj, UNUSED const uint32_t depth, UNUSED uint32_t offset) { -#ifdef SIMDJSON_SKIPSTRINGPARSING // for performance analysis, it is sometimes useful to skip parsing - pj.write_tape(0, '"');// don't bother with the string parsing at all - return true; // always succeeds -#else - pj.write_tape(pj.current_string_buf_loc - pj.string_buf, '"'); - const uint8_t *src = &buf[offset + 1]; // we know that buf at offset is a " - uint8_t *dst = pj.current_string_buf_loc + sizeof(uint32_t); - const uint8_t *const start_of_string = dst; - while (1) { - parse_string_helper helper = find_bs_bits_and_quote_bits(src, dst); - if(((helper.bs_bits - 1) & helper.quote_bits) != 0 ) { - // we encountered quotes first. Move dst to point to quotes and exit - - // find out where the quote is... - uint32_t quote_dist = trailingzeroes(helper.quote_bits); - - // NULL termination is still handy if you expect all your strings to be NULL terminated? - // It comes at a small cost - dst[quote_dist] = 0; - - uint32_t str_length = (dst - start_of_string) + quote_dist; - memcpy(pj.current_string_buf_loc,&str_length, sizeof(uint32_t)); - /////////////////////// - // Above, check for overflow in case someone has a crazy string (>=4GB?) - // But only add the overflow check when the document itself exceeds 4GB - // Currently unneeded because we refuse to parse docs larger or equal to 4GB. - //////////////////////// - - - // we advance the point, accounting for the fact that we have a NULL termination - pj.current_string_buf_loc = dst + quote_dist + 1; - -#ifdef JSON_TEST_STRINGS // for unit testing - foundString(buf + offset,start_of_string,pj.current_string_buf_loc - 1); -#endif // JSON_TEST_STRINGS - return true; - } - if(((helper.quote_bits - 1) & helper.bs_bits ) != 0 ) { - // find out where the backspace is - uint32_t bs_dist = trailingzeroes(helper.bs_bits); - uint8_t escape_char = src[bs_dist + 1]; - // we encountered backslash first. Handle backslash - if (escape_char == 'u') { - // move src/dst up to the start; they will be further adjusted - // within the unicode codepoint handling code. - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst)) { -#ifdef JSON_TEST_STRINGS // for unit testing - foundBadString(buf + offset); -#endif // JSON_TEST_STRINGS - return false; - } - } else { - // simple 1:1 conversion. Will eat bs_dist+2 characters in input and - // write bs_dist+1 characters to output - // note this may reach beyond the part of the buffer we've actually - // seen. I think this is ok - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { -#ifdef JSON_TEST_STRINGS // for unit testing - foundBadString(buf + offset); -#endif // JSON_TEST_STRINGS - return false; // bogus escape value is an error - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - // they are the same. Since they can't co-occur, it means we encountered - // neither. - if constexpr(T == instruction_set::sse4_2) { - src += 16; - dst += 16; - } else { - src += 32; - dst += 32; - } - } - } - // can't be reached - return true; -#endif // SIMDJSON_SKIPSTRINGPARSING + PARSE_STRING(architecture::arm64, buf, len, pj, depth, offset); } } - #endif -/* end file include/simdjson/stringparsing.h */ +#endif +/* end file include/simdjson/stringparsing_arm64.h */ /* begin file include/simdjson/numberparsing.h */ #ifndef SIMDJSON_NUMBERPARSING_H #define SIMDJSON_NUMBERPARSING_H @@ -38453,14 +38679,14 @@ really_inline bool is_not_structural_or_whitespace_or_exponent_or_decimal(unsigned char c) { return structural_or_whitespace_or_exponent_or_decimal_negated[c]; } - +}// simdjson #ifndef SIMDJSON_DISABLE_SWAR_NUMBER_PARSING -// #if defined (__AVX2__) || defined (__SSE4_2__) #define SWAR_NUMBER_PARSING #endif #ifdef SWAR_NUMBER_PARSING +namespace simdjson { // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html @@ -38478,9 +38704,10 @@ static inline bool is_made_of_eight_digits_fast(const char *chars) { (((val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0) >> 4)) == 0x3333333333333333); } - -#if defined (__AVX2__) || defined (__SSE4_2__) - +} +#ifdef IS_X86_64 +TARGET_WESTMERE +namespace simdjson { static inline uint32_t parse_eight_digits_unrolled(const char *chars) { // this actually computes *16* values so we are being wasteful. const __m128i ascii0 = _mm_set1_epi8('0'); @@ -38497,7 +38724,12 @@ static inline uint32_t parse_eight_digits_unrolled(const char *chars) { return _mm_cvtsi128_si32( t4); // only captures the sum of the first 8 digits, drop the rest } -#else +} +UNTARGET_REGION +#endif + +namespace simdjson { +#ifdef IS_ARM64 // we don't have SSE, so let us use a scalar function // credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/ static inline uint32_t parse_eight_digits_unrolled(const char *chars) { @@ -38507,9 +38739,9 @@ static inline uint32_t parse_eight_digits_unrolled(const char *chars) { val = (val & 0x00FF00FF00FF00FF) * 6553601 >> 16; return (val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32; } - - #endif + + #endif // @@ -38895,7 +39127,7 @@ static really_inline bool parse_number(const uint8_t *const buf, return is_structural_or_whitespace(*p); #endif // SIMDJSON_SKIPNUMBERPARSING } -} +}//simdjson #endif /* end file include/simdjson/numberparsing.h */ /* begin file include/simdjson/stage2_build_tape.h */ @@ -38963,519 +39195,15 @@ really_inline bool is_valid_null_atom(const uint8_t *loc) { return error == 0; } - -/************ - * The JSON is parsed to a tape, see the accompanying tape.md file - * for documentation. - ***********/ -template +template WARN_UNUSED ALLOW_SAME_PAGE_BUFFER_OVERRUN_QUALIFIER LENIENT_MEM_SANITIZER -int unified_machine(const uint8_t *buf, size_t len, ParsedJson &pj) { -#ifndef ALLOW_SAME_PAGE_BUFFER_OVERRUN - memset((uint8_t*)buf + len, 0, SIMDJSON_PADDING); // to please valgrind -#endif - uint32_t i = 0; // index of the structural character (0,1,2,3...) - uint32_t idx; // location of the structural character in the input (buf) - uint8_t c; // used to track the (structural) character we are looking at, updated - // by UPDATE_CHAR macro - uint32_t depth = 0; // could have an arbitrary starting depth - pj.init(); // sets isvalid to false - if(pj.bytecapacity < len) { - pj.errorcode = simdjson::CAPACITY; - return pj.errorcode; - } -// this macro reads the next structural character, updating idx, i and c. -#define UPDATE_CHAR() \ - { \ - idx = pj.structural_indexes[i++]; \ - c = buf[idx]; \ - } +int unified_machine(const uint8_t *buf, size_t len, ParsedJson &pj); - - ////////////////////////////// START STATE ///////////////////////////// -#ifdef SIMDJSON_USE_COMPUTED_GOTO - pj.ret_address[depth] = &&start_continue; -#else - pj.ret_address[depth] = 's'; -#endif - pj.containing_scope_offset[depth] = pj.get_current_loc(); - pj.write_tape(0, 'r'); // r for root, 0 is going to get overwritten - // the root is used, if nothing else, to capture the size of the tape - depth++; // everything starts at depth = 1, depth = 0 is just for the root, the root may contain an object, an array or something else. - if (depth >= pj.depthcapacity) { - goto fail; - } - - UPDATE_CHAR(); - switch (c) { - case '{': - pj.containing_scope_offset[depth] = pj.get_current_loc(); -#ifdef SIMDJSON_USE_COMPUTED_GOTO - pj.ret_address[depth] = &&start_continue; -#else - pj.ret_address[depth] = 's'; -#endif - depth++; - if (depth >= pj.depthcapacity) { - goto fail; - } - pj.write_tape(0, c); // strangely, moving this to object_begin slows things down - goto object_begin; - case '[': - pj.containing_scope_offset[depth] = pj.get_current_loc(); -#ifdef SIMDJSON_USE_COMPUTED_GOTO - pj.ret_address[depth] = &&start_continue; -#else - pj.ret_address[depth] = 's'; -#endif - depth++; - if (depth >= pj.depthcapacity) { - goto fail; - } - pj.write_tape(0, c); - goto array_begin; -#define SIMDJSON_ALLOWANYTHINGINROOT - // A JSON text is a serialized value. Note that certain previous - // specifications of JSON constrained a JSON text to be an object or an - // array. Implementations that generate only objects or arrays where a - // JSON text is called for will be interoperable in the sense that all - // implementations will accept these as conforming JSON texts. - // https://tools.ietf.org/html/rfc8259 -#ifdef SIMDJSON_ALLOWANYTHINGINROOT - case '"': { - if (!parse_string(buf, len, pj, depth, idx)) { - goto fail; - } - break; - } - case 't': { - // we need to make a copy to make sure that the string is space terminated. - // this only applies to the JSON document made solely of the true value. - // this will almost never be called in practice - char * copy = static_cast(malloc(len + SIMDJSON_PADDING)); - if(copy == nullptr) { - goto fail; - } - memcpy(copy, buf, len); - copy[len] = ' '; - if (!is_valid_true_atom(reinterpret_cast(copy) + idx)) { - free(copy); - goto fail; - } - free(copy); - pj.write_tape(0, c); - break; - } - case 'f': { - // we need to make a copy to make sure that the string is space terminated. - // this only applies to the JSON document made solely of the false value. - // this will almost never be called in practice - char * copy = static_cast(malloc(len + SIMDJSON_PADDING)); - if(copy == nullptr) { - goto fail; - } - memcpy(copy, buf, len); - copy[len] = ' '; - if (!is_valid_false_atom(reinterpret_cast(copy) + idx)) { - free(copy); - goto fail; - } - free(copy); - pj.write_tape(0, c); - break; - } - case 'n': { - // we need to make a copy to make sure that the string is space terminated. - // this only applies to the JSON document made solely of the null value. - // this will almost never be called in practice - char * copy = static_cast(malloc(len + SIMDJSON_PADDING)); - if(copy == nullptr) { - goto fail; - } - memcpy(copy, buf, len); - copy[len] = ' '; - if (!is_valid_null_atom(reinterpret_cast(copy) + idx)) { - free(copy); - goto fail; - } - free(copy); - pj.write_tape(0, c); - break; - } - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': { - // we need to make a copy to make sure that the string is space terminated. - // this is done only for JSON documents made of a sole number - // this will almost never be called in practice. We terminate with a space - // because we do not want to allow NULLs in the middle of a number (whereas a - // space in the middle of a number would be identified in stage 1). - char * copy = static_cast(malloc(len + SIMDJSON_PADDING)); - if(copy == nullptr) { - goto fail; - } - memcpy(copy, buf, len); - copy[len] = ' '; - if (!parse_number(reinterpret_cast(copy), pj, idx, false)) { - free(copy); - goto fail; - } - free(copy); - break; - } - case '-': { - // we need to make a copy to make sure that the string is NULL terminated. - // this is done only for JSON documents made of a sole number - // this will almost never be called in practice - char * copy = static_cast(malloc(len + SIMDJSON_PADDING)); - if(copy == nullptr) { - goto fail; - } - memcpy(copy, buf, len); - copy[len] = '\0'; - if (!parse_number(reinterpret_cast(copy), pj, idx, true)) { - free(copy); - goto fail; - } - free(copy); - break; - } -#endif // ALLOWANYTHINGINROOT - default: - goto fail; - } -start_continue: - // the string might not be NULL terminated. - if(i + 1 == pj.n_structural_indexes) { - goto succeed; - } else { - goto fail; - } - ////////////////////////////// OBJECT STATES ///////////////////////////// - -object_begin: - UPDATE_CHAR(); - switch (c) { - case '"': { - if (!parse_string(buf, len, pj, depth, idx)) { - goto fail; - } - goto object_key_state; - } - case '}': - goto scope_end; // could also go to object_continue - default: - goto fail; - } - -object_key_state: - UPDATE_CHAR(); - if (c != ':') { - goto fail; - } - UPDATE_CHAR(); - switch (c) { - case '"': { - if (!parse_string(buf, len, pj, depth, idx)) { - goto fail; - } - break; - } - case 't': - if (!is_valid_true_atom(buf + idx)) { - goto fail; - } - pj.write_tape(0, c); - break; - case 'f': - if (!is_valid_false_atom(buf + idx)) { - goto fail; - } - pj.write_tape(0, c); - break; - case 'n': - if (!is_valid_null_atom(buf + idx)) { - goto fail; - } - pj.write_tape(0, c); - break; - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': { - if (!parse_number(buf, pj, idx, false)) { - goto fail; - } - break; - } - case '-': { - if (!parse_number(buf, pj, idx, true)) { - goto fail; - } - break; - } - case '{': { - pj.containing_scope_offset[depth] = pj.get_current_loc(); - pj.write_tape(0, c); // here the compilers knows what c is so this gets optimized - // we have not yet encountered } so we need to come back for it -#ifdef SIMDJSON_USE_COMPUTED_GOTO - pj.ret_address[depth] = &&object_continue; -#else - pj.ret_address[depth] = 'o'; -#endif - // we found an object inside an object, so we need to increment the depth - depth++; - if (depth >= pj.depthcapacity) { - goto fail; - } - - goto object_begin; - } - case '[': { - pj.containing_scope_offset[depth] = pj.get_current_loc(); - pj.write_tape(0, c); // here the compilers knows what c is so this gets optimized - // we have not yet encountered } so we need to come back for it -#ifdef SIMDJSON_USE_COMPUTED_GOTO - pj.ret_address[depth] = &&object_continue; -#else - pj.ret_address[depth] = 'o'; -#endif - // we found an array inside an object, so we need to increment the depth - depth++; - if (depth >= pj.depthcapacity) { - goto fail; - } - goto array_begin; - } - default: - goto fail; - } - -object_continue: - UPDATE_CHAR(); - switch (c) { - case ',': - UPDATE_CHAR(); - if (c != '"') { - goto fail; - } else { - if (!parse_string(buf, len, pj, depth, idx)) { - goto fail; - } - goto object_key_state; - } - case '}': - goto scope_end; - default: - goto fail; - } - - ////////////////////////////// COMMON STATE ///////////////////////////// - -scope_end: - // write our tape location to the header scope - depth--; - pj.write_tape(pj.containing_scope_offset[depth], c); - pj.annotate_previousloc(pj.containing_scope_offset[depth], - pj.get_current_loc()); - // goto saved_state -#ifdef SIMDJSON_USE_COMPUTED_GOTO - goto *pj.ret_address[depth]; -#else - if(pj.ret_address[depth] == 'a') { - goto array_continue; - } else if (pj.ret_address[depth] == 'o') { - goto object_continue; - } else goto start_continue; -#endif - - ////////////////////////////// ARRAY STATES ///////////////////////////// -array_begin: - UPDATE_CHAR(); - if (c == ']') { - goto scope_end; // could also go to array_continue - } - -main_array_switch: - // we call update char on all paths in, so we can peek at c on the - // on paths that can accept a close square brace (post-, and at start) - switch (c) { - case '"': { - if (!parse_string(buf, len, pj, depth, idx)) { - goto fail; - } - break; - } - case 't': - if (!is_valid_true_atom(buf + idx)) { - goto fail; - } - pj.write_tape(0, c); - break; - case 'f': - if (!is_valid_false_atom(buf + idx)) { - goto fail; - } - pj.write_tape(0, c); - break; - case 'n': - if (!is_valid_null_atom(buf + idx)) { - goto fail; - } - pj.write_tape(0, c); - break; // goto array_continue; - - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': { - if (!parse_number(buf, pj, idx, false)) { - goto fail; - } - break; // goto array_continue; - } - case '-': { - if (!parse_number(buf, pj, idx, true)) { - goto fail; - } - break; // goto array_continue; - } - case '{': { - // we have not yet encountered ] so we need to come back for it - pj.containing_scope_offset[depth] = pj.get_current_loc(); - pj.write_tape(0, c); // here the compilers knows what c is so this gets optimized -#ifdef SIMDJSON_USE_COMPUTED_GOTO - pj.ret_address[depth] = &&array_continue; -#else - pj.ret_address[depth] = 'a'; -#endif - // we found an object inside an array, so we need to increment the depth - depth++; - if (depth >= pj.depthcapacity) { - goto fail; - } - - goto object_begin; - } - case '[': { - // we have not yet encountered ] so we need to come back for it - pj.containing_scope_offset[depth] = pj.get_current_loc(); - pj.write_tape(0, c); // here the compilers knows what c is so this gets optimized -#ifdef SIMDJSON_USE_COMPUTED_GOTO - pj.ret_address[depth] = &&array_continue; -#else - pj.ret_address[depth] = 'a'; -#endif - // we found an array inside an array, so we need to increment the depth - depth++; - if (depth >= pj.depthcapacity) { - goto fail; - } - goto array_begin; - } - default: - goto fail; - } - -array_continue: - UPDATE_CHAR(); - switch (c) { - case ',': - UPDATE_CHAR(); - goto main_array_switch; - case ']': - goto scope_end; - default: - goto fail; - } - - ////////////////////////////// FINAL STATES ///////////////////////////// - -succeed: - depth --; - if(depth != 0) { - fprintf(stderr, "internal bug\n"); - abort(); - } - if(pj.containing_scope_offset[depth] != 0) { - fprintf(stderr, "internal bug\n"); - abort(); - } - pj.annotate_previousloc(pj.containing_scope_offset[depth], - pj.get_current_loc()); - pj.write_tape(pj.containing_scope_offset[depth], 'r'); // r is root - - pj.isvalid = true; - pj.errorcode = simdjson::SUCCESS; - return pj.errorcode; -fail: - // we do not need the next line because this is done by pj.init(), pessimistically. - // pj.isvalid = false; - // At this point in the code, we have all the time in the world. - // Note that we know exactly where we are in the document so we could, - // without any overhead on the processing code, report a specific location. - // We could even trigger special code paths to assess what happened carefully, - // all without any added cost. - if (depth >= pj.depthcapacity) { - pj.errorcode = simdjson::DEPTH_ERROR; - return pj.errorcode; - } - switch(c) { - case '"': - pj.errorcode = simdjson::STRING_ERROR; - return pj.errorcode; - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': - case '-': - pj.errorcode = simdjson::NUMBER_ERROR; - return pj.errorcode; - case 't': - pj.errorcode = simdjson::T_ATOM_ERROR; - return pj.errorcode; - case 'n': - pj.errorcode = simdjson::N_ATOM_ERROR; - return pj.errorcode; - case 'f': - pj.errorcode = simdjson::F_ATOM_ERROR; - return pj.errorcode; - default: - break; - } - pj.errorcode = simdjson::TAPE_ERROR; - return pj.errorcode; -} - -template +template int unified_machine(const char *buf, size_t len, ParsedJson &pj) { return unified_machine(reinterpret_cast(buf), len, pj); } + } #endif @@ -39503,9 +39231,9 @@ using json_parse_functype = int (const uint8_t *buf, size_t len, ParsedJson &pj, extern json_parse_functype *json_parse_ptr; // json_parse_implementation is the generic function, it is specialized for various -// SIMD instruction sets, e.g., as json_parse_implementation -// or json_parse_implementation -template +// architectures, e.g., as json_parse_implementation +// or json_parse_implementation +template int json_parse_implementation(const uint8_t *buf, size_t len, ParsedJson &pj, bool reallocifneeded = true) { if (pj.bytecapacity < len) { return simdjson::CAPACITY; @@ -39539,7 +39267,7 @@ int json_parse_implementation(const uint8_t *buf, size_t len, ParsedJson &pj, bo reallocated = true; } // if (true) OR if ( (reinterpret_cast(buf + len - 1) % pagesize ) + SIMDJSON_PADDING < static_cast(pagesize) ) { } // if(reallocifneeded) { - int stage1_is_ok = find_structural_bits(buf, len, pj); + int stage1_is_ok = simdjson::find_structural_bits(buf, len, pj); if(stage1_is_ok != simdjson::SUCCESS) { pj.errorcode = stage1_is_ok; return pj.errorcode; diff --git a/src/jsonminifier.cpp b/src/jsonminifier.cpp index f62d3a8d4..f8baf5f2d 100644 --- a/src/jsonminifier.cpp +++ b/src/jsonminifier.cpp @@ -63,6 +63,30 @@ size_t jsonminify(const unsigned char *bytes, size_t howmany, #include namespace simdjson { + + +// some intrinsics are missing under GCC? +#ifndef __clang__ +#ifndef _MSC_VER +static __m256i inline _mm256_loadu2_m128i(__m128i const *__addr_hi, + __m128i const *__addr_lo) { + __m256i __v256 = _mm256_castsi128_si256(_mm_loadu_si128(__addr_lo)); + return _mm256_insertf128_si256(__v256, _mm_loadu_si128(__addr_hi), 1); +} + +static inline void _mm256_storeu2_m128i(__m128i *__addr_hi, __m128i *__addr_lo, + __m256i __a) { + __m128i __v128; + __v128 = _mm256_castsi256_si128(__a); + _mm_storeu_si128(__addr_lo, __v128); + __v128 = _mm256_extractf128_si256(__a, 1); + _mm_storeu_si128(__addr_hi, __v128); +} +#endif +#endif + + + // a straightforward comparison of a mask against input. static uint64_t cmp_mask_against_input_mini(__m256i input_lo, __m256i input_hi, __m256i mask) { diff --git a/src/jsonparser.cpp b/src/jsonparser.cpp index 296028277..d3b0b60d7 100644 --- a/src/jsonparser.cpp +++ b/src/jsonparser.cpp @@ -6,52 +6,45 @@ #include #endif #include "simdjson/simdjson.h" +#include "simdjson/isadetection.h" +#include "simdjson/portability.h" namespace simdjson { + +architecture find_best_supported_implementation() { + constexpr uint32_t haswell_flags = SIMDExtensions::AVX2 | SIMDExtensions::PCLMULQDQ + | SIMDExtensions::BMI1 | SIMDExtensions::BMI2; + constexpr uint32_t westmere_flags = SIMDExtensions::SSE42 | SIMDExtensions::PCLMULQDQ; + + uint32_t supports = detect_supported_architectures(); + // Order from best to worst (within architecture) + if ((haswell_flags & supports) == haswell_flags) return architecture::haswell; + if ((westmere_flags & supports) == westmere_flags) return architecture::westmere; + if (SIMDExtensions::NEON) return architecture::arm64; + + return architecture::none; +} + // Responsible to select the best json_parse implementation int json_parse_dispatch(const uint8_t *buf, size_t len, ParsedJson &pj, bool reallocifneeded) { - // Versions for each implementation -#ifdef __AVX2__ - json_parse_functype* avx_implementation = &json_parse_implementation; -#endif -#if defined(__SSE4_2__) || (defined(_MSC_VER) && defined(_M_AMD64)) - json_parse_functype* sse4_2_implementation = &json_parse_implementation; -#endif -#if defined(__ARM_NEON) || (defined(_MSC_VER) && defined(_M_ARM64)) - json_parse_functype* neon_implementation = &json_parse_implementation; -#endif - - // Determining which implementation is the more suitable - // Should be done at runtime. Does not make any sense on preprocessor. -#ifdef __AVX2__ - instruction_set best_implementation = instruction_set::avx2; -#elif defined (__SSE4_2__) || (defined(_MSC_VER) && defined(_M_AMD64)) - instruction_set best_implementation = instruction_set::sse4_2; -#elif defined (__ARM_NEON) || (defined(_MSC_VER) && defined(_M_ARM64)) - instruction_set best_implementation = instruction_set::neon; -#else - instruction_set best_implementation = instruction_set::none; -#endif - + architecture best_implementation = find_best_supported_implementation(); // Selecting the best implementation switch (best_implementation) { -#ifdef __AVX2__ - case instruction_set::avx2 : - json_parse_ptr = avx_implementation; +#ifdef IS_X86_64 + case architecture::haswell: + json_parse_ptr = &json_parse_implementation; + break; + case architecture::westmere: + json_parse_ptr = &json_parse_implementation; break; #endif -#if defined(__SSE4_2__) || (defined(_MSC_VER) && defined(_M_AMD64)) - case instruction_set::sse4_2 : - json_parse_ptr = sse4_2_implementation; - break; -#endif -#if defined(__ARM_NEON) || (defined(_MSC_VER) && defined(_M_ARM64)) - case instruction_set::neon : - json_parse_ptr = neon_implementation; +#ifdef IS_ARM64 + case architecture::arm64: + json_parse_ptr = &json_parse_implementation; break; #endif default : - std::cerr << "No implemented simd instruction set supported" << std::endl; + std::cerr << "The processor is not supported by simdjson." << std::endl; return simdjson::UNEXPECTED_ERROR; } diff --git a/src/stage1_find_marks.cpp b/src/stage1_find_marks.cpp index e0ee94cf9..0e593f7e5 100644 --- a/src/stage1_find_marks.cpp +++ b/src/stage1_find_marks.cpp @@ -1 +1,37 @@ -// File kept in case we want to reuse it soon. (many configuration files to edit) +#include "simdjson/portability.h" + + +#ifdef IS_X86_64 + +#include "simdjson/stage1_find_marks_haswell.h" +#include "simdjson/stage1_find_marks_westmere.h" +TARGET_HASWELL +namespace simdjson { +template<> +int find_structural_bits(const uint8_t *buf, size_t len, ParsedJson &pj) { + FIND_STRUCTURAL_BITS(architecture::haswell, buf, len, pj); +} +} // simdjson +UNTARGET_REGION + +TARGET_WESTMERE +namespace simdjson { +template<> +int find_structural_bits(const uint8_t *buf, size_t len, ParsedJson &pj) { + FIND_STRUCTURAL_BITS(architecture::westmere, buf, len, pj); +} +} // simdjson +UNTARGET_REGION + +#endif + + +#ifdef IS_ARM64 +#include "simdjson/stage1_find_marks_arm64.h" +namespace simdjson { +template<> +int find_structural_bits(const uint8_t *buf, size_t len, ParsedJson &pj) { + FIND_STRUCTURAL_BITS(architecture::arm64, buf, len, pj); +} +} +#endif diff --git a/src/stage2_build_tape.cpp b/src/stage2_build_tape.cpp index e0ee94cf9..2c333d70b 100644 --- a/src/stage2_build_tape.cpp +++ b/src/stage2_build_tape.cpp @@ -1 +1,534 @@ -// File kept in case we want to reuse it soon. (many configuration files to edit) +#include "simdjson/stage2_build_tape.h" + +namespace simdjson { + +// this macro reads the next structural character, updating idx, i and c. +#define UPDATE_CHAR() \ +{ \ + idx = pj.structural_indexes[i++]; \ + c = buf[idx]; \ +} + +#ifdef SIMDJSON_USE_COMPUTED_GOTO +#define SET_GOTO_ARRAY_CONTINUE() pj.ret_address[depth] = &&array_continue; +#define SET_GOTO_OBJECT_CONTINUE() pj.ret_address[depth] = &&object_continue; +#define SET_GOTO_START_CONTINUE() pj.ret_address[depth] = &&start_continue; +#define GOTO_CONTINUE() goto *pj.ret_address[depth]; +#else +#define SET_GOTO_ARRAY_CONTINUE() pj.ret_address[depth] = 'a'; +#define SET_GOTO_OBJECT_CONTINUE() pj.ret_address[depth] = 'o'; +#define SET_GOTO_START_CONTINUE() pj.ret_address[depth] = 's'; +#define GOTO_CONTINUE() { \ + if(pj.ret_address[depth] == 'a') { \ + goto array_continue; \ + } else if (pj.ret_address[depth] == 'o') { \ + goto object_continue; \ + } else { \ + goto start_continue; \ + } \ +} +#endif + +/************ + * The JSON is parsed to a tape, see the accompanying tape.md file + * for documentation. + ***********/ +// We need to compile that code for multiple architectures. However, target attributes can be used +// only once by function definition. Huge macro seemed better than huge code duplication. +// int UNIFIED_MACHINE(const uint8_t *buf, size_t len, ParsedJson &pj) +#define UNIFIED_MACHINE(T, buf, len, pj) { \ + if (ALLOW_SAME_PAGE_BUFFER_OVERRUN) { \ + memset((uint8_t*)buf + len, 0, SIMDJSON_PADDING); /* to please valgrind */ \ + } \ + uint32_t i = 0; /* index of the structural character (0,1,2,3...) */ \ + uint32_t idx; /* location of the structural character in the input (buf) */ \ + uint8_t c; /* used to track the (structural) character we are looking at, updated */ \ + /* by UPDATE_CHAR macro */ \ + uint32_t depth = 0; /* could have an arbitrary starting depth */ \ + pj.init(); /* sets isvalid to false */ \ + if(pj.bytecapacity < len) { \ + pj.errorcode = simdjson::CAPACITY; \ + return pj.errorcode; \ + } \ + \ + /*//////////////////////////// START STATE ///////////////////////////// */ \ + SET_GOTO_START_CONTINUE() \ + pj.containing_scope_offset[depth] = pj.get_current_loc(); \ + pj.write_tape(0, 'r'); /* r for root, 0 is going to get overwritten */ \ + /* the root is used, if nothing else, to capture the size of the tape */ \ + depth++; /* everything starts at depth = 1, depth = 0 is just for the root, the root may contain an object, an array or something else. */ \ + if (depth >= pj.depthcapacity) { \ + goto fail; \ + } \ + \ + UPDATE_CHAR(); \ + switch (c) { \ + case '{': \ + pj.containing_scope_offset[depth] = pj.get_current_loc(); \ + SET_GOTO_START_CONTINUE(); \ + depth++; \ + if (depth >= pj.depthcapacity) { \ + goto fail; \ + } \ + pj.write_tape(0, c); /* strangely, moving this to object_begin slows things down */ \ + goto object_begin; \ + case '[': \ + pj.containing_scope_offset[depth] = pj.get_current_loc(); \ + SET_GOTO_START_CONTINUE(); \ + depth++; \ + if (depth >= pj.depthcapacity) { \ + goto fail; \ + } \ + pj.write_tape(0, c); \ + goto array_begin; \ +/* #define SIMDJSON_ALLOWANYTHINGINROOT */ \ + /* A JSON text is a serialized value. Note that certain previous */ \ + /* specifications of JSON constrained a JSON text to be an object or an */ \ + /* array. Implementations that generate only objects or arrays where a */ \ + /* JSON text is called for will be interoperable in the sense that all */ \ + /* implementations will accept these as conforming JSON texts. */ \ + /* https://tools.ietf.org/html/rfc8259 */ \ +/* #ifdef SIMDJSON_ALLOWANYTHINGINROOT */ \ + case '"': { \ + if (!parse_string(buf, len, pj, depth, idx)) { \ + goto fail; \ + } \ + break; \ + } \ + case 't': { \ + /* we need to make a copy to make sure that the string is space terminated. */ \ + /* this only applies to the JSON document made solely of the true value. */ \ + /* this will almost never be called in practice */ \ + char * copy = static_cast(malloc(len + SIMDJSON_PADDING)); \ + if(copy == nullptr) { \ + goto fail; \ + } \ + memcpy(copy, buf, len); \ + copy[len] = ' '; \ + if (!is_valid_true_atom(reinterpret_cast(copy) + idx)) { \ + free(copy); \ + goto fail; \ + } \ + free(copy); \ + pj.write_tape(0, c); \ + break; \ + } \ + case 'f': { \ + /* we need to make a copy to make sure that the string is space terminated. */ \ + /* this only applies to the JSON document made solely of the false value. */ \ + /* this will almost never be called in practice */ \ + char * copy = static_cast(malloc(len + SIMDJSON_PADDING)); \ + if(copy == nullptr) { \ + goto fail; \ + } \ + memcpy(copy, buf, len); \ + copy[len] = ' '; \ + if (!is_valid_false_atom(reinterpret_cast(copy) + idx)) { \ + free(copy); \ + goto fail; \ + } \ + free(copy); \ + pj.write_tape(0, c); \ + break; \ + } \ + case 'n': { \ + /* we need to make a copy to make sure that the string is space terminated. */ \ + /* this only applies to the JSON document made solely of the null value. */ \ + /* this will almost never be called in practice */ \ + char * copy = static_cast(malloc(len + SIMDJSON_PADDING)); \ + if(copy == nullptr) { \ + goto fail; \ + } \ + memcpy(copy, buf, len); \ + copy[len] = ' '; \ + if (!is_valid_null_atom(reinterpret_cast(copy) + idx)) { \ + free(copy); \ + goto fail; \ + } \ + free(copy); \ + pj.write_tape(0, c); \ + break; \ + } \ + case '0': \ + case '1': \ + case '2': \ + case '3': \ + case '4': \ + case '5': \ + case '6': \ + case '7': \ + case '8': \ + case '9': { \ + /* we need to make a copy to make sure that the string is space terminated. */ \ + /* this is done only for JSON documents made of a sole number */ \ + /* this will almost never be called in practice. We terminate with a space */ \ + /* because we do not want to allow NULLs in the middle of a number (whereas a */ \ + /* space in the middle of a number would be identified in stage 1). */ \ + char * copy = static_cast(malloc(len + SIMDJSON_PADDING)); \ + if(copy == nullptr) { \ + goto fail; \ + } \ + memcpy(copy, buf, len); \ + copy[len] = ' '; \ + if (!parse_number(reinterpret_cast(copy), pj, idx, false)) { \ + free(copy); \ + goto fail; \ + } \ + free(copy); \ + break; \ + } \ + case '-': { \ + /* we need to make a copy to make sure that the string is NULL terminated. */ \ + /* this is done only for JSON documents made of a sole number */ \ + /* this will almost never be called in practice */ \ + char * copy = static_cast(malloc(len + SIMDJSON_PADDING)); \ + if(copy == nullptr) { \ + goto fail; \ + } \ + memcpy(copy, buf, len); \ + copy[len] = ' '; \ + if (!parse_number(reinterpret_cast(copy), pj, idx, true)) { \ + free(copy); \ + goto fail; \ + } \ + free(copy); \ + break; \ + } \ +/* #endif // ALLOWANYTHINGINROOT */ \ + default: \ + goto fail; \ + } \ +start_continue: \ + /* the string might not be NULL terminated. */ \ + if(i + 1 == pj.n_structural_indexes) { \ + goto succeed; \ + } else { \ + goto fail; \ + } \ + /*//////////////////////////// OBJECT STATES ///////////////////////////// */ \ + \ +object_begin: \ + UPDATE_CHAR(); \ + switch (c) { \ + case '"': { \ + if (!parse_string(buf, len, pj, depth, idx)) { \ + goto fail; \ + } \ + goto object_key_state; \ + } \ + case '}': \ + goto scope_end; /* could also go to object_continue */ \ + default: \ + goto fail; \ + } \ + \ +object_key_state: \ + UPDATE_CHAR(); \ + if (c != ':') { \ + goto fail; \ + } \ + UPDATE_CHAR(); \ + switch (c) { \ + case '"': { \ + if (!parse_string(buf, len, pj, depth, idx)) { \ + goto fail; \ + } \ + break; \ + } \ + case 't': \ + if (!is_valid_true_atom(buf + idx)) { \ + goto fail; \ + } \ + pj.write_tape(0, c); \ + break; \ + case 'f': \ + if (!is_valid_false_atom(buf + idx)) { \ + goto fail; \ + } \ + pj.write_tape(0, c); \ + break; \ + case 'n': \ + if (!is_valid_null_atom(buf + idx)) { \ + goto fail; \ + } \ + pj.write_tape(0, c); \ + break; \ + case '0': \ + case '1': \ + case '2': \ + case '3': \ + case '4': \ + case '5': \ + case '6': \ + case '7': \ + case '8': \ + case '9': { \ + if (!parse_number(buf, pj, idx, false)) { \ + goto fail; \ + } \ + break; \ + } \ + case '-': { \ + if (!parse_number(buf, pj, idx, true)) { \ + goto fail; \ + } \ + break; \ + } \ + case '{': { \ + pj.containing_scope_offset[depth] = pj.get_current_loc(); \ + pj.write_tape(0, c); /* here the compilers knows what c is so this gets optimized */ \ + /* we have not yet encountered } so we need to come back for it */ \ + SET_GOTO_OBJECT_CONTINUE() \ + /* we found an object inside an object, so we need to increment the depth */ \ + depth++; \ + if (depth >= pj.depthcapacity) { \ + goto fail; \ + } \ + \ + goto object_begin; \ + } \ + case '[': { \ + pj.containing_scope_offset[depth] = pj.get_current_loc(); \ + pj.write_tape(0, c); /* here the compilers knows what c is so this gets optimized */ \ + /* we have not yet encountered } so we need to come back for it */ \ + SET_GOTO_OBJECT_CONTINUE() \ + /* we found an array inside an object, so we need to increment the depth */ \ + depth++; \ + if (depth >= pj.depthcapacity) { \ + goto fail; \ + } \ + goto array_begin; \ + } \ + default: \ + goto fail; \ + } \ + \ +object_continue: \ + UPDATE_CHAR(); \ + switch (c) { \ + case ',': \ + UPDATE_CHAR(); \ + if (c != '"') { \ + goto fail; \ + } else { \ + if (!parse_string(buf, len, pj, depth, idx)) { \ + goto fail; \ + } \ + goto object_key_state; \ + } \ + case '}': \ + goto scope_end; \ + default: \ + goto fail; \ + } \ + \ + /*//////////////////////////// COMMON STATE ///////////////////////////// */ \ + \ +scope_end: \ + /* write our tape location to the header scope */ \ + depth--; \ + pj.write_tape(pj.containing_scope_offset[depth], c); \ + pj.annotate_previousloc(pj.containing_scope_offset[depth], \ + pj.get_current_loc()); \ + /* goto saved_state */ \ + GOTO_CONTINUE() \ + \ + /*//////////////////////////// ARRAY STATES ///////////////////////////// */ \ +array_begin: \ + UPDATE_CHAR(); \ + if (c == ']') { \ + goto scope_end; /* could also go to array_continue */ \ + } \ + \ +main_array_switch: \ + /* we call update char on all paths in, so we can peek at c on the */ \ + /* on paths that can accept a close square brace (post-, and at start) */ \ + switch (c) { \ + case '"': { \ + if (!parse_string(buf, len, pj, depth, idx)) { \ + goto fail; \ + } \ + break; \ + } \ + case 't': \ + if (!is_valid_true_atom(buf + idx)) { \ + goto fail; \ + } \ + pj.write_tape(0, c); \ + break; \ + case 'f': \ + if (!is_valid_false_atom(buf + idx)) { \ + goto fail; \ + } \ + pj.write_tape(0, c); \ + break; \ + case 'n': \ + if (!is_valid_null_atom(buf + idx)) { \ + goto fail; \ + } \ + pj.write_tape(0, c); \ + break; /* goto array_continue; */ \ + \ + case '0': \ + case '1': \ + case '2': \ + case '3': \ + case '4': \ + case '5': \ + case '6': \ + case '7': \ + case '8': \ + case '9': { \ + if (!parse_number(buf, pj, idx, false)) { \ + goto fail; \ + } \ + break; /* goto array_continue; */ \ + } \ + case '-': { \ + if (!parse_number(buf, pj, idx, true)) { \ + goto fail; \ + } \ + break; /* goto array_continue; */ \ + } \ + case '{': { \ + /* we have not yet encountered ] so we need to come back for it */ \ + pj.containing_scope_offset[depth] = pj.get_current_loc(); \ + pj.write_tape(0, c); /* here the compilers knows what c is so this gets optimized */ \ + SET_GOTO_ARRAY_CONTINUE() \ + /* we found an object inside an array, so we need to increment the depth */ \ + depth++; \ + if (depth >= pj.depthcapacity) { \ + goto fail; \ + } \ + \ + goto object_begin; \ + } \ + case '[': { \ + /* we have not yet encountered ] so we need to come back for it */ \ + pj.containing_scope_offset[depth] = pj.get_current_loc(); \ + pj.write_tape(0, c); /* here the compilers knows what c is so this gets optimized */ \ + SET_GOTO_ARRAY_CONTINUE() \ + /* we found an array inside an array, so we need to increment the depth */ \ + depth++; \ + if (depth >= pj.depthcapacity) { \ + goto fail; \ + } \ + goto array_begin; \ + } \ + default: \ + goto fail; \ + } \ + \ +array_continue: \ + UPDATE_CHAR(); \ + switch (c) { \ + case ',': \ + UPDATE_CHAR(); \ + goto main_array_switch; \ + case ']': \ + goto scope_end; \ + default: \ + goto fail; \ + } \ + \ + /*//////////////////////////// FINAL STATES ///////////////////////////// */ \ + \ +succeed: \ + depth --; \ + if(depth != 0) { \ + fprintf(stderr, "internal bug\n"); \ + abort(); \ + } \ + if(pj.containing_scope_offset[depth] != 0) { \ + fprintf(stderr, "internal bug\n"); \ + abort(); \ + } \ + pj.annotate_previousloc(pj.containing_scope_offset[depth], \ + pj.get_current_loc()); \ + pj.write_tape(pj.containing_scope_offset[depth], 'r'); /* r is root */ \ + \ + pj.isvalid = true; \ + pj.errorcode = simdjson::SUCCESS; \ + return pj.errorcode; \ +fail: \ + /* we do not need the next line because this is done by pj.init(), pessimistically. */ \ + /* pj.isvalid = false; */ \ + /* At this point in the code, we have all the time in the world. */ \ + /* Note that we know exactly where we are in the document so we could, */ \ + /* without any overhead on the processing code, report a specific location. */ \ + /* We could even trigger special code paths to assess what happened carefully, */ \ + /* all without any added cost. */ \ + if (depth >= pj.depthcapacity) { \ + pj.errorcode = simdjson::DEPTH_ERROR; \ + return pj.errorcode; \ + } \ + switch(c) { \ + case '"': \ + pj.errorcode = simdjson::STRING_ERROR; \ + return pj.errorcode; \ + case '0': \ + case '1': \ + case '2': \ + case '3': \ + case '4': \ + case '5': \ + case '6': \ + case '7': \ + case '8': \ + case '9': \ + case '-': \ + pj.errorcode = simdjson::NUMBER_ERROR; \ + return pj.errorcode; \ + case 't': \ + pj.errorcode = simdjson::T_ATOM_ERROR; \ + return pj.errorcode; \ + case 'n': \ + pj.errorcode = simdjson::N_ATOM_ERROR; \ + return pj.errorcode; \ + case 'f': \ + pj.errorcode = simdjson::F_ATOM_ERROR; \ + return pj.errorcode; \ + default: \ + break; \ + } \ + pj.errorcode = simdjson::TAPE_ERROR; \ + return pj.errorcode; \ +} \ + + +} + + +#ifdef IS_X86_64 +TARGET_HASWELL +namespace simdjson { +template<> +WARN_UNUSED ALLOW_SAME_PAGE_BUFFER_OVERRUN_QUALIFIER LENIENT_MEM_SANITIZER +int unified_machine(const uint8_t *buf, size_t len, ParsedJson &pj) { + UNIFIED_MACHINE(architecture::haswell, buf, len, pj); +} +} +UNTARGET_REGION + +TARGET_WESTMERE +namespace simdjson { +template<> +WARN_UNUSED ALLOW_SAME_PAGE_BUFFER_OVERRUN_QUALIFIER LENIENT_MEM_SANITIZER +int unified_machine(const uint8_t *buf, size_t len, ParsedJson &pj) { + UNIFIED_MACHINE(architecture::westmere, buf, len, pj); +} +} +UNTARGET_REGION +#endif // IS_X86_64 + +#ifdef IS_ARM64 +namespace simdjson { +template<> +WARN_UNUSED ALLOW_SAME_PAGE_BUFFER_OVERRUN_QUALIFIER LENIENT_MEM_SANITIZER +int unified_machine(const uint8_t *buf, size_t len, ParsedJson &pj) { + UNIFIED_MACHINE(architecture::arm64, buf, len, pj); +} +} +#endif + + diff --git a/tools/cmake/FindOptions.cmake b/tools/cmake/FindOptions.cmake index 89f2c6111..45096f226 100644 --- a/tools/cmake/FindOptions.cmake +++ b/tools/cmake/FindOptions.cmake @@ -22,9 +22,9 @@ if(SIMDJSON_DISABLE_AVX) else() # some compilers like clang do not automagically define __AVX2__ and __BMI2__ even when the hardware supports it if(NOT MSVC) - set (OPT_FLAGS "${OPT_FLAGS} -mavx2 -mbmi -mbmi2 -mpclmul") + #set (OPT_FLAGS "${OPT_FLAGS} -mpclmul") else() - set (OPT_FLAGS "${OPT_FLAGS} /arch:AVX2") + #set (OPT_FLAGS "${OPT_FLAGS} /arch:AVX2") endif() endif()