diff --git a/.drone.yml b/.drone.yml index df3e9b7a2..90236f481 100644 --- a/.drone.yml +++ b/.drone.yml @@ -8,7 +8,7 @@ steps: CC: gcc CXX: g++ BUILD_FLAGS: -- -j - CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback + CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=icelake;haswell;westmere;fallback CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly commands: - echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list @@ -19,6 +19,7 @@ steps: - cmake $CMAKE_FLAGS .. - cmake --build . $BUILD_FLAGS - ctest $CTEST_FLAGS -L acceptance -LE per_implementation + - SIMDJSON_FORCE_IMPLEMENTATION=icelake ctest $CTEST_FLAGS -L per_implementation - SIMDJSON_FORCE_IMPLEMENTATION=haswell ctest $CTEST_FLAGS -L per_implementation - SIMDJSON_FORCE_IMPLEMENTATION=westmere ctest $CTEST_FLAGS -L per_implementation - SIMDJSON_FORCE_IMPLEMENTATION=fallback ctest $CTEST_FLAGS -L per_implementation @@ -35,7 +36,7 @@ steps: CC: clang-6.0 CXX: clang++-6.0 BUILD_FLAGS: -- -j - CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback + CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=icelake;haswell;westmere;fallback CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly commands: - mkdir build @@ -43,6 +44,7 @@ steps: - cmake $CMAKE_FLAGS .. - cmake --build . $BUILD_FLAGS - ctest $CTEST_FLAGS -L acceptance -LE per_implementation + - SIMDJSON_FORCE_IMPLEMENTATION=icelake ctest $CTEST_FLAGS -L per_implementation - SIMDJSON_FORCE_IMPLEMENTATION=haswell ctest $CTEST_FLAGS -L per_implementation - SIMDJSON_FORCE_IMPLEMENTATION=westmere ctest $CTEST_FLAGS -L per_implementation - SIMDJSON_FORCE_IMPLEMENTATION=fallback ctest $CTEST_FLAGS -L per_implementation @@ -100,7 +102,7 @@ steps: CC: gcc CXX: g++ BUILD_FLAGS: -- -j - CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback + CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=icelake;haswell;westmere;fallback CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly commands: - echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list @@ -111,6 +113,7 @@ steps: - cmake $CMAKE_FLAGS .. - cmake --build . $BUILD_FLAGS - ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L acceptance -LE per_implementation + - SIMDJSON_FORCE_IMPLEMENTATION=icelake ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation - SIMDJSON_FORCE_IMPLEMENTATION=haswell ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation - SIMDJSON_FORCE_IMPLEMENTATION=westmere ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation - SIMDJSON_FORCE_IMPLEMENTATION=fallback ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation @@ -126,7 +129,7 @@ steps: environment: CC: clang-9 CXX: clang++-9 - CMAKE_FLAGS: -DSIMDJSON_SANITIZE=ON -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback + CMAKE_FLAGS: -DSIMDJSON_SANITIZE=ON -DSIMDJSON_IMPLEMENTATION=icelake;haswell;westmere;fallback BUILD_FLAGS: -- -j CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly commands: @@ -135,6 +138,7 @@ steps: - cmake $CMAKE_FLAGS .. - cmake --build . $BUILD_FLAGS - ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L acceptance -LE per_implementation + - SIMDJSON_FORCE_IMPLEMENTATION=icelake ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation - SIMDJSON_FORCE_IMPLEMENTATION=haswell ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation - SIMDJSON_FORCE_IMPLEMENTATION=westmere ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation - SIMDJSON_FORCE_IMPLEMENTATION=fallback ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation diff --git a/.github/workflows/msys2-clang.yml b/.github/workflows/msys2-clang.yml index dda9ed615..20a08f4cd 100644 --- a/.github/workflows/msys2-clang.yml +++ b/.github/workflows/msys2-clang.yml @@ -38,6 +38,6 @@ jobs: run: | mkdir build cd build - cmake -DSIMDJSON_DEVELOPER_MODE=ON -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_BUILD_TYPE=${{ matrix.type }} -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_DO_NOT_USE_THREADS_NO_MATTER_WHAT=ON .. + cmake -DSIMDJSON_DEVELOPER_MODE=ON -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_BUILD_TYPE=${{ matrix.type }} -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_DO_NOT_USE_THREADS_NO_MATTER_WHAT=ON .. cmake --build . --verbose ctest -j4 --output-on-failure -LE explicitonly diff --git a/.github/workflows/ubuntu22.yml b/.github/workflows/ubuntu22.yml new file mode 100644 index 000000000..d7677f67d --- /dev/null +++ b/.github/workflows/ubuntu22.yml @@ -0,0 +1,33 @@ +name: Ubuntu 22.04 CI (GCC 11) + +on: [push, pull_request] + +jobs: + ubuntu-build: + if: >- + ! contains(toJSON(github.event.commits.*.message), '[skip ci]') && + ! contains(toJSON(github.event.commits.*.message), '[skip github]') + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v2 + - uses: actions/cache@v2 + with: + path: dependencies/.cache + key: ${{ hashFiles('dependencies/CMakeLists.txt') }} + - name: Use cmake + run: | + mkdir builddebug && + cd builddebug && + cmake -DCMAKE_BUILD_TYPE=Debug -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF .. && + cmake --build . && + ctest -j --output-on-failure -LE explicitonly && + cd .. && + mkdir build && + cd build && + cmake -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination .. && + cmake --build . && + ctest -j --output-on-failure -LE explicitonly && + cmake --install . && + echo -e '#include \nint main(int argc,char**argv) {simdjson::dom::parser parser;simdjson::dom::element tweets = parser.load(argv[1]); }' > tmp.cpp && c++ -Idestination/include -Ldestination/lib -std=c++17 -Wl,-rpath,destination/lib -o linkandrun tmp.cpp -lsimdjson && ./linkandrun jsonexamples/twitter.json && + cd ../tests/installation_tests/find && + mkdir build && cd build && cmake -DCMAKE_INSTALL_PREFIX:PATH=../../../build/destination .. && cmake --build . diff --git a/.github/workflows/vs17-ci.yml b/.github/workflows/vs17-ci.yml index eaabbcfb7..ef2d27826 100644 --- a/.github/workflows/vs17-ci.yml +++ b/.github/workflows/vs17-ci.yml @@ -13,16 +13,16 @@ jobs: fail-fast: false matrix: include: - - {gen: Visual Studio 17 2022, arch: Win32, static: ON} - - {gen: Visual Studio 17 2022, arch: Win32, static: OFF} - - {gen: Visual Studio 17 2022, arch: x64, static: ON} - - {gen: Visual Studio 17 2022, arch: x64, static: OFF} + - {gen: Visual Studio 17 2022, arch: Win32, shared: ON} + - {gen: Visual Studio 17 2022, arch: Win32, shared: OFF} + - {gen: Visual Studio 17 2022, arch: x64, shared: ON} + - {gen: Visual Studio 17 2022, arch: x64, shared: OFF} steps: - name: checkout uses: actions/checkout@v2 - name: Configure run: | - cmake -G "${{matrix.gen}}" -A ${{matrix.arch}} -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_BUILD_STATIC=${{matrix.static}} -B build + cmake -G "${{matrix.gen}}" -A ${{matrix.arch}} -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -DBUILD_SHARED_LIBS=${{matrix.shared}} -B build - name: Build Debug run: cmake --build build --config Debug --verbose - name: Build Release diff --git a/.github/workflows/vs17-clang-ci.yml b/.github/workflows/vs17-clang-ci.yml index 5cca0d494..5da062765 100644 --- a/.github/workflows/vs17-clang-ci.yml +++ b/.github/workflows/vs17-clang-ci.yml @@ -9,23 +9,33 @@ jobs: ! contains(toJSON(github.event.commits.*.message), '[skip github]') name: windows-vs17 runs-on: windows-latest + strategy: + fail-fast: false + matrix: + include: + - {gen: Visual Studio 17 2022, arch: x64} steps: - - uses: actions/checkout@v2 - - uses: actions/cache@v2 - with: - path: dependencies/.cache - key: ${{ hashFiles('dependencies/CMakeLists.txt') }} - - name: 'Run CMake with VS17 Clang' - uses: lukka/run-cmake@v3 - with: - cmakeListsOrSettingsJson: CMakeListsTxtAdvanced - cmakeListsTxtPath: '${{ github.workspace }}/CMakeLists.txt' - buildDirectory: "${{ github.workspace }}/../../_temp/windows" - cmakeBuildType: Release - buildWithCMake: true - cmakeAppendedArgs: -T ClangCL -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF - buildWithCMakeArgs: --config Release - - - name: 'Run CTest' - run: ctest -C Release -LE explicitonly --output-on-failure - working-directory: "${{ github.workspace }}/../../_temp/windows" \ No newline at end of file + - name: checkout + uses: actions/checkout@v2 + - name: Configure + run: | + cmake -G "${{matrix.gen}}" -A ${{matrix.arch}} -T ClangCL -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -B build + - name: Build Debug + run: cmake --build build --config Debug --verbose + - name: Build Release + run: cmake --build build --config Release --verbose + - name: Run Release tests + run: | + cd build + ctest -C Release -LE explicitonly --output-on-failure + - name: Run Debug tests + run: | + cd build + ctest -C Debug -LE explicitonly --output-on-failure + - name: Install + run: | + cmake --install build --config Release + - name: Test Installation + run: | + cmake -G "${{matrix.gen}}" -A ${{matrix.arch}} -B build_install_test tests/installation_tests/find + cmake --build build_install_test --config Release \ No newline at end of file diff --git a/.github/workflows/vs17-ninja-ci.yml b/.github/workflows/vs17-ninja-ci.yml deleted file mode 100644 index 7fdcfbc91..000000000 --- a/.github/workflows/vs17-ninja-ci.yml +++ /dev/null @@ -1,45 +0,0 @@ -name: VS17-Ninja-CI - -on: [push, pull_request] - -jobs: - ci: - if: >- - ! contains(toJSON(github.event.commits.*.message), '[skip ci]') && - ! contains(toJSON(github.event.commits.*.message), '[skip github]') - name: windows-vs17 - runs-on: windows-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/cache@v2 - with: - path: dependencies/.cache - key: ${{ hashFiles('dependencies/CMakeLists.txt') }} - - name: 'Run CMake with VS17' - uses: lukka/run-cmake@v2 - with: - cmakeListsOrSettingsJson: CMakeListsTxtAdvanced - cmakeListsTxtPath: '${{ github.workspace }}/CMakeLists.txt' - buildDirectory: "${{ github.workspace }}/../../_temp/windows" - cmakeBuildType: Release - buildWithCMake: true - cmakeAppendedArgs: -G Ninja -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF - buildWithCMakeArgs: --config Release - - name: 'Run CTest' - run: ctest -C Release -LE explicitonly --output-on-failure - working-directory: "${{ github.workspace }}/../../_temp/windows" - - name: 'Install with CMake' - uses: lukka/run-cmake@v3 - with: - cmakeListsTxtPath: '${{ github.workspace }}/CMakeLists.txt' - buildWithCMakeArgs: '--target install' - - name: 'Test Installation with CMake' - uses: lukka/run-cmake@v3 - with: - cmakeListsOrSettingsJson: CMakeListsTxtAdvanced - cmakeListsTxtPath: '${{ github.workspace }}/tests/installation_tests/find/CMakeLists.txt' - cmakeBuildType: Release - buildWithCMake: true - buildDirectory: '${{ github.workspace }}/tests/installation_tests/find/buildDirectory' - cmakeAppendedArgs: -G Ninja - buildWithCMakeArgs: '--config Release --verbose' diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 8d6065438..0955cc2bd 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -38,5 +38,8 @@ Brendan Knapp Danila Kutenin Pavel Pavlov Hao Chen -# if you have contributed to the project and your name does not +Nicolas Boyer +Kim Walisch and Jatin Bhateja (AVX-512 bitset decoder) +Eric Zhang (AVX-512 kernel) +# if you have contributed to the project and your name does not # appear in this list, please let us know! diff --git a/benchmark/dom/statisticalmodel.cpp b/benchmark/dom/statisticalmodel.cpp index afc81df2e..677876f59 100644 --- a/benchmark/dom/statisticalmodel.cpp +++ b/benchmark/dom/statisticalmodel.cpp @@ -50,9 +50,9 @@ simdjson_really_inline void simdjson_process_atom(stat_t &s, } else if(element.is()) { s.float_count++; } else if (element.is()) { - simdjson::error_code err; bool v; - err = element.get(v); + simdjson::error_code error; + if ((error = element.get(v))) { std::cerr << error << std::endl; abort(); } if (v) { s.true_count++; } else { diff --git a/cmake/developer-options.cmake b/cmake/developer-options.cmake index c79aae3b3..fb421c955 100644 --- a/cmake/developer-options.cmake +++ b/cmake/developer-options.cmake @@ -175,6 +175,18 @@ if(CMAKE_C_COMPILER_ID MATCHES "Intel") set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -static-intel") endif() +option( + SIMDJSON_AVX512_ALLOWED + "Enable AVX-512 instructions (only affects processors and compilers with AVX-512 support)." + ON +) +if(SIMDJSON_AVX512_ALLOWED) + add_compile_definitions(SIMDJSON_AVX512_ALLOWED=1) +else() + add_compile_definitions(SIMDJSON_AVX512_ALLOWED=0) + message(STATUS "AVX-512 instructions are not allowed.") +endif() + include(CheckSymbolExists) check_symbol_exists(fork unistd.h HAVE_POSIX_FORK) check_symbol_exists(wait sys/wait.h HAVE_POSIX_WAIT) diff --git a/cmake/implementation-flags.cmake b/cmake/implementation-flags.cmake index d40869f18..88e937bf2 100644 --- a/cmake/implementation-flags.cmake +++ b/cmake/implementation-flags.cmake @@ -1,7 +1,7 @@ # # Implementation selection # -set(SIMDJSON_ALL_IMPLEMENTATIONS fallback westmere haswell arm64 ppc64) +set(SIMDJSON_ALL_IMPLEMENTATIONS fallback westmere haswell icelake arm64 ppc64) set( SIMDJSON_IMPLEMENTATION "" @@ -15,7 +15,7 @@ set( SIMDJSON_EXCLUDE_IMPLEMENTATION "" CACHE STRING "\ Semicolon-separated list of implementations to exclude \ -(haswell/westmere/arm64/ppc64/fallback). By default, excludes any \ +(icelake/haswell/westmere/arm64/ppc64/fallback). By default, excludes any \ implementations that are unsupported at compile time or cannot be selected at \ runtime." ) diff --git a/doc/performance.md b/doc/performance.md index 79a71ea21..ea1ced120 100644 --- a/doc/performance.md +++ b/doc/performance.md @@ -160,6 +160,8 @@ You should not expect the simdjson library to cause *downclocking* of your recen - [Whenever 512-bit AVX-512 instructions are used](https://lemire.me/blog/2018/09/07/avx-512-when-and-how-to-use-these-new-instructions/). - Whenever heavy 256-bit or wider instructions are used. Heavy instructions are those involving floating point operations or integer multiplications (since these execute on the floating point unit). -The simdjson library does not currently support AVX-512 instructions and it does not make use of heavy 256-bit instructions. We do use vectorized multiplications, but only using 128-bit registers. Thus there should be no downclocking due to simdjson on recent processors. +The simdjson library does not make use of heavy 256-bit instructions. We do use vectorized multiplications, but only using 128-bit registers. Thus there should be no downclocking due to simdjson on recent processors, except when AVX-512 is allowed and +detected. However, we only allow AVX-512 on recent processors (Ice Lake/Tiger Lake or better) where [little to no frequency throttling is expected](https://travisdowns.github.io/blog/2020/08/19/icl-avx512-freq.html). If you can still concerned, you can easily disable AVX-512 with the CMake option `SIMDJSON_AVX512_ALLOWED` set to `OFF` (e.g., `cmake -D SIMDJSON_AVX512_ALLOWED=OFF -B build && cmake --build build`) or by setting +the macro `SIMDJSON_AVX512_ALLOWED` to `0` in C++ prior to importing the headers. You may still be worried about which SIMD instruction set is used by simdjson. Thankfully, [you can always determine and change which architecture-specific implementation is used](implementation-selection.md) by simdjson. Thus even if your CPU supports AVX2, you do not need to use AVX2. You are in control. diff --git a/include/simdjson/builtin.h b/include/simdjson/builtin.h index 2e0e14561..c0c04a557 100644 --- a/include/simdjson/builtin.h +++ b/include/simdjson/builtin.h @@ -5,7 +5,9 @@ // Determine the best builtin implementation #ifndef SIMDJSON_BUILTIN_IMPLEMENTATION -#if SIMDJSON_CAN_ALWAYS_RUN_HASWELL +#if SIMDJSON_CAN_ALWAYS_RUN_ICELAKE +#define SIMDJSON_BUILTIN_IMPLEMENTATION icelake +#elif SIMDJSON_CAN_ALWAYS_RUN_HASWELL #define SIMDJSON_BUILTIN_IMPLEMENTATION haswell #elif SIMDJSON_CAN_ALWAYS_RUN_WESTMERE #define SIMDJSON_BUILTIN_IMPLEMENTATION westmere diff --git a/include/simdjson/haswell/intrinsics.h b/include/simdjson/haswell/intrinsics.h index 801461231..da15dfa6c 100644 --- a/include/simdjson/haswell/intrinsics.h +++ b/include/simdjson/haswell/intrinsics.h @@ -41,11 +41,7 @@ // has it as a macro. #ifndef _blsr_u64 // we roll our own -SIMDJSON_TARGET_HASWELL -static simdjson_really_inline uint64_t _blsr_u64(uint64_t n) { - return (n - 1) & n; -} -SIMDJSON_UNTARGET_HASWELL +#define _blsr_u64(n) ((n - 1) & n) #endif // _blsr_u64 #endif // SIMDJSON_CLANG_VISUAL_STUDIO diff --git a/include/simdjson/icelake.h b/include/simdjson/icelake.h new file mode 100644 index 000000000..be77589da --- /dev/null +++ b/include/simdjson/icelake.h @@ -0,0 +1,47 @@ +#ifndef SIMDJSON_ICELAKE_H +#define SIMDJSON_ICELAKE_H + +#include "simdjson/implementation-base.h" + +#if SIMDJSON_IMPLEMENTATION_ICELAKE + +#if SIMDJSON_CAN_ALWAYS_RUN_ICELAKE +#define SIMDJSON_TARGET_ICELAKE +#define SIMDJSON_UNTARGET_ICELAKE +#else +#define SIMDJSON_TARGET_ICELAKE SIMDJSON_TARGET_REGION("avx512f,avx512dq,avx512cd,avx512bw,avx512vbmi,avx512vbmi2,avx512vl,avx2,bmi,pclmul,lzcnt") +#define SIMDJSON_UNTARGET_ICELAKE SIMDJSON_UNTARGET_REGION +#endif + +namespace simdjson { +/** + * Implementation for Icelake (Intel AVX512). + */ +namespace icelake { +} // namespace icelake +} // namespace simdjson + +// +// These two need to be included outside SIMDJSON_TARGET_ICELAKE +// +#include "simdjson/icelake/implementation.h" +#include "simdjson/icelake/intrinsics.h" + +// +// The rest need to be inside the region +// +#include "simdjson/icelake/begin.h" + +// Declarations +#include "simdjson/generic/dom_parser_implementation.h" +#include "simdjson/icelake/bitmanipulation.h" +#include "simdjson/icelake/bitmask.h" +#include "simdjson/icelake/simd.h" +#include "simdjson/generic/jsoncharutils.h" +#include "simdjson/generic/atomparsing.h" +#include "simdjson/icelake/stringparsing.h" +#include "simdjson/icelake/numberparsing.h" +#include "simdjson/icelake/end.h" + +#endif // SIMDJSON_IMPLEMENTATION_ICELAKE +#endif // SIMDJSON_ICELAKE_H diff --git a/include/simdjson/icelake/begin.h b/include/simdjson/icelake/begin.h new file mode 100644 index 000000000..2f5c07063 --- /dev/null +++ b/include/simdjson/icelake/begin.h @@ -0,0 +1,2 @@ +#define SIMDJSON_IMPLEMENTATION icelake +SIMDJSON_TARGET_ICELAKE diff --git a/include/simdjson/icelake/bitmanipulation.h b/include/simdjson/icelake/bitmanipulation.h new file mode 100644 index 000000000..51a8aa340 --- /dev/null +++ b/include/simdjson/icelake/bitmanipulation.h @@ -0,0 +1,61 @@ +#ifndef SIMDJSON_ICELAKE_BITMANIPULATION_H +#define SIMDJSON_ICELAKE_BITMANIPULATION_H + +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { +namespace { + +// We sometimes call trailing_zero on inputs that are zero, +// but the algorithms do not end up using the returned value. +// Sadly, sanitizers are not smart enough to figure it out. +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_really_inline int trailing_zeroes(uint64_t input_num) { +#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO + return (int)_tzcnt_u64(input_num); +#else // SIMDJSON_REGULAR_VISUAL_STUDIO + //////// + // You might expect the next line to be equivalent to + // return (int)_tzcnt_u64(input_num); + // but the generated code differs and might be less efficient? + //////// + return __builtin_ctzll(input_num); +#endif // SIMDJSON_REGULAR_VISUAL_STUDIO +} + +/* result might be undefined when input_num is zero */ +simdjson_really_inline uint64_t clear_lowest_bit(uint64_t input_num) { + return _blsr_u64(input_num); +} + +/* result might be undefined when input_num is zero */ +simdjson_really_inline int leading_zeroes(uint64_t input_num) { + return int(_lzcnt_u64(input_num)); +} + +#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO +simdjson_really_inline unsigned __int64 count_ones(uint64_t input_num) { + // note: we do not support legacy 32-bit Windows + return __popcnt64(input_num);// Visual Studio wants two underscores +} +#else +simdjson_really_inline long long int count_ones(uint64_t input_num) { + return _popcnt64(input_num); +} +#endif + +simdjson_really_inline bool add_overflow(uint64_t value1, uint64_t value2, + uint64_t *result) { +#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO + return _addcarry_u64(0, value1, value2, + reinterpret_cast(result)); +#else + return __builtin_uaddll_overflow(value1, value2, + reinterpret_cast(result)); +#endif +} + +} // unnamed namespace +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson + +#endif // SIMDJSON_ICELAKE_BITMANIPULATION_H diff --git a/include/simdjson/icelake/bitmask.h b/include/simdjson/icelake/bitmask.h new file mode 100644 index 000000000..7aead6faf --- /dev/null +++ b/include/simdjson/icelake/bitmask.h @@ -0,0 +1,25 @@ +#ifndef SIMDJSON_ICELAKE_BITMASK_H +#define SIMDJSON_ICELAKE_BITMASK_H + +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { +namespace { + +// +// Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. +// +// For example, prefix_xor(00100100) == 00011100 +// +simdjson_really_inline uint64_t prefix_xor(const uint64_t bitmask) { + // There should be no such thing with a processor supporting avx2 + // but not clmul. + __m128i all_ones = _mm_set1_epi8('\xFF'); + __m128i result = _mm_clmulepi64_si128(_mm_set_epi64x(0ULL, bitmask), all_ones, 0); + return _mm_cvtsi128_si64(result); +} + +} // unnamed namespace +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson + +#endif // SIMDJSON_ICELAKE_BITMASK_H diff --git a/include/simdjson/icelake/end.h b/include/simdjson/icelake/end.h new file mode 100644 index 000000000..b32c1075d --- /dev/null +++ b/include/simdjson/icelake/end.h @@ -0,0 +1,2 @@ +SIMDJSON_UNTARGET_ICELAKE +#undef SIMDJSON_IMPLEMENTATION diff --git a/include/simdjson/icelake/implementation.h b/include/simdjson/icelake/implementation.h new file mode 100644 index 000000000..87e9f05f9 --- /dev/null +++ b/include/simdjson/icelake/implementation.h @@ -0,0 +1,31 @@ +#ifndef SIMDJSON_ICELAKE_IMPLEMENTATION_H +#define SIMDJSON_ICELAKE_IMPLEMENTATION_H + +#include "simdjson/implementation.h" + +// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_ICELAKE +namespace simdjson { +namespace icelake { + +using namespace simdjson; + +class implementation final : public simdjson::implementation { +public: + simdjson_really_inline implementation() : simdjson::implementation( + "icelake", + "Intel/AMD AVX512", + internal::instruction_set::AVX2 | internal::instruction_set::PCLMULQDQ | internal::instruction_set::BMI1 | internal::instruction_set::BMI2 | internal::instruction_set::AVX512F | internal::instruction_set::AVX512DQ | internal::instruction_set::AVX512CD | internal::instruction_set::AVX512BW | internal::instruction_set::AVX512VL | internal::instruction_set::AVX512VBMI2 + ) {} + simdjson_warn_unused error_code create_dom_parser_implementation( + size_t capacity, + size_t max_length, + std::unique_ptr& dst + ) const noexcept final; + simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final; + simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final; +}; + +} // namespace icelake +} // namespace simdjson + +#endif // SIMDJSON_ICELAKE_IMPLEMENTATION_H diff --git a/include/simdjson/icelake/intrinsics.h b/include/simdjson/icelake/intrinsics.h new file mode 100644 index 000000000..e5715f112 --- /dev/null +++ b/include/simdjson/icelake/intrinsics.h @@ -0,0 +1,56 @@ +#ifndef SIMDJSON_ICELAKE_INTRINSICS_H +#define SIMDJSON_ICELAKE_INTRINSICS_H + +#include "simdjson/base.h" + +#ifdef SIMDJSON_VISUAL_STUDIO +// under clang within visual studio, this will include +#include // visual studio or clang +#else +#include // elsewhere +#endif // SIMDJSON_VISUAL_STUDIO + +#ifdef SIMDJSON_CLANG_VISUAL_STUDIO +/** + * You are not supposed, normally, to include these + * headers directly. Instead you should either include intrin.h + * or x86intrin.h. However, when compiling with clang + * under Windows (i.e., when _MSC_VER is set), these headers + * only get included *if* the corresponding features are detected + * from macros: + * e.g., if __AVX2__ is set... in turn, we normally set these + * macros by compiling against the corresponding architecture + * (e.g., arch:AVX2, -mavx2, etc.) which compiles the whole + * software with these advanced instructions. In simdjson, we + * want to compile the whole program for a generic target, + * and only target our specific kernels. As a workaround, + * we directly include the needed headers. These headers would + * normally guard against such usage, but we carefully included + * (or ) before, so the headers + * are fooled. + */ +#include // for _blsr_u64 +#include // for __lzcnt64 +#include // for most things (AVX2, AVX512, _popcnt64) +#include +#include +#include +#include +#include // for _mm_clmulepi64_si128 +// Important: we need the AVX-512 headers: +#include +#include +#include +#include +#include +#include +#include +// unfortunately, we may not get _blsr_u64, but, thankfully, clang +// has it as a macro. +#ifndef _blsr_u64 +// we roll our own +#define _blsr_u64(n) ((n - 1) & n) +#endif // _blsr_u64 +#endif // SIMDJSON_CLANG_VISUAL_STUDIO + +#endif // SIMDJSON_ICELAKE_INTRINSICS_H diff --git a/include/simdjson/icelake/numberparsing.h b/include/simdjson/icelake/numberparsing.h new file mode 100644 index 000000000..a88ced593 --- /dev/null +++ b/include/simdjson/icelake/numberparsing.h @@ -0,0 +1,34 @@ +#ifndef SIMDJSON_ICELAKE_NUMBERPARSING_H +#define SIMDJSON_ICELAKE_NUMBERPARSING_H + +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { +namespace { + +static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { + // this actually computes *16* values so we are being wasteful. + const __m128i ascii0 = _mm_set1_epi8('0'); + const __m128i mul_1_10 = + _mm_setr_epi8(10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1); + const __m128i mul_1_100 = _mm_setr_epi16(100, 1, 100, 1, 100, 1, 100, 1); + const __m128i mul_1_10000 = + _mm_setr_epi16(10000, 1, 10000, 1, 10000, 1, 10000, 1); + const __m128i input = _mm_sub_epi8( + _mm_loadu_si128(reinterpret_cast(chars)), ascii0); + const __m128i t1 = _mm_maddubs_epi16(input, mul_1_10); + const __m128i t2 = _mm_madd_epi16(t1, mul_1_100); + const __m128i t3 = _mm_packus_epi32(t2, t2); + const __m128i t4 = _mm_madd_epi16(t3, mul_1_10000); + return _mm_cvtsi128_si32( + t4); // only captures the sum of the first 8 digits, drop the rest +} + +} // unnamed namespace +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson + +#define SIMDJSON_SWAR_NUMBER_PARSING 1 + +#include "simdjson/generic/numberparsing.h" + +#endif // SIMDJSON_ICELAKE_NUMBERPARSING_H diff --git a/include/simdjson/icelake/simd.h b/include/simdjson/icelake/simd.h new file mode 100644 index 000000000..44436b638 --- /dev/null +++ b/include/simdjson/icelake/simd.h @@ -0,0 +1,339 @@ +#ifndef SIMDJSON_ICELAKE_SIMD_H +#define SIMDJSON_ICELAKE_SIMD_H + +#include "simdjson/internal/simdprune_tables.h" + +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { +namespace { +namespace simd { + + // Forward-declared so they can be used by splat and friends. + template + struct base { + __m512i value; + + // Zero constructor + simdjson_really_inline base() : value{__m512i()} {} + + // Conversion from SIMD register + simdjson_really_inline base(const __m512i _value) : value(_value) {} + + // Conversion to SIMD register + simdjson_really_inline operator const __m512i&() const { return this->value; } + simdjson_really_inline operator __m512i&() { return this->value; } + + // Bit operations + simdjson_really_inline Child operator|(const Child other) const { return _mm512_or_si512(*this, other); } + simdjson_really_inline Child operator&(const Child other) const { return _mm512_and_si512(*this, other); } + simdjson_really_inline Child operator^(const Child other) const { return _mm512_xor_si512(*this, other); } + simdjson_really_inline Child bit_andnot(const Child other) const { return _mm512_andnot_si512(other, *this); } + simdjson_really_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } + simdjson_really_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } + simdjson_really_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } + }; + + // Forward-declared so they can be used by splat and friends. + template + struct simd8; + + template> + struct base8: base> { + typedef uint32_t bitmask_t; + typedef uint64_t bitmask2_t; + + simdjson_really_inline base8() : base>() {} + simdjson_really_inline base8(const __m512i _value) : base>(_value) {} + + simdjson_really_inline uint64_t operator==(const simd8 other) const { return _mm512_cmpeq_epi8_mask(*this, other); } + + static const int SIZE = sizeof(base::value); + + template + simdjson_really_inline simd8 prev(const simd8 prev_chunk) const { + return _mm512_alignr_epi8(*this, _mm512_permutex2var_epi64(prev_chunk, _mm512_set_epi64(13, 12, 11, 10, 9, 8, 7, 6), *this), 16 - N); + } + }; + + // SIMD byte mask type (returned by things like eq and gt) + template<> + struct simd8: base8 { + static simdjson_really_inline simd8 splat(bool _value) { return _mm512_set1_epi8(uint8_t(-(!!_value))); } + + simdjson_really_inline simd8() : base8() {} + simdjson_really_inline simd8(const __m512i _value) : base8(_value) {} + // Splat constructor + simdjson_really_inline simd8(bool _value) : base8(splat(_value)) {} + simdjson_really_inline bool any() const { return !!_mm512_test_epi8_mask (*this, *this); } + simdjson_really_inline simd8 operator~() const { return *this ^ true; } + }; + + template + struct base8_numeric: base8 { + static simdjson_really_inline simd8 splat(T _value) { return _mm512_set1_epi8(_value); } + static simdjson_really_inline simd8 zero() { return _mm512_setzero_si512(); } + static simdjson_really_inline simd8 load(const T values[64]) { + return _mm512_loadu_si512(reinterpret_cast(values)); + } + // Repeat 16 values as many times as necessary (usually for lookup tables) + static simdjson_really_inline simd8 repeat_16( + T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, + T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15, + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15, + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15, + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15 + ); + } + + simdjson_really_inline base8_numeric() : base8() {} + simdjson_really_inline base8_numeric(const __m512i _value) : base8(_value) {} + + // Store to array + simdjson_really_inline void store(T dst[64]) const { return _mm512_storeu_si512(reinterpret_cast<__m512i *>(dst), *this); } + + // Addition/subtraction are the same for signed and unsigned + simdjson_really_inline simd8 operator+(const simd8 other) const { return _mm512_add_epi8(*this, other); } + simdjson_really_inline simd8 operator-(const simd8 other) const { return _mm512_sub_epi8(*this, other); } + simdjson_really_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } + simdjson_really_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } + + // Override to distinguish from bool version + simdjson_really_inline simd8 operator~() const { return *this ^ 0xFFu; } + + // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) + template + simdjson_really_inline simd8 lookup_16(simd8 lookup_table) const { + return _mm512_shuffle_epi8(lookup_table, *this); + } + + // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). + // Passing a 0 value for mask would be equivalent to writing out every byte to output. + // Only the first 32 - count_ones(mask) bytes of the result are significant but 32 bytes + // get written. + // Design consideration: it seems like a function with the + // signature simd8 compress(uint32_t mask) would be + // sensible, but the AVX ISA makes this kind of approach difficult. + template + simdjson_really_inline void compress(uint64_t mask, L * output) const { + _mm512_mask_compressstoreu_epi8 (output,~mask,*this); + } + + template + simdjson_really_inline simd8 lookup_16( + L replace0, L replace1, L replace2, L replace3, + L replace4, L replace5, L replace6, L replace7, + L replace8, L replace9, L replace10, L replace11, + L replace12, L replace13, L replace14, L replace15) const { + return lookup_16(simd8::repeat_16( + replace0, replace1, replace2, replace3, + replace4, replace5, replace6, replace7, + replace8, replace9, replace10, replace11, + replace12, replace13, replace14, replace15 + )); + } + }; + + // Signed bytes + template<> + struct simd8 : base8_numeric { + simdjson_really_inline simd8() : base8_numeric() {} + simdjson_really_inline simd8(const __m512i _value) : base8_numeric(_value) {} + // Splat constructor + simdjson_really_inline simd8(int8_t _value) : simd8(splat(_value)) {} + // Array constructor + simdjson_really_inline simd8(const int8_t values[64]) : simd8(load(values)) {} + // Member-by-member initialization + simdjson_really_inline simd8( + int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, + int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15, + int8_t v16, int8_t v17, int8_t v18, int8_t v19, int8_t v20, int8_t v21, int8_t v22, int8_t v23, + int8_t v24, int8_t v25, int8_t v26, int8_t v27, int8_t v28, int8_t v29, int8_t v30, int8_t v31, + int8_t v32, int8_t v33, int8_t v34, int8_t v35, int8_t v36, int8_t v37, int8_t v38, int8_t v39, + int8_t v40, int8_t v41, int8_t v42, int8_t v43, int8_t v44, int8_t v45, int8_t v46, int8_t v47, + int8_t v48, int8_t v49, int8_t v50, int8_t v51, int8_t v52, int8_t v53, int8_t v54, int8_t v55, + int8_t v56, int8_t v57, int8_t v58, int8_t v59, int8_t v60, int8_t v61, int8_t v62, int8_t v63 + ) : simd8(_mm512_set_epi8( + v63, v62, v61, v60, v59, v58, v57, v56, + v55, v54, v53, v52, v51, v50, v49, v48, + v47, v46, v45, v44, v43, v42, v41, v40, + v39, v38, v37, v36, v35, v34, v33, v32, + v31, v30, v29, v28, v27, v26, v25, v24, + v23, v22, v21, v20, v19, v18, v17, v16, + v15, v14, v13, v12, v11, v10, v9, v8, + v7, v6, v5, v4, v3, v2, v1, v0 + )) {} + + // Repeat 16 values as many times as necessary (usually for lookup tables) + simdjson_really_inline static simd8 repeat_16( + int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, + int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15, + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15, + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15, + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15 + ); + } + + // Order-sensitive comparisons + simdjson_really_inline simd8 max_val(const simd8 other) const { return _mm512_max_epi8(*this, other); } + simdjson_really_inline simd8 min_val(const simd8 other) const { return _mm512_min_epi8(*this, other); } + + simdjson_really_inline simd8 operator>(const simd8 other) const { return _mm512_maskz_abs_epi8(_mm512_cmpgt_epi8_mask(*this, other),_mm512_set1_epi8(uint8_t(0x80))); } + simdjson_really_inline simd8 operator<(const simd8 other) const { return _mm512_maskz_abs_epi8(_mm512_cmpgt_epi8_mask(other, *this),_mm512_set1_epi8(uint8_t(0x80))); } + }; + + // Unsigned bytes + template<> + struct simd8: base8_numeric { + simdjson_really_inline simd8() : base8_numeric() {} + simdjson_really_inline simd8(const __m512i _value) : base8_numeric(_value) {} + // Splat constructor + simdjson_really_inline simd8(uint8_t _value) : simd8(splat(_value)) {} + // Array constructor + simdjson_really_inline simd8(const uint8_t values[64]) : simd8(load(values)) {} + // Member-by-member initialization + simdjson_really_inline simd8( + uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, + uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15, + uint8_t v16, uint8_t v17, uint8_t v18, uint8_t v19, uint8_t v20, uint8_t v21, uint8_t v22, uint8_t v23, + uint8_t v24, uint8_t v25, uint8_t v26, uint8_t v27, uint8_t v28, uint8_t v29, uint8_t v30, uint8_t v31, + uint8_t v32, uint8_t v33, uint8_t v34, uint8_t v35, uint8_t v36, uint8_t v37, uint8_t v38, uint8_t v39, + uint8_t v40, uint8_t v41, uint8_t v42, uint8_t v43, uint8_t v44, uint8_t v45, uint8_t v46, uint8_t v47, + uint8_t v48, uint8_t v49, uint8_t v50, uint8_t v51, uint8_t v52, uint8_t v53, uint8_t v54, uint8_t v55, + uint8_t v56, uint8_t v57, uint8_t v58, uint8_t v59, uint8_t v60, uint8_t v61, uint8_t v62, uint8_t v63 + ) : simd8(_mm512_set_epi8( + v63, v62, v61, v60, v59, v58, v57, v56, + v55, v54, v53, v52, v51, v50, v49, v48, + v47, v46, v45, v44, v43, v42, v41, v40, + v39, v38, v37, v36, v35, v34, v33, v32, + v31, v30, v29, v28, v27, v26, v25, v24, + v23, v22, v21, v20, v19, v18, v17, v16, + v15, v14, v13, v12, v11, v10, v9, v8, + v7, v6, v5, v4, v3, v2, v1, v0 + )) {} + + // Repeat 16 values as many times as necessary (usually for lookup tables) + simdjson_really_inline static simd8 repeat_16( + uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, + uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15, + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15, + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15, + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15 + ); + } + + // Saturated math + simdjson_really_inline simd8 saturating_add(const simd8 other) const { return _mm512_adds_epu8(*this, other); } + simdjson_really_inline simd8 saturating_sub(const simd8 other) const { return _mm512_subs_epu8(*this, other); } + + // Order-specific operations + simdjson_really_inline simd8 max_val(const simd8 other) const { return _mm512_max_epu8(*this, other); } + simdjson_really_inline simd8 min_val(const simd8 other) const { return _mm512_min_epu8(other, *this); } + // Same as >, but only guarantees true is nonzero (< guarantees true = -1) + simdjson_really_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } + // Same as <, but only guarantees true is nonzero (< guarantees true = -1) + simdjson_really_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } + simdjson_really_inline uint64_t operator<=(const simd8 other) const { return other.max_val(*this) == other; } + simdjson_really_inline uint64_t operator>=(const simd8 other) const { return other.min_val(*this) == other; } + simdjson_really_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } + simdjson_really_inline simd8 operator<(const simd8 other) const { return this->lt_bits(other).any_bits_set(); } + + // Bit-specific operations + simdjson_really_inline simd8 bits_not_set() const { return _mm512_mask_blend_epi8(*this == uint8_t(0), _mm512_set1_epi8(0), _mm512_set1_epi8(-1)); } + simdjson_really_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } + simdjson_really_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } + simdjson_really_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } + + simdjson_really_inline bool is_ascii() const { return _mm512_movepi8_mask(*this) == 0; } + simdjson_really_inline bool bits_not_set_anywhere() const { + return !_mm512_test_epi8_mask(*this, *this); + } + simdjson_really_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } + simdjson_really_inline bool bits_not_set_anywhere(simd8 bits) const { return !_mm512_test_epi8_mask(*this, bits); } + simdjson_really_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } + template + simdjson_really_inline simd8 shr() const { return simd8(_mm512_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); } + template + simdjson_really_inline simd8 shl() const { return simd8(_mm512_slli_epi16(*this, N)) & uint8_t(0xFFu << N); } + // Get one of the bits and make a bitmask out of it. + // e.g. value.get_bit<7>() gets the high bit + template + simdjson_really_inline uint64_t get_bit() const { return _mm512_movepi8_mask(_mm512_slli_epi16(*this, 7-N)); } + }; + + template + struct simd8x64 { + static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); + static_assert(NUM_CHUNKS == 1, "Icelake kernel should use one register per 64-byte block."); + const simd8 chunks[NUM_CHUNKS]; + + simd8x64(const simd8x64& o) = delete; // no copy allowed + simd8x64& operator=(const simd8& other) = delete; // no assignment allowed + simd8x64() = delete; // no default constructor allowed + + simdjson_really_inline simd8x64(const simd8 chunk0, const simd8 chunk1) : chunks{chunk0, chunk1} {} + simdjson_really_inline simd8x64(const simd8 chunk0) : chunks{chunk0} {} + simdjson_really_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr)} {} + + simdjson_really_inline uint64_t compress(uint64_t mask, T * output) const { + this->chunks[0].compress(mask, output); + return 64 - count_ones(mask); + } + + simdjson_really_inline void store(T ptr[64]) const { + this->chunks[0].store(ptr+sizeof(simd8)*0); + } + + simdjson_really_inline simd8 reduce_or() const { + return this->chunks[0]; + } + + simdjson_really_inline simd8x64 bit_or(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] | mask + ); + } + + simdjson_really_inline uint64_t eq(const T m) const { + const simd8 mask = simd8::splat(m); + return this->chunks[0] == mask; + } + + simdjson_really_inline uint64_t eq(const simd8x64 &other) const { + return this->chunks[0] == other.chunks[0]; + } + + simdjson_really_inline uint64_t lteq(const T m) const { + const simd8 mask = simd8::splat(m); + return this->chunks[0] <= mask; + } + }; // struct simd8x64 + +} // namespace simd + +} // unnamed namespace +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson + +#endif // SIMDJSON_ICELAKE_SIMD_H diff --git a/include/simdjson/icelake/stringparsing.h b/include/simdjson/icelake/stringparsing.h new file mode 100644 index 000000000..13feed474 --- /dev/null +++ b/include/simdjson/icelake/stringparsing.h @@ -0,0 +1,48 @@ +#ifndef SIMDJSON_ICELAKE_STRINGPARSING_H +#define SIMDJSON_ICELAKE_STRINGPARSING_H + +#include "simdjson/base.h" +#include "simdjson/icelake/simd.h" +#include "simdjson/icelake/bitmanipulation.h" + +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { +namespace { + +using namespace simd; + +// Holds backslashes and quotes locations. +struct backslash_and_quote { +public: + static constexpr uint32_t BYTES_PROCESSED = 32; + simdjson_really_inline static backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); + + simdjson_really_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } + simdjson_really_inline bool has_backslash() { return ((quote_bits - 1) & bs_bits) != 0; } + simdjson_really_inline int quote_index() { return trailing_zeroes(quote_bits); } + simdjson_really_inline int backslash_index() { return trailing_zeroes(bs_bits); } + + uint64_t bs_bits; + uint64_t quote_bits; +}; // struct backslash_and_quote + +simdjson_really_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { + // this can read up to 15 bytes beyond the buffer size, but we require + // SIMDJSON_PADDING of padding + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v(src); + // store to dest unconditionally - we can overwrite the bits we don't like later + v.store(dst); + return { + static_cast(v == '\\'), // bs_bits + static_cast(v == '"'), // quote_bits + }; +} + +} // unnamed namespace +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson + +#include "simdjson/generic/stringparsing.h" + +#endif // SIMDJSON_ICELAKE_STRINGPARSING_H diff --git a/include/simdjson/implementations.h b/include/simdjson/implementations.h index cfa292cd5..0deae4880 100644 --- a/include/simdjson/implementations.h +++ b/include/simdjson/implementations.h @@ -13,10 +13,32 @@ #endif #define SIMDJSON_CAN_ALWAYS_RUN_ARM64 SIMDJSON_IMPLEMENTATION_ARM64 && SIMDJSON_IS_ARM64 +#ifdef __has_include +// How do we detect that a compiler supports vbmi2? +// For sure if the following header is found, we are ok? +#if __has_include() +#define SIMDJSON_COMPILER_SUPPORTS_VBMI2 1 +#endif +#endif + +// Default Icelake to on if this is x86-64. Even if we're not compiled for it, it could be selected +// at runtime. +#ifndef SIMDJSON_IMPLEMENTATION_ICELAKE +#define SIMDJSON_IMPLEMENTATION_ICELAKE ((SIMDJSON_IS_X86_64) && (SIMDJSON_AVX512_ALLOWED) && (SIMDJSON_COMPILER_SUPPORTS_VBMI2)) +#endif + +#ifdef _MSC_VER +// To see why (__BMI__) && (__PCLMUL__) && (__LZCNT__) are not part of this next line, see +// https://github.com/simdjson/simdjson/issues/1247 +#define SIMDJSON_CAN_ALWAYS_RUN_ICELAKE ((SIMDJSON_IMPLEMENTATION_ICELAKE) && (__AVX2__) && (__AVX512F__) && (__AVX512DQ__) && (__AVX512CD__) && (__AVX512BW__) && (__AVX512VL__) && (__AVX512VBMI2__)) +#else +#define SIMDJSON_CAN_ALWAYS_RUN_ICELAKE ((SIMDJSON_IMPLEMENTATION_ICELAKE) && (__AVX2__) && (__BMI__) && (__PCLMUL__) && (__LZCNT__) && (__AVX512F__) && (__AVX512DQ__) && (__AVX512CD__) && (__AVX512BW__) && (__AVX512VL__) && (__AVX512VBMI2__)) +#endif + // Default Haswell to on if this is x86-64. Even if we're not compiled for it, it could be selected // at runtime. #ifndef SIMDJSON_IMPLEMENTATION_HASWELL -#define SIMDJSON_IMPLEMENTATION_HASWELL (SIMDJSON_IS_X86_64) +#define SIMDJSON_IMPLEMENTATION_HASWELL SIMDJSON_IS_X86_64 #endif #ifdef _MSC_VER // To see why (__BMI__) && (__PCLMUL__) && (__LZCNT__) are not part of this next line, see @@ -26,7 +48,7 @@ #define SIMDJSON_CAN_ALWAYS_RUN_HASWELL ((SIMDJSON_IMPLEMENTATION_HASWELL) && (SIMDJSON_IS_X86_64) && (__AVX2__) && (__BMI__) && (__PCLMUL__) && (__LZCNT__)) #endif -// Default Westmere to on if this is x86-64, unless we'll always select Haswell. +// Default Westmere to on if this is x86-64. Note that the macro SIMDJSON_REQUIRES_HASWELL appears unused. #ifndef SIMDJSON_IMPLEMENTATION_WESTMERE #define SIMDJSON_IMPLEMENTATION_WESTMERE (SIMDJSON_IS_X86_64 && !SIMDJSON_REQUIRES_HASWELL) #endif @@ -49,6 +71,7 @@ SIMDJSON_DISABLE_UNDESIRED_WARNINGS // Implementations #include "simdjson/arm64.h" #include "simdjson/fallback.h" +#include "simdjson/icelake.h" #include "simdjson/haswell.h" #include "simdjson/ppc64.h" #include "simdjson/westmere.h" diff --git a/include/simdjson/internal/isadetection.h b/include/simdjson/internal/isadetection.h index 743390789..17adb43f3 100644 --- a/include/simdjson/internal/isadetection.h +++ b/include/simdjson/internal/isadetection.h @@ -66,7 +66,16 @@ enum instruction_set { PCLMULQDQ = 0x10, BMI1 = 0x20, BMI2 = 0x40, - ALTIVEC = 0x80 + ALTIVEC = 0x80, + AVX512F = 0x100, + AVX512DQ = 0x200, + AVX512IFMA = 0x400, + AVX512PF = 0x800, + AVX512ER = 0x1000, + AVX512CD = 0x2000, + AVX512BW = 0x4000, + AVX512VL = 0x8000, + AVX512VBMI2 = 0x10000 }; #if defined(__PPC64__) @@ -96,11 +105,20 @@ static inline uint32_t detect_supported_architectures() { namespace { // Can be found on Intel ISA Reference for CPUID -constexpr uint32_t cpuid_avx2_bit = 1 << 5; ///< @private Bit 5 of EBX for EAX=0x7 -constexpr uint32_t cpuid_bmi1_bit = 1 << 3; ///< @private bit 3 of EBX for EAX=0x7 -constexpr uint32_t cpuid_bmi2_bit = 1 << 8; ///< @private bit 8 of EBX for EAX=0x7 -constexpr uint32_t cpuid_sse42_bit = 1 << 20; ///< @private bit 20 of ECX for EAX=0x1 -constexpr uint32_t cpuid_pclmulqdq_bit = 1 << 1; ///< @private bit 1 of ECX for EAX=0x1 +constexpr uint32_t cpuid_avx2_bit = 1 << 5; ///< @private Bit 5 of EBX for EAX=0x7 +constexpr uint32_t cpuid_bmi1_bit = 1 << 3; ///< @private bit 3 of EBX for EAX=0x7 +constexpr uint32_t cpuid_bmi2_bit = 1 << 8; ///< @private bit 8 of EBX for EAX=0x7 +constexpr uint32_t cpuid_avx512f_bit = 1 << 16; ///< @private bit 16 of EBX for EAX=0x7 +constexpr uint32_t cpuid_avx512dq_bit = 1 << 17; ///< @private bit 17 of EBX for EAX=0x7 +constexpr uint32_t cpuid_avx512ifma_bit = 1 << 21; ///< @private bit 21 of EBX for EAX=0x7 +constexpr uint32_t cpuid_avx512pf_bit = 1 << 26; ///< @private bit 26 of EBX for EAX=0x7 +constexpr uint32_t cpuid_avx512er_bit = 1 << 27; ///< @private bit 27 of EBX for EAX=0x7 +constexpr uint32_t cpuid_avx512cd_bit = 1 << 28; ///< @private bit 28 of EBX for EAX=0x7 +constexpr uint32_t cpuid_avx512bw_bit = 1 << 30; ///< @private bit 30 of EBX for EAX=0x7 +constexpr uint32_t cpuid_avx512vl_bit = 1 << 31; ///< @private bit 31 of EBX for EAX=0x7 +constexpr uint32_t cpuid_avx512vbmi2_bit = 1 << 6; ///< @private bit 6 of ECX for EAX=0x7 +constexpr uint32_t cpuid_sse42_bit = 1 << 20; ///< @private bit 20 of ECX for EAX=0x1 +constexpr uint32_t cpuid_pclmulqdq_bit = 1 << 1; ///< @private bit 1 of ECX for EAX=0x1 } @@ -146,6 +164,42 @@ static inline uint32_t detect_supported_architectures() { host_isa |= instruction_set::BMI2; } + if (ebx & cpuid_avx512f_bit) { + host_isa |= instruction_set::AVX512F; + } + + if (ebx & cpuid_avx512dq_bit) { + host_isa |= instruction_set::AVX512DQ; + } + + if (ebx & cpuid_avx512ifma_bit) { + host_isa |= instruction_set::AVX512IFMA; + } + + if (ebx & cpuid_avx512pf_bit) { + host_isa |= instruction_set::AVX512PF; + } + + if (ebx & cpuid_avx512er_bit) { + host_isa |= instruction_set::AVX512ER; + } + + if (ebx & cpuid_avx512cd_bit) { + host_isa |= instruction_set::AVX512CD; + } + + if (ebx & cpuid_avx512bw_bit) { + host_isa |= instruction_set::AVX512BW; + } + + if (ebx & cpuid_avx512vl_bit) { + host_isa |= instruction_set::AVX512VL; + } + + if (ecx & cpuid_avx512vbmi2_bit) { + host_isa |= instruction_set::AVX512VBMI2; + } + // EBX for EAX=0x1 eax = 0x1; cpuid(&eax, &ebx, &ecx, &edx); diff --git a/src/generic/stage1/json_structural_indexer.h b/src/generic/stage1/json_structural_indexer.h index b0c34cb1d..f3db24b97 100644 --- a/src/generic/stage1/json_structural_indexer.h +++ b/src/generic/stage1/json_structural_indexer.h @@ -25,6 +25,12 @@ public: // base_ptr[base] incrementing base as we go // will potentially store extra values beyond end of valid bits, so base_ptr // needs to be large enough to handle this + // + // If the kernel sets SIMDJSON_CUSTOM_BIT_INDEXER, then it will provide its own + // version of the code. +#ifdef SIMDJSON_CUSTOM_BIT_INDEXER + simdjson_really_inline void write(uint32_t idx, uint64_t bits); +#else simdjson_really_inline void write(uint32_t idx, uint64_t bits) { // In some instances, the next branch is expensive because it is mispredicted. // Unfortunately, in other cases, @@ -117,6 +123,8 @@ public: this->tail += cnt; #endif } +#endif // SIMDJSON_CUSTOM_BIT_INDEXER + }; class json_structural_indexer { diff --git a/src/generic/stage1/utf8_lookup4_algorithm.h b/src/generic/stage1/utf8_lookup4_algorithm.h index a0cfb1f5b..7d44419aa 100644 --- a/src/generic/stage1/utf8_lookup4_algorithm.h +++ b/src/generic/stage1/utf8_lookup4_algorithm.h @@ -111,12 +111,25 @@ using namespace simd; simdjson_really_inline simd8 is_incomplete(const simd8 input) { // If the previous input's last 3 bytes match this, they're too short (they ended at EOF): // ... 1111____ 111_____ 11______ +#if SIMDJSON_IMPLEMENTATION_ICELAKE + static const uint8_t max_array[64] = { + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 0b11110000u-1, 0b11100000u-1, 0b11000000u-1 + }; +#else static const uint8_t max_array[32] = { 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 0b11110000u-1, 0b11100000u-1, 0b11000000u-1 }; +#endif const simd8 max_value(&max_array[sizeof(max_array)-sizeof(simd8)]); return input.gt_bits(max_value); } @@ -154,9 +167,13 @@ using namespace simd; this->error |= this->prev_incomplete; } else { // you might think that a for-loop would work, but under Visual Studio, it is not good enough. - static_assert((simd8x64::NUM_CHUNKS == 2) || (simd8x64::NUM_CHUNKS == 4), - "We support either two or four chunks per 64-byte block."); - if(simd8x64::NUM_CHUNKS == 2) { + static_assert((simd8x64::NUM_CHUNKS == 1) + ||(simd8x64::NUM_CHUNKS == 2) + || (simd8x64::NUM_CHUNKS == 4), + "We support one, two or four chunks per 64-byte block."); + if(simd8x64::NUM_CHUNKS == 1) { + this->check_utf8_bytes(input.chunks[0], this->prev_input_block); + } if(simd8x64::NUM_CHUNKS == 2) { this->check_utf8_bytes(input.chunks[0], this->prev_input_block); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); } else if(simd8x64::NUM_CHUNKS == 4) { diff --git a/src/icelake/dom_parser_implementation.cpp b/src/icelake/dom_parser_implementation.cpp new file mode 100644 index 000000000..ab3ed9bdc --- /dev/null +++ b/src/icelake/dom_parser_implementation.cpp @@ -0,0 +1,206 @@ +#include "simdjson/icelake/begin.h" + +// +// Stage 1 +// + +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { +namespace { + +using namespace simd; + +struct json_character_block { + static simdjson_really_inline json_character_block classify(const simd::simd8x64& in); + // ASCII white-space ('\r','\n','\t',' ') + simdjson_really_inline uint64_t whitespace() const noexcept; + // non-quote structural characters (comma, colon, braces, brackets) + simdjson_really_inline uint64_t op() const noexcept; + // neither a structural character nor a white-space, so letters, numbers and quotes + simdjson_really_inline uint64_t scalar() const noexcept; + + uint64_t _whitespace; // ASCII white-space ('\r','\n','\t',' ') + uint64_t _op; // structural characters (comma, colon, braces, brackets but not quotes) +}; + +simdjson_really_inline uint64_t json_character_block::whitespace() const noexcept { return _whitespace; } +simdjson_really_inline uint64_t json_character_block::op() const noexcept { return _op; } +simdjson_really_inline uint64_t json_character_block::scalar() const noexcept { return ~(op() | whitespace()); } + +// This identifies structural characters (comma, colon, braces, brackets), +// and ASCII white-space ('\r','\n','\t',' '). +simdjson_really_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { + // These lookups rely on the fact that anything < 127 will match the lower 4 bits, which is why + // we can't use the generic lookup_16. + const auto whitespace_table = simd8::repeat_16(' ', 100, 100, 100, 17, 100, 113, 2, 100, '\t', '\n', 112, 100, '\r', 100, 100); + + // The 6 operators (:,[]{}) have these values: + // + // , 2C + // : 3A + // [ 5B + // { 7B + // ] 5D + // } 7D + // + // If you use | 0x20 to turn [ and ] into { and }, the lower 4 bits of each character is unique. + // We exploit this, using a simd 4-bit lookup to tell us which character match against, and then + // match it (against | 0x20). + // + // To prevent recognizing other characters, everything else gets compared with 0, which cannot + // match due to the | 0x20. + // + // NOTE: Due to the | 0x20, this ALSO treats and (control characters 0C and 1A) like , + // and :. This gets caught in stage 2, which checks the actual character to ensure the right + // operators are in the right places. + const auto op_table = simd8::repeat_16( + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, ':', '{', // : = 3A, [ = 5B, { = 7B + ',', '}', 0, 0 // , = 2C, ] = 5D, } = 7D + ); + + // We compute whitespace and op separately. If later code only uses one or the + // other, given the fact that all functions are aggressively inlined, we can + // hope that useless computations will be omitted. This is namely case when + // minifying (we only need whitespace). + + const uint64_t whitespace = in.eq({ + _mm512_shuffle_epi8(whitespace_table, in.chunks[0]) + }); + // Turn [ and ] into { and } + const simd8x64 curlified{ + in.chunks[0] | 0x20 + }; + const uint64_t op = curlified.eq({ + _mm512_shuffle_epi8(op_table, in.chunks[0]) + }); + + return { whitespace, op }; +} + +simdjson_really_inline bool is_ascii(const simd8x64& input) { + return input.reduce_or().is_ascii(); +} + +simdjson_unused simdjson_really_inline simd8 must_be_continuation(const simd8 prev1, const simd8 prev2, const simd8 prev3) { + simd8 is_second_byte = prev1.saturating_sub(0b11000000u-1); // Only 11______ will be > 0 + simd8 is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0 + simd8 is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0 + // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine. + return simd8(is_second_byte | is_third_byte | is_fourth_byte) > int8_t(0); +} + +simdjson_really_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { + simd8 is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0 + simd8 is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0 + // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine. + return simd8(is_third_byte | is_fourth_byte) > int8_t(0); +} + +} // unnamed namespace +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson + +#include "generic/stage1/utf8_lookup4_algorithm.h" +// defining SIMDJSON_CUSTOM_BIT_INDEXER allows us to provide our own bit_indexer::write +#define SIMDJSON_CUSTOM_BIT_INDEXER +#include "generic/stage1/json_structural_indexer.h" +// We must not forget to undefine it now: +#undef SIMDJSON_CUSTOM_BIT_INDEXER + +/** + * We provide a custom version of bit_indexer::write using + * naked intrinsics. + * TODO: make this code more elegant. + */ +namespace simdjson { namespace SIMDJSON_IMPLEMENTATION { namespace { namespace stage1 { +simdjson_really_inline void bit_indexer::write(uint32_t idx, uint64_t bits) { + // In some instances, the next branch is expensive because it is mispredicted. + // Unfortunately, in other cases, + // it helps tremendously. + if (bits == 0) { return; } + + const __m512i indexes = _mm512_maskz_compress_epi8(bits, _mm512_set_epi32( + 0x3f3e3d3c, 0x3b3a3938, 0x37363534, 0x33323130, + 0x2f2e2d2c, 0x2b2a2928, 0x27262524, 0x23222120, + 0x1f1e1d1c, 0x1b1a1918, 0x17161514, 0x13121110, + 0x0f0e0d0c, 0x0b0a0908, 0x07060504, 0x03020100 + )); + const __m512i start_index = _mm512_set1_epi32(idx); + + const auto count = count_ones(bits); + __m512i t0 = _mm512_cvtepu8_epi32(_mm512_castsi512_si128(indexes)); + _mm512_storeu_si512(this->tail, _mm512_add_epi32(t0, start_index)); + + if(count > 16) { + const __m512i t1 = _mm512_cvtepu8_epi32(_mm512_extracti32x4_epi32(indexes, 1)); + _mm512_storeu_si512(this->tail + 16, _mm512_add_epi32(t1, start_index)); + if(count > 32) { + const __m512i t2 = _mm512_cvtepu8_epi32(_mm512_extracti32x4_epi32(indexes, 2)); + _mm512_storeu_si512(this->tail + 32, _mm512_add_epi32(t2, start_index)); + if(count > 48) { + const __m512i t3 = _mm512_cvtepu8_epi32(_mm512_extracti32x4_epi32(indexes, 3)); + _mm512_storeu_si512(this->tail + 48, _mm512_add_epi32(t3, start_index)); + } + } + } + this->tail += count; +} +}}}} + +#include "generic/stage1/utf8_validator.h" + +// +// Stage 2 +// +#include "generic/stage2/tape_builder.h" + +// +// Implementation-specific overrides +// +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { +namespace { +namespace stage1 { + +simdjson_really_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { + if (!backslash) { uint64_t escaped = prev_escaped; prev_escaped = 0; return escaped; } + return find_escaped_branchless(backslash); +} + +} // namespace stage1 +} // unnamed namespace + +simdjson_warn_unused error_code implementation::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept { + return icelake::stage1::json_minifier::minify<128>(buf, len, dst, dst_len); +} + +simdjson_warn_unused error_code dom_parser_implementation::stage1(const uint8_t *_buf, size_t _len, stage1_mode streaming) noexcept { + this->buf = _buf; + this->len = _len; + return icelake::stage1::json_structural_indexer::index<128>(_buf, _len, *this, streaming); +} + +simdjson_warn_unused bool implementation::validate_utf8(const char *buf, size_t len) const noexcept { + return icelake::stage1::generic_validate_utf8(buf,len); +} + +simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document &_doc) noexcept { + return stage2::tape_builder::parse_document(*this, _doc); +} + +simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::document &_doc) noexcept { + return stage2::tape_builder::parse_document(*this, _doc); +} + +simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { + auto error = stage1(_buf, _len, stage1_mode::regular); + if (error) { return error; } + return stage2(_doc); +} + +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson + +#include "simdjson/icelake/end.h" diff --git a/src/icelake/implementation.cpp b/src/icelake/implementation.cpp new file mode 100644 index 000000000..0a8151bd5 --- /dev/null +++ b/src/icelake/implementation.cpp @@ -0,0 +1,24 @@ +#include "simdjson/icelake/begin.h" + +namespace simdjson { +namespace SIMDJSON_IMPLEMENTATION { + +simdjson_warn_unused error_code implementation::create_dom_parser_implementation( + size_t capacity, + size_t max_depth, + std::unique_ptr& dst +) const noexcept { + dst.reset( new (std::nothrow) dom_parser_implementation() ); + if (!dst) { return MEMALLOC; } + if (auto err = dst->set_capacity(capacity)) + return err; + if (auto err = dst->set_max_depth(max_depth)) + return err; + return SUCCESS; +} + +} // namespace SIMDJSON_IMPLEMENTATION +} // namespace simdjson + +#include "simdjson/icelake/end.h" + diff --git a/src/implementation.cpp b/src/implementation.cpp index 653ac2ceb..0a9d380aa 100644 --- a/src/implementation.cpp +++ b/src/implementation.cpp @@ -14,6 +14,12 @@ namespace internal { // Static array of known implementations. We're hoping these get baked into the executable // without requiring a static initializer. +#if SIMDJSON_IMPLEMENTATION_ICELAKE +static const icelake::implementation* get_icelake_singleton() { + static const icelake::implementation icelake_singleton{}; + return &icelake_singleton; +} +#endif #if SIMDJSON_IMPLEMENTATION_HASWELL static const haswell::implementation* get_haswell_singleton() { static const haswell::implementation haswell_singleton{}; @@ -73,6 +79,9 @@ private: static const std::initializer_list& get_available_implementation_pointers() { static const std::initializer_list available_implementation_pointers { +#if SIMDJSON_IMPLEMENTATION_ICELAKE + get_icelake_singleton(), +#endif #if SIMDJSON_IMPLEMENTATION_HASWELL get_haswell_singleton(), #endif diff --git a/src/internal/simdprune_tables.cpp b/src/internal/simdprune_tables.cpp index 55c5fdf65..977094d8a 100644 --- a/src/internal/simdprune_tables.cpp +++ b/src/internal/simdprune_tables.cpp @@ -1,4 +1,4 @@ -#if SIMDJSON_IMPLEMENTATION_ARM64 || SIMDJSON_IMPLEMENTATION_HASWELL || SIMDJSON_IMPLEMENTATION_WESTMERE || SIMDJSON_IMPLEMENTATION_PPC64 +#if SIMDJSON_IMPLEMENTATION_ARM64 || SIMDJSON_IMPLEMENTATION_ICELAKE || SIMDJSON_IMPLEMENTATION_HASWELL || SIMDJSON_IMPLEMENTATION_WESTMERE || SIMDJSON_IMPLEMENTATION_PPC64 #include @@ -128,4 +128,4 @@ SIMDJSON_DLLIMPORTEXPORT const uint64_t thintable_epi8[256] = { } // namespace internal } // namespace simdjson -#endif // SIMDJSON_IMPLEMENTATION_ARM64 || SIMDJSON_IMPLEMENTATION_HASWELL || SIMDJSON_IMPLEMENTATION_WESTMERE || SIMDJSON_IMPLEMENTATION_PPC64 +#endif // SIMDJSON_IMPLEMENTATION_ARM64 || SIMDJSON_IMPLEMENTATION_ICELAKE || SIMDJSON_IMPLEMENTATION_HASWELL || SIMDJSON_IMPLEMENTATION_WESTMERE || SIMDJSON_IMPLEMENTATION_PPC64 diff --git a/src/simdjson.cpp b/src/simdjson.cpp index 8d17791cd..b094bca0d 100644 --- a/src/simdjson.cpp +++ b/src/simdjson.cpp @@ -19,6 +19,10 @@ SIMDJSON_DISABLE_UNDESIRED_WARNINGS #include "fallback/implementation.cpp" #include "fallback/dom_parser_implementation.cpp" #endif +#if SIMDJSON_IMPLEMENTATION_ICELAKE +#include "icelake/implementation.cpp" +#include "icelake/dom_parser_implementation.cpp" +#endif #if SIMDJSON_IMPLEMENTATION_HASWELL #include "haswell/implementation.cpp" #include "haswell/dom_parser_implementation.cpp" diff --git a/tests/ondemand/ondemand_array_error_tests.cpp b/tests/ondemand/ondemand_array_error_tests.cpp index 7d82c51f0..1c24f7250 100644 --- a/tests/ondemand/ondemand_array_error_tests.cpp +++ b/tests/ondemand/ondemand_array_error_tests.cpp @@ -16,7 +16,7 @@ namespace array_error_tests { size_t count = 0; for (auto elem : std::forward(array)) { std::cout << "-"; std::cout.flush(); - V actual; + V actual{}; auto actual_error = elem.get(actual); if (count >= N) { if (count >= (N+N2)) { diff --git a/tests/ondemand/ondemand_readme_examples.cpp b/tests/ondemand/ondemand_readme_examples.cpp index 052082409..aa8efbda2 100644 --- a/tests/ondemand/ondemand_readme_examples.cpp +++ b/tests/ondemand/ondemand_readme_examples.cpp @@ -752,7 +752,7 @@ bool stream_capacity_example() { if( error ) { /* handle the error */ } for (auto doc: stream) { if(counter < 6) { - int64_t val; + int64_t val{}; error = doc.at_pointer("/4").get(val); if( error ) { /* handle the error */ } std::cout << "5 = " << val << std::endl;