From a53d95099c47fa60112029fe8e57aaadfb3fcaf4 Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Mon, 29 Jul 2019 13:28:02 -0400 Subject: [PATCH] Intrinsic-based flatten (#234) * Providing a flatten function with intrinsics (for Visual Studio). --- Makefile | 2 +- include/simdjson/portability.h | 2 +- .../stage1_find_marks_flatten_haswell.h | 86 +++++++++++++++++++ include/simdjson/stage1_find_marks_haswell.h | 2 +- include/simdjson/stage1_find_marks_macros.h | 8 +- jsonchecker/pass20.json | 1 + src/stage1_find_marks.cpp | 6 +- 7 files changed, 97 insertions(+), 10 deletions(-) create mode 100644 include/simdjson/stage1_find_marks_flatten_haswell.h create mode 100644 jsonchecker/pass20.json diff --git a/Makefile b/Makefile index e6b1c4bfa..cb1350b57 100644 --- a/Makefile +++ b/Makefile @@ -62,7 +62,7 @@ TESTEXECUTABLES=jsoncheck numberparsingcheck stringparsingcheck pointercheck COMPARISONEXECUTABLES=minifiercompetition parsingcompetition parseandstatcompetition distinctuseridcompetition allparserscheckfile allparsingcompetition SUPPLEMENTARYEXECUTABLES=parse_noutf8validation parse_nonumberparsing parse_nostringparsing -HEADERS= include/simdjson/simdutf8check_haswell.h include/simdjson/simdutf8check_westmere.h include/simdjson/simdutf8check_arm64.h include/simdjson/stringparsing.h include/simdjson/stringparsing_arm64.h include/simdjson/stringparsing_haswell.h include/simdjson/stringparsing_macros.h include/simdjson/stringparsing_westmere.h include/simdjson/numberparsing.h include/simdjson/jsonparser.h include/simdjson/common_defs.h include/simdjson/jsonioutil.h benchmark/benchmark.h benchmark/linux/linux-perf-events.h include/simdjson/parsedjson.h include/simdjson/stage1_find_marks.h include/simdjson/stage1_find_marks_arm64.h include/simdjson/stage1_find_marks_haswell.h include/simdjson/stage1_find_marks_westmere.h include/simdjson/stage1_find_marks_macros.h include/simdjson/stage2_build_tape.h include/simdjson/jsoncharutils.h include/simdjson/jsonformatutils.h +HEADERS= include/simdjson/simdutf8check_haswell.h include/simdjson/simdutf8check_westmere.h include/simdjson/simdutf8check_arm64.h include/simdjson/stringparsing.h include/simdjson/stringparsing_arm64.h include/simdjson/stringparsing_haswell.h include/simdjson/stringparsing_macros.h include/simdjson/stringparsing_westmere.h include/simdjson/numberparsing.h include/simdjson/jsonparser.h include/simdjson/common_defs.h include/simdjson/jsonioutil.h benchmark/benchmark.h benchmark/linux/linux-perf-events.h include/simdjson/parsedjson.h include/simdjson/stage1_find_marks.h include/simdjson/stage1_find_marks_arm64.h include/simdjson/stage1_find_marks_haswell.h include/simdjson/stage1_find_marks_westmere.h include/simdjson/stage1_find_marks_macros.h include/simdjson/stage2_build_tape.h include/simdjson/jsoncharutils.h include/simdjson/jsonformatutils.h include/simdjson/stage1_find_marks_flatten.h include/simdjson/stage1_find_marks_flatten_haswell.h LIBFILES=src/jsonioutil.cpp src/jsonparser.cpp src/simdjson.cpp src/stage1_find_marks.cpp src/stage2_build_tape.cpp src/parsedjson.cpp src/parsedjsoniterator.cpp MINIFIERHEADERS=include/simdjson/jsonminifier.h include/simdjson/simdprune_tables.h MINIFIERLIBFILES=src/jsonminifier.cpp diff --git a/include/simdjson/portability.h b/include/simdjson/portability.h index 8963d29a4..37d1532a7 100644 --- a/include/simdjson/portability.h +++ b/include/simdjson/portability.h @@ -99,7 +99,7 @@ static inline bool mul_overflow(uint64_t value1, uint64_t value2, uint64_t *re /* result might be undefined when input_num is zero */ static inline int trailingzeroes(uint64_t input_num) { -#ifdef __BMI2__ +#ifdef __BMI__// tzcnt is BMI1 return _tzcnt_u64(input_num); #else return __builtin_ctzll(input_num); diff --git a/include/simdjson/stage1_find_marks_flatten_haswell.h b/include/simdjson/stage1_find_marks_flatten_haswell.h new file mode 100644 index 000000000..5b12c1388 --- /dev/null +++ b/include/simdjson/stage1_find_marks_flatten_haswell.h @@ -0,0 +1,86 @@ +#ifndef SIMDJSON_STAGE1_FIND_MARKS_FLATTEN_HASWELL_H +#define SIMDJSON_STAGE1_FIND_MARKS_FLATTEN_HASWELL_H + +// This file provides the same function as +// stage1_find_marks_flatten.h, but uses Intel intrinsics. +// This should provide better performance on Visual Studio +// and other compilers that do a conservative optimization. + +#include "simdjson/common_defs.h" +#include "simdjson/portability.h" + +TARGET_HASWELL +namespace simdjson { +namespace haswell { + +// flatten out values in 'bits' assuming that they are are to have values of idx +// plus their position in the bitvector, and store these indexes at +// base_ptr[base] incrementing base as we go +// will potentially store extra values beyond end of valid bits, so base_ptr +// needs to be large enough to handle this +really_inline void flatten_bits(uint32_t *base_ptr, uint32_t &base, + uint32_t idx, uint64_t bits) { + // In some instances, the next branch is expensive because it is mispredicted. + // Unfortunately, in other cases, + // it helps tremendously. + if(bits == 0) return; + uint32_t cnt = _popcnt64(bits); + uint32_t next_base = base + cnt; + idx -= 64; + base_ptr += base; + { + base_ptr[0] = idx + _mm_tzcnt_64(bits); + bits = _blsr_u64(bits); + base_ptr[1] = idx + _mm_tzcnt_64(bits); + bits = _blsr_u64(bits); + base_ptr[2] = idx + _mm_tzcnt_64(bits); + bits = _blsr_u64(bits); + base_ptr[3] = idx + _mm_tzcnt_64(bits); + bits = _blsr_u64(bits); + base_ptr[4] = idx + _mm_tzcnt_64(bits); + bits = _blsr_u64(bits); + base_ptr[5] = idx + _mm_tzcnt_64(bits); + bits = _blsr_u64(bits); + base_ptr[6] = idx + _mm_tzcnt_64(bits); + bits = _blsr_u64(bits); + base_ptr[7] = idx + _mm_tzcnt_64(bits); + bits = _blsr_u64(bits); + base_ptr += 8; + } + // We hope that the next branch is easily predicted. + if (cnt > 8) { + base_ptr[0] = idx + _mm_tzcnt_64(bits); + bits = _blsr_u64(bits); + base_ptr[1] = idx + _mm_tzcnt_64(bits); + bits = _blsr_u64(bits); + base_ptr[2] = idx + _mm_tzcnt_64(bits); + bits = _blsr_u64(bits); + base_ptr[3] = idx + _mm_tzcnt_64(bits); + bits = _blsr_u64(bits); + base_ptr[4] = idx + _mm_tzcnt_64(bits); + bits = _blsr_u64(bits); + base_ptr[5] = idx + _mm_tzcnt_64(bits); + bits = _blsr_u64(bits); + base_ptr[6] = idx + _mm_tzcnt_64(bits); + bits = _blsr_u64(bits); + base_ptr[7] = idx + _mm_tzcnt_64(bits); + bits = _blsr_u64(bits); + base_ptr += 8; + } + if (cnt > 16) { // unluckly: we rarely get here + // since it means having one structural or pseudo-structral element + // every 4 characters (possible with inputs like "","","",...). + do { + base_ptr[0] = idx + _mm_tzcnt_64(bits); + bits = _blsr_u64(bits); + base_ptr++; + } while(bits != 0); + } + base = next_base; +} +} // haswell +} // simdjson +UNTARGET_REGION + + +#endif // SIMDJSON_STAGE1_FIND_MARKS_FLATTEN_H \ No newline at end of file diff --git a/include/simdjson/stage1_find_marks_haswell.h b/include/simdjson/stage1_find_marks_haswell.h index a2e707614..6cb979160 100644 --- a/include/simdjson/stage1_find_marks_haswell.h +++ b/include/simdjson/stage1_find_marks_haswell.h @@ -3,7 +3,7 @@ #include "simdjson/stage1_find_marks.h" #include "simdjson/stage1_find_marks_macros.h" -#include "simdjson/stage1_find_marks_flatten.h" +#include "simdjson/stage1_find_marks_flatten_haswell.h" #include "simdjson/simdutf8check_haswell.h" #ifdef IS_X86_64 diff --git a/include/simdjson/stage1_find_marks_macros.h b/include/simdjson/stage1_find_marks_macros.h index 5e43f2c7e..3b9bcbdfe 100644 --- a/include/simdjson/stage1_find_marks_macros.h +++ b/include/simdjson/stage1_find_marks_macros.h @@ -87,7 +87,7 @@ // We need to compile that code for multiple architectures. However, target attributes can be used // only once by function definition. Huge macro seemed better than huge code duplication. // FIND_STRUCTURAL_BITS(architecture T, const uint8_t *buf, size_t len, ParsedJson &pj) -#define FIND_STRUCTURAL_BITS(T, buf, len, pj) { \ +#define FIND_STRUCTURAL_BITS(T, buf, len, pj, flat) { \ if (len > pj.bytecapacity) { \ std::cerr << "Your ParsedJson object only supports documents up to " \ << pj.bytecapacity << " bytes but you are trying to process " << len \ @@ -141,7 +141,7 @@ \ /* take the previous iterations structural bits, not our current iteration, */ \ /* and flatten */ \ - flatten_bits(base_ptr, base, idx, structurals); \ + flat(base_ptr, base, idx, structurals); \ \ uint64_t whitespace; \ find_whitespace_and_structurals(in, whitespace, structurals); \ @@ -175,7 +175,7 @@ \ /* take the previous iterations structural bits, not our current iteration, */ \ /* and flatten */ \ - flatten_bits(base_ptr, base, idx, structurals); \ + flat(base_ptr, base, idx, structurals); \ \ uint64_t whitespace; \ find_whitespace_and_structurals(in, whitespace, structurals); \ @@ -192,7 +192,7 @@ } \ \ /* finally, flatten out the remaining structurals from the last iteration */ \ - flatten_bits(base_ptr, base, idx, structurals); \ + flat(base_ptr, base, idx, structurals); \ \ pj.n_structural_indexes = base; \ /* a valid JSON file cannot have zero structural indexes - we should have */ \ diff --git a/jsonchecker/pass20.json b/jsonchecker/pass20.json new file mode 100644 index 000000000..d6a932a86 --- /dev/null +++ b/jsonchecker/pass20.json @@ -0,0 +1 @@ +1.2e000000010 diff --git a/src/stage1_find_marks.cpp b/src/stage1_find_marks.cpp index 0e593f7e5..605dbc976 100644 --- a/src/stage1_find_marks.cpp +++ b/src/stage1_find_marks.cpp @@ -9,7 +9,7 @@ TARGET_HASWELL namespace simdjson { template<> int find_structural_bits(const uint8_t *buf, size_t len, ParsedJson &pj) { - FIND_STRUCTURAL_BITS(architecture::haswell, buf, len, pj); + FIND_STRUCTURAL_BITS(architecture::haswell, buf, len, pj, simdjson::haswell::flatten_bits); } } // simdjson UNTARGET_REGION @@ -18,7 +18,7 @@ TARGET_WESTMERE namespace simdjson { template<> int find_structural_bits(const uint8_t *buf, size_t len, ParsedJson &pj) { - FIND_STRUCTURAL_BITS(architecture::westmere, buf, len, pj); + FIND_STRUCTURAL_BITS(architecture::westmere, buf, len, pj, simdjson::flatten_bits); } } // simdjson UNTARGET_REGION @@ -31,7 +31,7 @@ UNTARGET_REGION namespace simdjson { template<> int find_structural_bits(const uint8_t *buf, size_t len, ParsedJson &pj) { - FIND_STRUCTURAL_BITS(architecture::arm64, buf, len, pj); + FIND_STRUCTURAL_BITS(architecture::arm64, buf, len, pj, simdjson::flatten_bits); } } #endif