From aa78b70d69daed04d2ba82b084ef3c76d409bf7d Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Mon, 1 Jul 2019 14:18:30 -0400 Subject: [PATCH] Introducing a "native" instruction set so that you do not need to do #ifdef to select the right SIMD set all the time. Fixing indentation. Removing some obsolete WARN_UNUSED. Fixing a weird warning with optind variable. --- benchmark/parse.cpp | 14 +++------- benchmark/statisticalmodel.cpp | 7 ++--- include/simdjson/jsonparser.h | 40 +++++++++++++++------------- include/simdjson/simdjson.h | 15 ++++++++++- include/simdjson/stage1_find_marks.h | 4 +-- src/jsonparser.cpp | 2 +- tools/jsonstats.cpp | 10 +++---- 7 files changed, 50 insertions(+), 42 deletions(-) diff --git a/benchmark/parse.cpp b/benchmark/parse.cpp index fcad31d7d..132de5496 100644 --- a/benchmark/parse.cpp +++ b/benchmark/parse.cpp @@ -144,11 +144,8 @@ int main(int argc, char *argv[]) { std::cout << "[verbose] allocated memory for parsed JSON " << std::endl; } unified.start(); -#ifdef __AVX2__ - isok = (find_structural_bits(p.data(), p.size(), pj) == simdjson::SUCCESS); -#elif defined (__ARM_NEON) - isok = (find_structural_bits(p.data(), p.size(), pj) == simdjson::SUCCESS); -#endif + // The default template is simdjson::instruction_set::native. + isok = (find_structural_bits<>(p.data(), p.size(), pj) == simdjson::SUCCESS); unified.end(results); cy1 += results[0]; cl1 += results[1]; @@ -189,11 +186,8 @@ int main(int argc, char *argv[]) { } auto start = std::chrono::steady_clock::now(); -#ifdef __AVX2__ - isok = (find_structural_bits(p.data(), p.size(), pj) == simdjson::SUCCESS); -#elif defined (__ARM_NEON) - isok = (find_structural_bits(p.data(), p.size(), pj) == simdjson::SUCCESS); -#endif + // The default template is simdjson::instruction_set::native. + isok = (find_structural_bits<>(p.data(), p.size(), pj) == simdjson::SUCCESS); isok = isok && (simdjson::SUCCESS == unified_machine(p.data(), p.size(), pj)); auto end = std::chrono::steady_clock::now(); std::chrono::duration secs = end - start; diff --git a/benchmark/statisticalmodel.cpp b/benchmark/statisticalmodel.cpp index c0b331b16..c2c06b832 100644 --- a/benchmark/statisticalmodel.cpp +++ b/benchmark/statisticalmodel.cpp @@ -180,11 +180,8 @@ int main(int argc, char *argv[]) { results.resize(evts.size()); for (uint32_t i = 0; i < iterations; i++) { unified.start(); -#ifdef __AVX2__ - bool isok = (find_structural_bits(p.data(), p.size(), pj) == simdjson::SUCCESS); -#elif defined (__ARM_NEON) - bool isok = (find_structural_bits(p.data(), p.size(), pj) == simdjson::SUCCESS); -#endif + // The default template is simdjson::instruction_set::native. + bool isok = (find_structural_bits<>(p.data(), p.size(), pj) == simdjson::SUCCESS); unified.end(results); cy1 += results[0]; diff --git a/include/simdjson/jsonparser.h b/include/simdjson/jsonparser.h index 4c0cf71f6..03844660a 100644 --- a/include/simdjson/jsonparser.h +++ b/include/simdjson/jsonparser.h @@ -15,12 +15,20 @@ #include #endif +// The function that users are expected to call is json_parse. +// We have more than one such function because we want to support several +// instruction sets. + // function pointer type for json_parse using json_parse_functype = int (const uint8_t *buf, size_t len, ParsedJson &pj, bool reallocifneeded); // Pointer that holds the json_parse implementation corresponding to the available SIMD instruction set extern json_parse_functype *json_parse_ptr; + +// json_parse_implementation is the generic function, it is specialized for various +// SIMD instruction sets, e.g., as json_parse_implementation +// or json_parse_implementation template int json_parse_implementation(const uint8_t *buf, size_t len, ParsedJson &pj, bool reallocifneeded = true) { if (pj.bytecapacity < len) { @@ -29,22 +37,22 @@ int json_parse_implementation(const uint8_t *buf, size_t len, ParsedJson &pj, bo bool reallocated = false; if(reallocifneeded) { #ifdef ALLOW_SAME_PAGE_BUFFER_OVERRUN - // realloc is needed if the end of the memory crosses a page + // realloc is needed if the end of the memory crosses a page #ifdef _MSC_VER - SYSTEM_INFO sysInfo; - GetSystemInfo(&sysInfo); - long pagesize = sysInfo.dwPageSize; + SYSTEM_INFO sysInfo; + GetSystemInfo(&sysInfo); + long pagesize = sysInfo.dwPageSize; #else long pagesize = sysconf (_SC_PAGESIZE); #endif - ////////////// - // We want to check that buf + len - 1 and buf + len - 1 + SIMDJSON_PADDING - // are in the same page. - // That is, we want to check that - // (buf + len - 1) / pagesize == (buf + len - 1 + SIMDJSON_PADDING) / pagesize - // That's true if (buf + len - 1) % pagesize + SIMDJSON_PADDING < pagesize. - /////////// - if ( (reinterpret_cast(buf + len - 1) % pagesize ) + SIMDJSON_PADDING < static_cast(pagesize) ) { + ////////////// + // We want to check that buf + len - 1 and buf + len - 1 + SIMDJSON_PADDING + // are in the same page. + // That is, we want to check that + // (buf + len - 1) / pagesize == (buf + len - 1 + SIMDJSON_PADDING) / pagesize + // That's true if (buf + len - 1) % pagesize + SIMDJSON_PADDING < pagesize. + /////////// + if ( (reinterpret_cast(buf + len - 1) % pagesize ) + SIMDJSON_PADDING < static_cast(pagesize) ) { #else // SIMDJSON_SAFE_SAME_PAGE_READ_OVERRUN if(true) { // if not SIMDJSON_SAFE_SAME_PAGE_READ_OVERRUN, we always reallocate #endif @@ -53,8 +61,8 @@ int json_parse_implementation(const uint8_t *buf, size_t len, ParsedJson &pj, bo if(buf == NULL) return simdjson::MEMALLOC; memcpy((void*)buf,tmpbuf,len); reallocated = true; - } - } + } // if (true) OR if ( (reinterpret_cast(buf + len - 1) % pagesize ) + SIMDJSON_PADDING < static_cast(pagesize) ) { + } // if(reallocifneeded) { int stage1_is_ok = find_structural_bits(buf, len, pj); if(stage1_is_ok != simdjson::SUCCESS) { pj.errorcode = stage1_is_ok; @@ -81,7 +89,6 @@ int json_parse_implementation(const uint8_t *buf, size_t len, ParsedJson &pj, bo // all bytes at and after buf + len are ignored (can be garbage). // The ParsedJson object can be reused. -WARN_UNUSED inline int json_parse(const uint8_t *buf, size_t len, ParsedJson &pj, bool reallocifneeded = true) { return json_parse_ptr(buf, len, pj, reallocifneeded); } @@ -102,7 +109,6 @@ inline int json_parse(const uint8_t *buf, size_t len, ParsedJson &pj, bool reall // The input buf should be readable up to buf + len + SIMDJSON_PADDING if reallocifneeded is false, // all bytes at and after buf + len are ignored (can be garbage). // The ParsedJson object can be reused. -WARN_UNUSED inline int json_parse(const char * buf, size_t len, ParsedJson &pj, bool reallocifneeded = true) { return json_parse_ptr(reinterpret_cast(buf), len, pj, reallocifneeded); } @@ -120,7 +126,6 @@ int json_parse(const char * buf, ParsedJson &pj) = delete; // // A temporary buffer is created when needed during processing // (a copy of the input string is made). -WARN_UNUSED inline int json_parse(const std::string &s, ParsedJson &pj) { return json_parse(s.data(), s.length(), pj, true); } @@ -135,7 +140,6 @@ inline int json_parse(const std::string &s, ParsedJson &pj) { // // You can also check validity // by calling pj.isValid(). The same ParsedJson can be reused for other documents. -WARN_UNUSED inline int json_parse(const padded_string &s, ParsedJson &pj) { return json_parse(s.data(), s.length(), pj, false); } diff --git a/include/simdjson/simdjson.h b/include/simdjson/simdjson.h index 343c727d5..7f88d635b 100644 --- a/include/simdjson/simdjson.h +++ b/include/simdjson/simdjson.h @@ -8,7 +8,20 @@ struct simdjson { avx2, sse4_2, neon, - none + none, +// the 'native' enum class value should point at a good default on the current machine +#ifdef __AVX2__ + native = avx2 +#elif defined(__ARM_NEON) + native = neon +#else + // Let us assume that we have an old x64 processor, but one that has SSE (i.e., something + // that came out in the second decade of the XXIst century. + // It would be nicer to check explicitly, but there many not be a good way to do so + // that is cross-platform. + // Under Visual Studio, there is no way to check for SSE4.2 support at compile-time. + native = sse4_2 +#endif }; enum errorValues { diff --git a/include/simdjson/stage1_find_marks.h b/include/simdjson/stage1_find_marks.h index 3504fdbe1..2cf0e7d98 100644 --- a/include/simdjson/stage1_find_marks.h +++ b/include/simdjson/stage1_find_marks.h @@ -697,7 +697,7 @@ really_inline uint64_t finalize_structurals( return structurals; } -template +template WARN_UNUSED /*never_inline*/ int find_structural_bits(const uint8_t *buf, size_t len, ParsedJson &pj) { @@ -848,7 +848,7 @@ WARN_UNUSED #endif } -template +template WARN_UNUSED int find_structural_bits(const char *buf, size_t len, ParsedJson &pj) { return find_structural_bits(reinterpret_cast(buf), len, pj); diff --git a/src/jsonparser.cpp b/src/jsonparser.cpp index 14238c36d..c2cf26d98 100644 --- a/src/jsonparser.cpp +++ b/src/jsonparser.cpp @@ -63,7 +63,7 @@ ParsedJson build_parsed_json(const uint8_t *buf, size_t len, bool reallocifneede ParsedJson pj; bool ok = pj.allocateCapacity(len); if(ok) { - (void)json_parse(buf, len, pj, reallocifneeded); + json_parse(buf, len, pj, reallocifneeded); } else { std::cerr << "failure during memory allocation " << std::endl; } diff --git a/tools/jsonstats.cpp b/tools/jsonstats.cpp index 81e75ecea..8e7462097 100644 --- a/tools/jsonstats.cpp +++ b/tools/jsonstats.cpp @@ -115,15 +115,15 @@ stat_t simdjson_computestats(const padded_string &p) { int main(int argc, char *argv[]) { - int optind = 1; - if (optind >= argc) { + int myoptind = 1; + if (myoptind >= argc) { std::cerr << "Reads json, prints stats. " << std::endl; std::cerr << "Usage: " << argv[0] << " " << std::endl; exit(1); } - const char *filename = argv[optind]; - if (optind + 1 < argc) { - std::cerr << "warning: ignoring everything after " << argv[optind + 1] << std::endl; + const char *filename = argv[myoptind]; + if (myoptind + 1 < argc) { + std::cerr << "warning: ignoring everything after " << argv[myoptind + 1] << std::endl; } padded_string p; try {