diff --git a/.gitignore b/.gitignore index 4e744e571..89d8f4e2d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .vscode/ +.idea/ /VS/ /build/ /benchbranch/ diff --git a/CMakeLists.txt b/CMakeLists.txt index a2a3cf187..38997d335 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,6 +11,9 @@ option(SIMDJSON_DISABLE_AVX "Forcefully disable AVX even if hardware supports it set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_MACOSX_RPATH OFF) +set(CMAKE_THREAD_PREFER_PTHREAD ON) +set(THREADS_PREFER_PTHREAD_FLAG ON) + if (NOT CMAKE_BUILD_TYPE) message(STATUS "No build type selected, default to Release") set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build." FORCE) @@ -35,6 +38,11 @@ set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/tools/cmake") find_package(CTargets) find_package(Options) +find_package(Threads REQUIRED) + +if(CMAKE_USE_PTHREADS_INIT) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread") +endif() install(DIRECTORY include/${SIMDJSON_LIB_NAME} DESTINATION include) set (TEST_DATA_DIR "${CMAKE_CURRENT_SOURCE_DIR}/jsonchecker/") diff --git a/Makefile b/Makefile index a52b91de7..c7fbe71a4 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ else ARCHFLAGS ?= -msse4.2 -mpclmul # lowest supported feature set? endif -CXXFLAGS = $(ARCHFLAGS) -std=c++17 -Wall -Wextra -Wshadow -Iinclude -Isrc -Ibenchmark/linux $(EXTRAFLAGS) +CXXFLAGS = $(ARCHFLAGS) -std=c++17 -pthread -Wall -Wextra -Wshadow -Iinclude -Isrc -Ibenchmark/linux $(EXTRAFLAGS) CFLAGS = $(ARCHFLAGS) -Idependencies/ujson4c/3rdparty -Idependencies/ujson4c/src $(EXTRAFLAGS) diff --git a/amalgamation.sh b/amalgamation.sh index c83723e4d..87484613b 100755 --- a/amalgamation.sh +++ b/amalgamation.sh @@ -181,16 +181,16 @@ echo "Giving final instructions:" CPPBIN=${DEMOCPP%%.*} echo "Try :" -echo "c++ -O3 -std=c++17 -o ${CPPBIN} ${DEMOCPP} && ./${CPPBIN} ../jsonexamples/twitter.json ../jsonexamples/amazon_cellphones.ndjson" +echo "c++ -O3 -std=c++17 -pthread -o ${CPPBIN} ${DEMOCPP} && ./${CPPBIN} ../jsonexamples/twitter.json ../jsonexamples/amazon_cellphones.ndjson" SINGLEHDR=$SCRIPTPATH/singleheader echo "Copying files to $SCRIPTPATH/singleheader " mkdir -p $SINGLEHDR -echo "c++ -O3 -std=c++17 -o ${CPPBIN} ${DEMOCPP} && ./${CPPBIN} ../jsonexamples/twitter.json ../jsonexamples/amazon_cellphones.ndjson" > $SINGLEHDR/README.md +echo "c++ -O3 -std=c++17 -pthread -o ${CPPBIN} ${DEMOCPP} && ./${CPPBIN} ../jsonexamples/twitter.json ../jsonexamples/amazon_cellphones.ndjson" > $SINGLEHDR/README.md cp ${AMAL_C} ${AMAL_H} ${DEMOCPP} $SINGLEHDR ls $SINGLEHDR -cd $SINGLEHDR && c++ -O3 -std=c++17 -o ${CPPBIN} ${DEMOCPP} && ./${CPPBIN} ../jsonexamples/twitter.json ../jsonexamples/amazon_cellphones.ndjson +cd $SINGLEHDR && c++ -O3 -std=c++17 -pthread -o ${CPPBIN} ${DEMOCPP} && ./${CPPBIN} ../jsonexamples/twitter.json ../jsonexamples/amazon_cellphones.ndjson lowercase(){ echo "$1" | tr 'A-Z' 'a-z' diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index b12397634..624bda2ea 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -7,4 +7,7 @@ target_include_directories(${SIMDJSON_LIB_NAME} add_cpp_benchmark(parse) add_cpp_benchmark(statisticalmodel) add_cpp_benchmark(parse_stream) + +target_link_libraries(parse_stream Threads::Threads) + add_executable(perfdiff perfdiff.cpp) diff --git a/benchmark/parse_stream.cpp b/benchmark/parse_stream.cpp index 507f90ccc..25b04706c 100755 --- a/benchmark/parse_stream.cpp +++ b/benchmark/parse_stream.cpp @@ -8,7 +8,7 @@ #include "simdjson/parsedjson.h" #define NB_ITERATION 5 -#define MIN_BATCH_SIZE 100000 +#define MIN_BATCH_SIZE 200000 #define MAX_BATCH_SIZE 10000000 bool test_baseline = false; @@ -74,7 +74,7 @@ if(test_per_batch) { std::wclog << "Jsonstream: Speed per batch_size... from " << MIN_BATCH_SIZE << " bytes to " << MAX_BATCH_SIZE << " bytes..." << std::endl; std::cout << "Batch Size\t" << "Gigabytes/second\t" << "Nb of documents parsed" << std::endl; - for (size_t i = MIN_BATCH_SIZE; i <= MAX_BATCH_SIZE; i += (MAX_BATCH_SIZE - MIN_BATCH_SIZE) / 200) { + for (size_t i = MIN_BATCH_SIZE; i <= MAX_BATCH_SIZE; i += (MAX_BATCH_SIZE - MIN_BATCH_SIZE) / 30) { batch_size_res.insert(std::pair(i, 0)); int count; for (size_t j = 0; j < 5; j++) { diff --git a/include/simdjson/jsonstream.h b/include/simdjson/jsonstream.h index a29b7a74e..1eb9907f8 100644 --- a/include/simdjson/jsonstream.h +++ b/include/simdjson/jsonstream.h @@ -3,6 +3,7 @@ #include +#include #include "simdjson/stage1_find_marks.h" #include "simdjson/stage2_build_tape.h" #include "simdjson/simdjson.h" @@ -35,8 +36,17 @@ namespace simdjson { bool error_on_last_attempt{false}; bool load_next_batch{true}; size_t current_buffer_loc{0}; + size_t last_json_buffer_loc{0}; + size_t thread_current_buffer_loc{0}; size_t n_parsed_docs{0}; size_t n_bytes_parsed{0}; + + std::thread stage_1_thread; + simdjson::ParsedJson pj_thread; + +#ifdef SIMDJSON_THREADS_ENABLED + size_t find_last_json(const ParsedJson &pj); +#endif }; } diff --git a/include/simdjson/portability.h b/include/simdjson/portability.h index 8b363af5c..ee7ec7410 100644 --- a/include/simdjson/portability.h +++ b/include/simdjson/portability.h @@ -42,6 +42,11 @@ #define TARGET_WESTMERE TARGET_REGION("sse4.2,pclmul") #define TARGET_ARM64 +// Is threading enabled? +#if defined(BOOST_HAS_THREADS) || defined(_REENTRANT) || defined(_MT) +#define SIMDJSON_THREADS_ENABLED 1 +#endif + #ifdef _MSC_VER #include #else diff --git a/singleheader/README.md b/singleheader/README.md index 38f591d90..754861b76 100755 --- a/singleheader/README.md +++ b/singleheader/README.md @@ -1 +1 @@ -c++ -O3 -std=c++17 -o amalgamation_demo amalgamation_demo.cpp && ./amalgamation_demo ../jsonexamples/twitter.json ../jsonexamples/amazon_cellphones.ndjson +c++ -O3 -std=c++17 -pthread -o amalgamation_demo amalgamation_demo.cpp && ./amalgamation_demo ../jsonexamples/twitter.json ../jsonexamples/amazon_cellphones.ndjson diff --git a/singleheader/amalgamation_demo.cpp b/singleheader/amalgamation_demo.cpp index c8637f864..4a4d368dd 100755 --- a/singleheader/amalgamation_demo.cpp +++ b/singleheader/amalgamation_demo.cpp @@ -1,4 +1,4 @@ -/* auto-generated on Thu 07 Nov 2019 05:05:37 PM EST. Do not edit! */ +/* auto-generated on Wed Nov 20 11:15:43 EST 2019. Do not edit! */ #include #include "simdjson.h" diff --git a/singleheader/simdjson.cpp b/singleheader/simdjson.cpp index 22cf5d443..108416054 100755 --- a/singleheader/simdjson.cpp +++ b/singleheader/simdjson.cpp @@ -1,4 +1,4 @@ -/* auto-generated on Thu 07 Nov 2019 05:05:37 PM EST. Do not edit! */ +/* auto-generated on Wed Nov 20 11:15:43 EST 2019. Do not edit! */ #include "simdjson.h" /* used for http://dmalloc.com/ Dmalloc - Debug Malloc Library */ @@ -36466,15 +36466,14 @@ ParsedJson build_parsed_json(const uint8_t *buf, size_t len, #include using namespace simdjson; - void find_the_best_supported_implementation(); + typedef int (*stage1_functype)(const char *buf, size_t len, ParsedJson &pj, bool streaming); typedef int (*stage2_functype)(const char *buf, size_t len, ParsedJson &pj, size_t &next_json); stage1_functype best_stage1; stage2_functype best_stage2; - JsonStream::JsonStream(const char *buf, size_t len, size_t batchSize) : _buf(buf), _len(len), _batch_size(batchSize) { find_the_best_supported_implementation(); @@ -36493,11 +36492,10 @@ void JsonStream::set_new_buffer(const char *buf, size_t len) { } int JsonStream::json_parse(ParsedJson &pj) { - //return json_parse_ptr.load(std::memory_order_relaxed)(buf, len, batch_size, pj, realloc_if_needed); - if (pj.byte_capacity == 0) { const bool allocok = pj.allocate_capacity(_batch_size, _batch_size); - if (!allocok) { + const bool allocok_thread = pj_thread.allocate_capacity(_batch_size, _batch_size); + if (!allocok || !allocok_thread) { std::cerr << "can't allocate memory" << std::endl; return false; } @@ -36505,31 +36503,64 @@ int JsonStream::json_parse(ParsedJson &pj) { else if (pj.byte_capacity < _batch_size) { return simdjson::CAPACITY; } - - //Quick heuristic to see if it's worth parsing the remaining data in the batch - if(!load_next_batch && n_bytes_parsed > 0) { - const auto remaining_data = _batch_size - current_buffer_loc; - const auto avg_doc_len = (float) n_bytes_parsed / n_parsed_docs; - - if(remaining_data < avg_doc_len) - load_next_batch = true; - } +#ifdef SIMDJSON_THREADS_ENABLED + if(current_buffer_loc == last_json_buffer_loc) + load_next_batch = true; +#endif if (load_next_batch){ +#ifdef SIMDJSON_THREADS_ENABLED + //First time loading + if(!stage_1_thread.joinable()){ + _buf = &_buf[current_buffer_loc]; + _len -= current_buffer_loc; + n_bytes_parsed += current_buffer_loc; + _batch_size = std::min(_batch_size, _len); + int stage1_is_ok = (*best_stage1)(_buf, _batch_size, pj, true); + + if (stage1_is_ok != simdjson::SUCCESS) { + pj.error_code = stage1_is_ok; + return pj.error_code; + } + } + + //the second thread is running or done. + else{ + stage_1_thread.join(); + std::swap(pj.structural_indexes, pj_thread.structural_indexes); + pj.n_structural_indexes = pj_thread.n_structural_indexes; + + _buf = &_buf[last_json_buffer_loc]; + _len -= last_json_buffer_loc; + n_bytes_parsed += last_json_buffer_loc; + last_json_buffer_loc = 0; //because we want to use it in the if above. + } + + if(_len-_batch_size > 0) { + last_json_buffer_loc = find_last_json(pj); + _batch_size = std::min(_batch_size, _len - last_json_buffer_loc); + if(_batch_size>0) + stage_1_thread = std::thread( + static_cast(*best_stage1), + &_buf[last_json_buffer_loc], _batch_size, + std::ref(pj_thread), + true); + + } +#else _buf = &_buf[current_buffer_loc]; _len -= current_buffer_loc; n_bytes_parsed += current_buffer_loc; _batch_size = std::min(_batch_size, _len); - int stage1_is_ok = (*best_stage1)(_buf, _batch_size, pj, true); if (stage1_is_ok != simdjson::SUCCESS) { pj.error_code = stage1_is_ok; return pj.error_code; } - +#endif load_next_batch = false; //If we loaded a perfect amount of documents last time, we need to skip the first element, @@ -36552,7 +36583,9 @@ int JsonStream::json_parse(ParsedJson &pj) { load_next_batch = true; } - else current_buffer_loc = pj.structural_indexes[next_json]; + else { + current_buffer_loc = pj.structural_indexes[next_json]; + } } //TODO: have a more precise error check //Give it two chances for now. We assume the error is because the json was not loaded completely in this batch. @@ -36562,10 +36595,51 @@ int JsonStream::json_parse(ParsedJson &pj) { error_on_last_attempt = true; res = json_parse(pj); } - return res; } +#ifdef SIMDJSON_THREADS_ENABLED +size_t JsonStream::find_last_json(const ParsedJson &pj) { + auto last_i = pj.n_structural_indexes - 1; + if (pj.structural_indexes[last_i] == _batch_size) + last_i = pj.n_structural_indexes - 2; + auto arr_cnt = 0; + auto obj_cnt = 0; + for (auto i = last_i; i > 0; i--) { + auto idxb = pj.structural_indexes[i]; + switch (_buf[idxb]) { + case ':': + case ',': + continue; + case '}': + obj_cnt--; + continue; + case ']': + arr_cnt--; + continue; + case '{': + obj_cnt++; + break; + case '[': + arr_cnt++; + break; + } + auto idxa = pj.structural_indexes[i - 1]; + switch (_buf[idxa]) { + case '{': + case '[': + case ':': + case ',': + continue; + } + if (!arr_cnt && !obj_cnt) + return pj.structural_indexes[last_i+1]; + return idxb; + } + return 0; +} +#endif + size_t JsonStream::get_current_buffer_loc() const { return current_buffer_loc; } @@ -36578,18 +36652,16 @@ size_t JsonStream::get_n_bytes_parsed() const { return n_bytes_parsed; } - //// TODO: generalize this set of functions. We don't want to have a copy in jsonparser.cpp void find_the_best_supported_implementation() { + uint32_t supports = detect_supported_architectures(); + // Order from best to worst (within architecture) +#ifdef IS_X86_64 constexpr uint32_t haswell_flags = instruction_set::AVX2 | instruction_set::PCLMULQDQ | instruction_set::BMI1 | instruction_set::BMI2; constexpr uint32_t westmere_flags = instruction_set::SSE42 | instruction_set::PCLMULQDQ; - - uint32_t supports = detect_supported_architectures(); - // Order from best to worst (within architecture) -#ifdef IS_X86_64 if ((haswell_flags & supports) == haswell_flags) { best_stage1 = simdjson::find_structural_bits; best_stage2 = simdjson::unified_machine; @@ -36938,17 +37010,26 @@ namespace simdjson::arm64::simd { template struct simd8x64 { - const simd8 chunks[4]; + static const int NUM_CHUNKS = 64 / sizeof(simd8); + const simd8 chunks[NUM_CHUNKS]; really_inline simd8x64() : chunks{simd8(), simd8(), simd8(), simd8()} {} really_inline simd8x64(const simd8 chunk0, const simd8 chunk1, const simd8 chunk2, const simd8 chunk3) : chunks{chunk0, chunk1, chunk2, chunk3} {} really_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr), simd8::load(ptr+16), simd8::load(ptr+32), simd8::load(ptr+48)} {} really_inline void store(T ptr[64]) { - this->chunks[0].store(ptr); - this->chunks[0].store(ptr+16); - this->chunks[0].store(ptr+32); - this->chunks[0].store(ptr+48); + this->chunks[0].store(ptr+sizeof(simd8)*0); + this->chunks[1].store(ptr+sizeof(simd8)*1); + this->chunks[2].store(ptr+sizeof(simd8)*2); + this->chunks[3].store(ptr+sizeof(simd8)*3); + } + + template + static really_inline void each_index(F const& each) { + each(0); + each(1); + each(2); + each(3); } template @@ -37222,15 +37303,22 @@ namespace simdjson::haswell::simd { template struct simd8x64 { - const simd8 chunks[2]; + static const int NUM_CHUNKS = 64 / sizeof(simd8); + const simd8 chunks[NUM_CHUNKS]; really_inline simd8x64() : chunks{simd8(), simd8()} {} really_inline simd8x64(const simd8 chunk0, const simd8 chunk1) : chunks{chunk0, chunk1} {} really_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr), simd8::load(ptr+32)} {} - really_inline void store(T *ptr) { - this->chunks[0].store(ptr); - this->chunks[0].store(ptr+sizeof(simd8)); + template + static really_inline void each_index(F const& each) { + each(0); + each(1); + } + + really_inline void store(T ptr[64]) { + this->chunks[0].store(ptr+sizeof(simd8)*0); + this->chunks[1].store(ptr+sizeof(simd8)*1); } template @@ -37478,17 +37566,26 @@ namespace simdjson::westmere::simd { template struct simd8x64 { - const simd8 chunks[4]; + static const int NUM_CHUNKS = 64 / sizeof(simd8); + const simd8 chunks[NUM_CHUNKS]; really_inline simd8x64() : chunks{simd8(), simd8(), simd8(), simd8()} {} really_inline simd8x64(const simd8 chunk0, const simd8 chunk1, const simd8 chunk2, const simd8 chunk3) : chunks{chunk0, chunk1, chunk2, chunk3} {} really_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr), simd8::load(ptr+16), simd8::load(ptr+32), simd8::load(ptr+48)} {} really_inline void store(T ptr[64]) { - this->chunks[0].store(ptr); - this->chunks[0].store(ptr+16); - this->chunks[0].store(ptr+32); - this->chunks[0].store(ptr+48); + this->chunks[0].store(ptr+sizeof(simd8)*0); + this->chunks[1].store(ptr+sizeof(simd8)*1); + this->chunks[2].store(ptr+sizeof(simd8)*2); + this->chunks[3].store(ptr+sizeof(simd8)*3); + } + + template + static really_inline void each_index(F const& each) { + each(0); + each(1); + each(2); + each(3); } template @@ -37768,7 +37865,7 @@ struct utf8_checker { // We assume the file in which it is included already includes // "simdjson/stage1_find_marks.h" (this simplifies amalgation) -static const size_t STEP_SIZE = 128; +namespace stage1 { class bit_indexer { public: @@ -37839,81 +37936,12 @@ public: json_structural_scanner(uint32_t *_structural_indexes) : structural_indexes{_structural_indexes} {} - // return a bitvector indicating where we have characters that end an odd-length - // sequence of backslashes (and thus change the behavior of the next character - // to follow). A even-length sequence of backslashes, and, for that matter, the - // largest even-length prefix of our odd-length sequence of backslashes, simply - // modify the behavior of the backslashes themselves. - // We also update the prev_iter_ends_odd_backslash reference parameter to - // indicate whether we end an iteration on an odd-length sequence of - // backslashes, which modifies our subsequent search for odd-length - // sequences of backslashes in an obvious way. - really_inline uint64_t follows_odd_sequence_of(const uint64_t match, uint64_t &overflow) { - const uint64_t even_bits = 0x5555555555555555ULL; - const uint64_t odd_bits = ~even_bits; - uint64_t start_edges = match & ~(match << 1); - /* flip lowest if we have an odd-length run at the end of the prior - * iteration */ - uint64_t even_start_mask = even_bits ^ overflow; - uint64_t even_starts = start_edges & even_start_mask; - uint64_t odd_starts = start_edges & ~even_start_mask; - uint64_t even_carries = match + even_starts; - - uint64_t odd_carries; - /* must record the carry-out of our odd-carries out of bit 63; this - * indicates whether the sense of any edge going to the next iteration - * should be flipped */ - bool new_overflow = add_overflow(match, odd_starts, &odd_carries); - - odd_carries |= overflow; /* push in bit zero as a - * potential end if we had an - * odd-numbered run at the - * end of the previous - * iteration */ - overflow = new_overflow ? 0x1ULL : 0x0ULL; - uint64_t even_carry_ends = even_carries & ~match; - uint64_t odd_carry_ends = odd_carries & ~match; - uint64_t even_start_odd_end = even_carry_ends & odd_bits; - uint64_t odd_start_even_end = odd_carry_ends & even_bits; - uint64_t odd_ends = even_start_odd_end | odd_start_even_end; - return odd_ends; - } - // - // Check if the current character immediately follows a matching character. + // Finish the scan and return any errors. // - // For example, this checks for quotes with backslashes in front of them: + // This may detect errors as well, such as unclosed string and certain UTF-8 errors. // - // const uint64_t backslashed_quote = in.eq('"') & immediately_follows(in.eq('\'), prev_backslash); - // - really_inline uint64_t follows(const uint64_t match, uint64_t &overflow) { - const uint64_t result = match << 1 | overflow; - overflow = match >> 63; - return result; - } - - // - // Check if the current character follows a matching character, with possible "filler" between. - // For example, this checks for empty curly braces, e.g. - // - // in.eq('}') & follows(in.eq('['), in.eq(' '), prev_empty_array) // { * } - // - really_inline uint64_t follows(const uint64_t match, const uint64_t filler, uint64_t &overflow) { - uint64_t follows_match = follows(match, overflow); - uint64_t result; - overflow |= add_overflow(follows_match, filler, &result); - return result; - } - - really_inline ErrorValues detect_errors_on_eof() { - if (prev_in_string) { - return UNCLOSED_STRING; - } - if (unescaped_chars_error) { - return UNESCAPED_CHARS; - } - return SUCCESS; - } + really_inline ErrorValues detect_errors_on_eof(); // // Return a mask of all string characters plus end quotes. @@ -37923,28 +37951,7 @@ public: // // Backslash sequences outside of quotes will be detected in stage 2. // - really_inline uint64_t find_strings(const simd::simd8x64 in) { - const uint64_t backslash = in.eq('\\'); - const uint64_t escaped = follows_odd_sequence_of(backslash, prev_escaped); - const uint64_t quote = in.eq('"') & ~escaped; - // prefix_xor flips on bits inside the string (and flips off the end quote). - const uint64_t in_string = prefix_xor(quote) ^ prev_in_string; - /* right shift of a signed value expected to be well-defined and standard - * compliant as of C++20, - * John Regher from Utah U. says this is fine code */ - prev_in_string = static_cast(static_cast(in_string) >> 63); - // Use ^ to turn the beginning quote off, and the end quote on. - return in_string ^ quote; - } - - really_inline uint64_t invalid_string_bytes(const uint64_t unescaped, const uint64_t quote_mask) { - /* All Unicode characters may be placed within the - * quotation marks, except for the characters that MUST be escaped: - * quotation mark, reverse solidus, and the control characters (U+0000 - * through U+001F). - * https://tools.ietf.org/html/rfc8259 */ - return quote_mask & unescaped; - } + really_inline uint64_t find_strings(const simd::simd8x64 in); // // Determine which characters are *structural*: @@ -37962,101 +37969,262 @@ public: // contents of a string the same as content outside. Errors and structurals inside the string or on // the trailing quote will need to be removed later when the correct string information is known. // - really_inline uint64_t find_potential_structurals(const simd::simd8x64 in) { - // These use SIMD so let's kick them off before running the regular 64-bit stuff ... - uint64_t whitespace, op; - find_whitespace_and_operators(in, whitespace, op); - - // Detect the start of a run of primitive characters. Includes numbers, booleans, and strings ("). - // Everything except whitespace, braces, colon and comma. - const uint64_t primitive = ~(op | whitespace); - const uint64_t follows_primitive = follows(primitive, prev_primitive); - const uint64_t start_primitive = primitive & ~follows_primitive; - - // Return final structurals - return op | start_primitive; - } + really_inline uint64_t find_potential_structurals(const simd::simd8x64 in); // - // Find the important bits of JSON in a 128-byte chunk, and add them to structural_indexes. + // Find the important bits of JSON in a STEP_SIZE-byte chunk, and add them to structural_indexes. // - // PERF NOTES: - // We pipe 2 inputs through these stages: - // 1. Load JSON into registers. This takes a long time and is highly parallelizable, so we load - // 2 inputs' worth at once so that by the time step 2 is looking for them input, it's available. - // 2. Scan the JSON for critical data: strings, primitives and operators. This is the critical path. - // The output of step 1 depends entirely on this information. These functions don't quite use - // up enough CPU: the second half of the functions is highly serial, only using 1 execution core - // at a time. The second input's scans has some dependency on the first ones finishing it, but - // they can make a lot of progress before they need that information. - // 3. Step 1 doesn't use enough capacity, so we run some extra stuff while we're waiting for that - // to finish: utf-8 checks and generating the output from the last iteration. - // - // The reason we run 2 inputs at a time, is steps 2 and 3 are *still* not enough to soak up all - // available capacity with just one input. Running 2 at a time seems to give the CPU a good enough - // workout. + template + really_inline void scan_step(const uint8_t *buf, const size_t idx, utf8_checker &utf8_checker); + // - really_inline void scan_step(const uint8_t *buf, const size_t idx, utf8_checker &utf8_checker) { - // - // Load up all 128 bytes into SIMD registers - // - simd::simd8x64 in_1(buf); - simd::simd8x64 in_2(buf+64); - - // - // Find the strings and potential structurals (operators / primitives). - // - // This will include false structurals that are *inside* strings--we'll filter strings out - // before we return. - // - uint64_t string_1 = this->find_strings(in_1); - uint64_t structurals_1 = this->find_potential_structurals(in_1); - uint64_t string_2 = this->find_strings(in_2); - uint64_t structurals_2 = this->find_potential_structurals(in_2); - - // - // Do miscellaneous work while the processor is busy calculating strings and structurals. - // - // After that, weed out structurals that are inside strings and find invalid string characters. - // - uint64_t unescaped_1 = in_1.lteq(0x1F); - utf8_checker.check_next_input(in_1); - this->structural_indexes.write_indexes(idx-64, prev_structurals); // Output *last* iteration's structurals to ParsedJson - this->prev_structurals = structurals_1 & ~string_1; - this->unescaped_chars_error |= unescaped_1 & string_1; - - uint64_t unescaped_2 = in_2.lteq(0x1F); - utf8_checker.check_next_input(in_2); - this->structural_indexes.write_indexes(idx, prev_structurals); // Output *last* iteration's structurals to ParsedJson - this->prev_structurals = structurals_2 & ~string_2; - this->unescaped_chars_error |= unescaped_2 & string_2; - } - - really_inline void scan(const uint8_t *buf, const size_t len, utf8_checker &utf8_checker) { - size_t lenminusstep = len < STEP_SIZE ? 0 : len - STEP_SIZE; - size_t idx = 0; - - for (; idx < lenminusstep; idx += STEP_SIZE) { - this->scan_step(&buf[idx], idx, utf8_checker); - } - - /* If we have a final chunk of less than 64 bytes, pad it to 64 with - * spaces before processing it (otherwise, we risk invalidating the UTF-8 - * checks). */ - if (likely(idx < len)) { - uint8_t tmp_buf[STEP_SIZE]; - memset(tmp_buf, 0x20, STEP_SIZE); - memcpy(tmp_buf, buf + idx, len - idx); - this->scan_step(&tmp_buf[0], idx, utf8_checker); - idx += STEP_SIZE; - } - - /* finally, flatten out the remaining structurals from the last iteration */ - this->structural_indexes.write_indexes(idx-64, this->prev_structurals); - } - + // Parse the entire input in STEP_SIZE-byte chunks. + // + template + really_inline void scan(const uint8_t *buf, const size_t len, utf8_checker &utf8_checker); }; +// return a bitvector indicating where we have characters that end an odd-length +// sequence of backslashes (and thus change the behavior of the next character +// to follow). A even-length sequence of backslashes, and, for that matter, the +// largest even-length prefix of our odd-length sequence of backslashes, simply +// modify the behavior of the backslashes themselves. +// We also update the prev_iter_ends_odd_backslash reference parameter to +// indicate whether we end an iteration on an odd-length sequence of +// backslashes, which modifies our subsequent search for odd-length +// sequences of backslashes in an obvious way. +really_inline uint64_t follows_odd_sequence_of(const uint64_t match, uint64_t &overflow) { + const uint64_t even_bits = 0x5555555555555555ULL; + const uint64_t odd_bits = ~even_bits; + uint64_t start_edges = match & ~(match << 1); + /* flip lowest if we have an odd-length run at the end of the prior + * iteration */ + uint64_t even_start_mask = even_bits ^ overflow; + uint64_t even_starts = start_edges & even_start_mask; + uint64_t odd_starts = start_edges & ~even_start_mask; + uint64_t even_carries = match + even_starts; + + uint64_t odd_carries; + /* must record the carry-out of our odd-carries out of bit 63; this + * indicates whether the sense of any edge going to the next iteration + * should be flipped */ + bool new_overflow = add_overflow(match, odd_starts, &odd_carries); + + odd_carries |= overflow; /* push in bit zero as a + * potential end if we had an + * odd-numbered run at the + * end of the previous + * iteration */ + overflow = new_overflow ? 0x1ULL : 0x0ULL; + uint64_t even_carry_ends = even_carries & ~match; + uint64_t odd_carry_ends = odd_carries & ~match; + uint64_t even_start_odd_end = even_carry_ends & odd_bits; + uint64_t odd_start_even_end = odd_carry_ends & even_bits; + uint64_t odd_ends = even_start_odd_end | odd_start_even_end; + return odd_ends; +} + +// +// Check if the current character immediately follows a matching character. +// +// For example, this checks for quotes with backslashes in front of them: +// +// const uint64_t backslashed_quote = in.eq('"') & immediately_follows(in.eq('\'), prev_backslash); +// +really_inline uint64_t follows(const uint64_t match, uint64_t &overflow) { + const uint64_t result = match << 1 | overflow; + overflow = match >> 63; + return result; +} + +// +// Check if the current character follows a matching character, with possible "filler" between. +// For example, this checks for empty curly braces, e.g. +// +// in.eq('}') & follows(in.eq('['), in.eq(' '), prev_empty_array) // { * } +// +really_inline uint64_t follows(const uint64_t match, const uint64_t filler, uint64_t &overflow) { + uint64_t follows_match = follows(match, overflow); + uint64_t result; + overflow |= add_overflow(follows_match, filler, &result); + return result; +} + +really_inline ErrorValues json_structural_scanner::detect_errors_on_eof() { + if (prev_in_string) { + return UNCLOSED_STRING; + } + if (unescaped_chars_error) { + return UNESCAPED_CHARS; + } + return SUCCESS; +} + +// +// Return a mask of all string characters plus end quotes. +// +// prev_escaped is overflow saying whether the next character is escaped. +// prev_in_string is overflow saying whether we're still in a string. +// +// Backslash sequences outside of quotes will be detected in stage 2. +// +really_inline uint64_t json_structural_scanner::find_strings(const simd::simd8x64 in) { + const uint64_t backslash = in.eq('\\'); + const uint64_t escaped = follows_odd_sequence_of(backslash, prev_escaped); + const uint64_t quote = in.eq('"') & ~escaped; + // prefix_xor flips on bits inside the string (and flips off the end quote). + const uint64_t in_string = prefix_xor(quote) ^ prev_in_string; + /* right shift of a signed value expected to be well-defined and standard + * compliant as of C++20, + * John Regher from Utah U. says this is fine code */ + prev_in_string = static_cast(static_cast(in_string) >> 63); + // Use ^ to turn the beginning quote off, and the end quote on. + return in_string ^ quote; +} + +// +// Determine which characters are *structural*: +// - braces: [] and {} +// - the start of primitives (123, true, false, null) +// - the start of invalid non-whitespace (+, &, ture, UTF-8) +// +// Also detects value sequence errors: +// - two values with no separator between ("hello" "world") +// - separators with no values ([1,] [1,,]and [,2]) +// +// This method will find all of the above whether it is in a string or not. +// +// To reduce dependency on the expensive "what is in a string" computation, this method treats the +// contents of a string the same as content outside. Errors and structurals inside the string or on +// the trailing quote will need to be removed later when the correct string information is known. +// +really_inline uint64_t json_structural_scanner::find_potential_structurals(const simd::simd8x64 in) { + // These use SIMD so let's kick them off before running the regular 64-bit stuff ... + uint64_t whitespace, op; + find_whitespace_and_operators(in, whitespace, op); + + // Detect the start of a run of primitive characters. Includes numbers, booleans, and strings ("). + // Everything except whitespace, braces, colon and comma. + const uint64_t primitive = ~(op | whitespace); + const uint64_t follows_primitive = follows(primitive, prev_primitive); + const uint64_t start_primitive = primitive & ~follows_primitive; + + // Return final structurals + return op | start_primitive; +} + +// +// Find the important bits of JSON in a 128-byte chunk, and add them to structural_indexes. +// +// PERF NOTES: +// We pipe 2 inputs through these stages: +// 1. Load JSON into registers. This takes a long time and is highly parallelizable, so we load +// 2 inputs' worth at once so that by the time step 2 is looking for them input, it's available. +// 2. Scan the JSON for critical data: strings, primitives and operators. This is the critical path. +// The output of step 1 depends entirely on this information. These functions don't quite use +// up enough CPU: the second half of the functions is highly serial, only using 1 execution core +// at a time. The second input's scans has some dependency on the first ones finishing it, but +// they can make a lot of progress before they need that information. +// 3. Step 1 doesn't use enough capacity, so we run some extra stuff while we're waiting for that +// to finish: utf-8 checks and generating the output from the last iteration. +// +// The reason we run 2 inputs at a time, is steps 2 and 3 are *still* not enough to soak up all +// available capacity with just one input. Running 2 at a time seems to give the CPU a good enough +// workout. +// +template<> +really_inline void json_structural_scanner::scan_step<128>(const uint8_t *buf, const size_t idx, utf8_checker &utf8_checker) { + // + // Load up all 128 bytes into SIMD registers + // + simd::simd8x64 in_1(buf); + simd::simd8x64 in_2(buf+64); + + // + // Find the strings and potential structurals (operators / primitives). + // + // This will include false structurals that are *inside* strings--we'll filter strings out + // before we return. + // + uint64_t string_1 = this->find_strings(in_1); + uint64_t structurals_1 = this->find_potential_structurals(in_1); + uint64_t string_2 = this->find_strings(in_2); + uint64_t structurals_2 = this->find_potential_structurals(in_2); + + // + // Do miscellaneous work while the processor is busy calculating strings and structurals. + // + // After that, weed out structurals that are inside strings and find invalid string characters. + // + uint64_t unescaped_1 = in_1.lteq(0x1F); + utf8_checker.check_next_input(in_1); + this->structural_indexes.write_indexes(idx-64, this->prev_structurals); // Output *last* iteration's structurals to ParsedJson + this->prev_structurals = structurals_1 & ~string_1; + this->unescaped_chars_error |= unescaped_1 & string_1; + + uint64_t unescaped_2 = in_2.lteq(0x1F); + utf8_checker.check_next_input(in_2); + this->structural_indexes.write_indexes(idx, this->prev_structurals); // Output *last* iteration's structurals to ParsedJson + this->prev_structurals = structurals_2 & ~string_2; + this->unescaped_chars_error |= unescaped_2 & string_2; +} + +// +// Find the important bits of JSON in a 64-byte chunk, and add them to structural_indexes. +// +template<> +really_inline void json_structural_scanner::scan_step<64>(const uint8_t *buf, const size_t idx, utf8_checker &utf8_checker) { + // + // Load up bytes into SIMD registers + // + simd::simd8x64 in_1(buf); + + // + // Find the strings and potential structurals (operators / primitives). + // + // This will include false structurals that are *inside* strings--we'll filter strings out + // before we return. + // + uint64_t string_1 = this->find_strings(in_1); + uint64_t structurals_1 = this->find_potential_structurals(in_1); + + // + // Do miscellaneous work while the processor is busy calculating strings and structurals. + // + // After that, weed out structurals that are inside strings and find invalid string characters. + // + uint64_t unescaped_1 = in_1.lteq(0x1F); + utf8_checker.check_next_input(in_1); + this->structural_indexes.write_indexes(idx-64, this->prev_structurals); // Output *last* iteration's structurals to ParsedJson + this->prev_structurals = structurals_1 & ~string_1; + this->unescaped_chars_error |= unescaped_1 & string_1; +} + +template +really_inline void json_structural_scanner::scan(const uint8_t *buf, const size_t len, utf8_checker &utf8_checker) { + size_t lenminusstep = len < STEP_SIZE ? 0 : len - STEP_SIZE; + size_t idx = 0; + + for (; idx < lenminusstep; idx += STEP_SIZE) { + this->scan_step(&buf[idx], idx, utf8_checker); + } + + /* If we have a final chunk of less than 64 bytes, pad it to 64 with + * spaces before processing it (otherwise, we risk invalidating the UTF-8 + * checks). */ + if (likely(idx < len)) { + uint8_t tmp_buf[STEP_SIZE]; + memset(tmp_buf, 0x20, STEP_SIZE); + memcpy(tmp_buf, buf + idx, len - idx); + this->scan_step(&tmp_buf[0], idx, utf8_checker); + idx += STEP_SIZE; + } + + /* finally, flatten out the remaining structurals from the last iteration */ + this->structural_indexes.write_indexes(idx-64, this->prev_structurals); +} + +template int find_structural_bits(const uint8_t *buf, size_t len, simdjson::ParsedJson &pj, bool streaming) { if (unlikely(len > pj.byte_capacity)) { std::cerr << "Your ParsedJson object only supports documents up to " @@ -38066,7 +38234,7 @@ int find_structural_bits(const uint8_t *buf, size_t len, simdjson::ParsedJson &p } utf8_checker utf8_checker{}; json_structural_scanner scanner{pj.structural_indexes}; - scanner.scan(buf, len, utf8_checker); + scanner.scan(buf, len, utf8_checker); simdjson::ErrorValues error = scanner.detect_errors_on_eof(); if (!streaming && unlikely(error != simdjson::SUCCESS)) { @@ -38092,13 +38260,15 @@ int find_structural_bits(const uint8_t *buf, size_t len, simdjson::ParsedJson &p return utf8_checker.errors(); } +} // namespace stage1 + } // namespace simdjson::arm64 namespace simdjson { template <> int find_structural_bits(const uint8_t *buf, size_t len, simdjson::ParsedJson &pj, bool streaming) { - return arm64::find_structural_bits(buf, len, pj, streaming); + return arm64::stage1::find_structural_bits<64>(buf, len, pj, streaming); } } // namespace simdjson @@ -38313,7 +38483,7 @@ struct utf8_checker { // We assume the file in which it is included already includes // "simdjson/stage1_find_marks.h" (this simplifies amalgation) -static const size_t STEP_SIZE = 128; +namespace stage1 { class bit_indexer { public: @@ -38384,81 +38554,12 @@ public: json_structural_scanner(uint32_t *_structural_indexes) : structural_indexes{_structural_indexes} {} - // return a bitvector indicating where we have characters that end an odd-length - // sequence of backslashes (and thus change the behavior of the next character - // to follow). A even-length sequence of backslashes, and, for that matter, the - // largest even-length prefix of our odd-length sequence of backslashes, simply - // modify the behavior of the backslashes themselves. - // We also update the prev_iter_ends_odd_backslash reference parameter to - // indicate whether we end an iteration on an odd-length sequence of - // backslashes, which modifies our subsequent search for odd-length - // sequences of backslashes in an obvious way. - really_inline uint64_t follows_odd_sequence_of(const uint64_t match, uint64_t &overflow) { - const uint64_t even_bits = 0x5555555555555555ULL; - const uint64_t odd_bits = ~even_bits; - uint64_t start_edges = match & ~(match << 1); - /* flip lowest if we have an odd-length run at the end of the prior - * iteration */ - uint64_t even_start_mask = even_bits ^ overflow; - uint64_t even_starts = start_edges & even_start_mask; - uint64_t odd_starts = start_edges & ~even_start_mask; - uint64_t even_carries = match + even_starts; - - uint64_t odd_carries; - /* must record the carry-out of our odd-carries out of bit 63; this - * indicates whether the sense of any edge going to the next iteration - * should be flipped */ - bool new_overflow = add_overflow(match, odd_starts, &odd_carries); - - odd_carries |= overflow; /* push in bit zero as a - * potential end if we had an - * odd-numbered run at the - * end of the previous - * iteration */ - overflow = new_overflow ? 0x1ULL : 0x0ULL; - uint64_t even_carry_ends = even_carries & ~match; - uint64_t odd_carry_ends = odd_carries & ~match; - uint64_t even_start_odd_end = even_carry_ends & odd_bits; - uint64_t odd_start_even_end = odd_carry_ends & even_bits; - uint64_t odd_ends = even_start_odd_end | odd_start_even_end; - return odd_ends; - } - // - // Check if the current character immediately follows a matching character. + // Finish the scan and return any errors. // - // For example, this checks for quotes with backslashes in front of them: + // This may detect errors as well, such as unclosed string and certain UTF-8 errors. // - // const uint64_t backslashed_quote = in.eq('"') & immediately_follows(in.eq('\'), prev_backslash); - // - really_inline uint64_t follows(const uint64_t match, uint64_t &overflow) { - const uint64_t result = match << 1 | overflow; - overflow = match >> 63; - return result; - } - - // - // Check if the current character follows a matching character, with possible "filler" between. - // For example, this checks for empty curly braces, e.g. - // - // in.eq('}') & follows(in.eq('['), in.eq(' '), prev_empty_array) // { * } - // - really_inline uint64_t follows(const uint64_t match, const uint64_t filler, uint64_t &overflow) { - uint64_t follows_match = follows(match, overflow); - uint64_t result; - overflow |= add_overflow(follows_match, filler, &result); - return result; - } - - really_inline ErrorValues detect_errors_on_eof() { - if (prev_in_string) { - return UNCLOSED_STRING; - } - if (unescaped_chars_error) { - return UNESCAPED_CHARS; - } - return SUCCESS; - } + really_inline ErrorValues detect_errors_on_eof(); // // Return a mask of all string characters plus end quotes. @@ -38468,28 +38569,7 @@ public: // // Backslash sequences outside of quotes will be detected in stage 2. // - really_inline uint64_t find_strings(const simd::simd8x64 in) { - const uint64_t backslash = in.eq('\\'); - const uint64_t escaped = follows_odd_sequence_of(backslash, prev_escaped); - const uint64_t quote = in.eq('"') & ~escaped; - // prefix_xor flips on bits inside the string (and flips off the end quote). - const uint64_t in_string = prefix_xor(quote) ^ prev_in_string; - /* right shift of a signed value expected to be well-defined and standard - * compliant as of C++20, - * John Regher from Utah U. says this is fine code */ - prev_in_string = static_cast(static_cast(in_string) >> 63); - // Use ^ to turn the beginning quote off, and the end quote on. - return in_string ^ quote; - } - - really_inline uint64_t invalid_string_bytes(const uint64_t unescaped, const uint64_t quote_mask) { - /* All Unicode characters may be placed within the - * quotation marks, except for the characters that MUST be escaped: - * quotation mark, reverse solidus, and the control characters (U+0000 - * through U+001F). - * https://tools.ietf.org/html/rfc8259 */ - return quote_mask & unescaped; - } + really_inline uint64_t find_strings(const simd::simd8x64 in); // // Determine which characters are *structural*: @@ -38507,101 +38587,262 @@ public: // contents of a string the same as content outside. Errors and structurals inside the string or on // the trailing quote will need to be removed later when the correct string information is known. // - really_inline uint64_t find_potential_structurals(const simd::simd8x64 in) { - // These use SIMD so let's kick them off before running the regular 64-bit stuff ... - uint64_t whitespace, op; - find_whitespace_and_operators(in, whitespace, op); - - // Detect the start of a run of primitive characters. Includes numbers, booleans, and strings ("). - // Everything except whitespace, braces, colon and comma. - const uint64_t primitive = ~(op | whitespace); - const uint64_t follows_primitive = follows(primitive, prev_primitive); - const uint64_t start_primitive = primitive & ~follows_primitive; - - // Return final structurals - return op | start_primitive; - } + really_inline uint64_t find_potential_structurals(const simd::simd8x64 in); // - // Find the important bits of JSON in a 128-byte chunk, and add them to structural_indexes. + // Find the important bits of JSON in a STEP_SIZE-byte chunk, and add them to structural_indexes. // - // PERF NOTES: - // We pipe 2 inputs through these stages: - // 1. Load JSON into registers. This takes a long time and is highly parallelizable, so we load - // 2 inputs' worth at once so that by the time step 2 is looking for them input, it's available. - // 2. Scan the JSON for critical data: strings, primitives and operators. This is the critical path. - // The output of step 1 depends entirely on this information. These functions don't quite use - // up enough CPU: the second half of the functions is highly serial, only using 1 execution core - // at a time. The second input's scans has some dependency on the first ones finishing it, but - // they can make a lot of progress before they need that information. - // 3. Step 1 doesn't use enough capacity, so we run some extra stuff while we're waiting for that - // to finish: utf-8 checks and generating the output from the last iteration. - // - // The reason we run 2 inputs at a time, is steps 2 and 3 are *still* not enough to soak up all - // available capacity with just one input. Running 2 at a time seems to give the CPU a good enough - // workout. + template + really_inline void scan_step(const uint8_t *buf, const size_t idx, utf8_checker &utf8_checker); + // - really_inline void scan_step(const uint8_t *buf, const size_t idx, utf8_checker &utf8_checker) { - // - // Load up all 128 bytes into SIMD registers - // - simd::simd8x64 in_1(buf); - simd::simd8x64 in_2(buf+64); - - // - // Find the strings and potential structurals (operators / primitives). - // - // This will include false structurals that are *inside* strings--we'll filter strings out - // before we return. - // - uint64_t string_1 = this->find_strings(in_1); - uint64_t structurals_1 = this->find_potential_structurals(in_1); - uint64_t string_2 = this->find_strings(in_2); - uint64_t structurals_2 = this->find_potential_structurals(in_2); - - // - // Do miscellaneous work while the processor is busy calculating strings and structurals. - // - // After that, weed out structurals that are inside strings and find invalid string characters. - // - uint64_t unescaped_1 = in_1.lteq(0x1F); - utf8_checker.check_next_input(in_1); - this->structural_indexes.write_indexes(idx-64, prev_structurals); // Output *last* iteration's structurals to ParsedJson - this->prev_structurals = structurals_1 & ~string_1; - this->unescaped_chars_error |= unescaped_1 & string_1; - - uint64_t unescaped_2 = in_2.lteq(0x1F); - utf8_checker.check_next_input(in_2); - this->structural_indexes.write_indexes(idx, prev_structurals); // Output *last* iteration's structurals to ParsedJson - this->prev_structurals = structurals_2 & ~string_2; - this->unescaped_chars_error |= unescaped_2 & string_2; - } - - really_inline void scan(const uint8_t *buf, const size_t len, utf8_checker &utf8_checker) { - size_t lenminusstep = len < STEP_SIZE ? 0 : len - STEP_SIZE; - size_t idx = 0; - - for (; idx < lenminusstep; idx += STEP_SIZE) { - this->scan_step(&buf[idx], idx, utf8_checker); - } - - /* If we have a final chunk of less than 64 bytes, pad it to 64 with - * spaces before processing it (otherwise, we risk invalidating the UTF-8 - * checks). */ - if (likely(idx < len)) { - uint8_t tmp_buf[STEP_SIZE]; - memset(tmp_buf, 0x20, STEP_SIZE); - memcpy(tmp_buf, buf + idx, len - idx); - this->scan_step(&tmp_buf[0], idx, utf8_checker); - idx += STEP_SIZE; - } - - /* finally, flatten out the remaining structurals from the last iteration */ - this->structural_indexes.write_indexes(idx-64, this->prev_structurals); - } - + // Parse the entire input in STEP_SIZE-byte chunks. + // + template + really_inline void scan(const uint8_t *buf, const size_t len, utf8_checker &utf8_checker); }; +// return a bitvector indicating where we have characters that end an odd-length +// sequence of backslashes (and thus change the behavior of the next character +// to follow). A even-length sequence of backslashes, and, for that matter, the +// largest even-length prefix of our odd-length sequence of backslashes, simply +// modify the behavior of the backslashes themselves. +// We also update the prev_iter_ends_odd_backslash reference parameter to +// indicate whether we end an iteration on an odd-length sequence of +// backslashes, which modifies our subsequent search for odd-length +// sequences of backslashes in an obvious way. +really_inline uint64_t follows_odd_sequence_of(const uint64_t match, uint64_t &overflow) { + const uint64_t even_bits = 0x5555555555555555ULL; + const uint64_t odd_bits = ~even_bits; + uint64_t start_edges = match & ~(match << 1); + /* flip lowest if we have an odd-length run at the end of the prior + * iteration */ + uint64_t even_start_mask = even_bits ^ overflow; + uint64_t even_starts = start_edges & even_start_mask; + uint64_t odd_starts = start_edges & ~even_start_mask; + uint64_t even_carries = match + even_starts; + + uint64_t odd_carries; + /* must record the carry-out of our odd-carries out of bit 63; this + * indicates whether the sense of any edge going to the next iteration + * should be flipped */ + bool new_overflow = add_overflow(match, odd_starts, &odd_carries); + + odd_carries |= overflow; /* push in bit zero as a + * potential end if we had an + * odd-numbered run at the + * end of the previous + * iteration */ + overflow = new_overflow ? 0x1ULL : 0x0ULL; + uint64_t even_carry_ends = even_carries & ~match; + uint64_t odd_carry_ends = odd_carries & ~match; + uint64_t even_start_odd_end = even_carry_ends & odd_bits; + uint64_t odd_start_even_end = odd_carry_ends & even_bits; + uint64_t odd_ends = even_start_odd_end | odd_start_even_end; + return odd_ends; +} + +// +// Check if the current character immediately follows a matching character. +// +// For example, this checks for quotes with backslashes in front of them: +// +// const uint64_t backslashed_quote = in.eq('"') & immediately_follows(in.eq('\'), prev_backslash); +// +really_inline uint64_t follows(const uint64_t match, uint64_t &overflow) { + const uint64_t result = match << 1 | overflow; + overflow = match >> 63; + return result; +} + +// +// Check if the current character follows a matching character, with possible "filler" between. +// For example, this checks for empty curly braces, e.g. +// +// in.eq('}') & follows(in.eq('['), in.eq(' '), prev_empty_array) // { * } +// +really_inline uint64_t follows(const uint64_t match, const uint64_t filler, uint64_t &overflow) { + uint64_t follows_match = follows(match, overflow); + uint64_t result; + overflow |= add_overflow(follows_match, filler, &result); + return result; +} + +really_inline ErrorValues json_structural_scanner::detect_errors_on_eof() { + if (prev_in_string) { + return UNCLOSED_STRING; + } + if (unescaped_chars_error) { + return UNESCAPED_CHARS; + } + return SUCCESS; +} + +// +// Return a mask of all string characters plus end quotes. +// +// prev_escaped is overflow saying whether the next character is escaped. +// prev_in_string is overflow saying whether we're still in a string. +// +// Backslash sequences outside of quotes will be detected in stage 2. +// +really_inline uint64_t json_structural_scanner::find_strings(const simd::simd8x64 in) { + const uint64_t backslash = in.eq('\\'); + const uint64_t escaped = follows_odd_sequence_of(backslash, prev_escaped); + const uint64_t quote = in.eq('"') & ~escaped; + // prefix_xor flips on bits inside the string (and flips off the end quote). + const uint64_t in_string = prefix_xor(quote) ^ prev_in_string; + /* right shift of a signed value expected to be well-defined and standard + * compliant as of C++20, + * John Regher from Utah U. says this is fine code */ + prev_in_string = static_cast(static_cast(in_string) >> 63); + // Use ^ to turn the beginning quote off, and the end quote on. + return in_string ^ quote; +} + +// +// Determine which characters are *structural*: +// - braces: [] and {} +// - the start of primitives (123, true, false, null) +// - the start of invalid non-whitespace (+, &, ture, UTF-8) +// +// Also detects value sequence errors: +// - two values with no separator between ("hello" "world") +// - separators with no values ([1,] [1,,]and [,2]) +// +// This method will find all of the above whether it is in a string or not. +// +// To reduce dependency on the expensive "what is in a string" computation, this method treats the +// contents of a string the same as content outside. Errors and structurals inside the string or on +// the trailing quote will need to be removed later when the correct string information is known. +// +really_inline uint64_t json_structural_scanner::find_potential_structurals(const simd::simd8x64 in) { + // These use SIMD so let's kick them off before running the regular 64-bit stuff ... + uint64_t whitespace, op; + find_whitespace_and_operators(in, whitespace, op); + + // Detect the start of a run of primitive characters. Includes numbers, booleans, and strings ("). + // Everything except whitespace, braces, colon and comma. + const uint64_t primitive = ~(op | whitespace); + const uint64_t follows_primitive = follows(primitive, prev_primitive); + const uint64_t start_primitive = primitive & ~follows_primitive; + + // Return final structurals + return op | start_primitive; +} + +// +// Find the important bits of JSON in a 128-byte chunk, and add them to structural_indexes. +// +// PERF NOTES: +// We pipe 2 inputs through these stages: +// 1. Load JSON into registers. This takes a long time and is highly parallelizable, so we load +// 2 inputs' worth at once so that by the time step 2 is looking for them input, it's available. +// 2. Scan the JSON for critical data: strings, primitives and operators. This is the critical path. +// The output of step 1 depends entirely on this information. These functions don't quite use +// up enough CPU: the second half of the functions is highly serial, only using 1 execution core +// at a time. The second input's scans has some dependency on the first ones finishing it, but +// they can make a lot of progress before they need that information. +// 3. Step 1 doesn't use enough capacity, so we run some extra stuff while we're waiting for that +// to finish: utf-8 checks and generating the output from the last iteration. +// +// The reason we run 2 inputs at a time, is steps 2 and 3 are *still* not enough to soak up all +// available capacity with just one input. Running 2 at a time seems to give the CPU a good enough +// workout. +// +template<> +really_inline void json_structural_scanner::scan_step<128>(const uint8_t *buf, const size_t idx, utf8_checker &utf8_checker) { + // + // Load up all 128 bytes into SIMD registers + // + simd::simd8x64 in_1(buf); + simd::simd8x64 in_2(buf+64); + + // + // Find the strings and potential structurals (operators / primitives). + // + // This will include false structurals that are *inside* strings--we'll filter strings out + // before we return. + // + uint64_t string_1 = this->find_strings(in_1); + uint64_t structurals_1 = this->find_potential_structurals(in_1); + uint64_t string_2 = this->find_strings(in_2); + uint64_t structurals_2 = this->find_potential_structurals(in_2); + + // + // Do miscellaneous work while the processor is busy calculating strings and structurals. + // + // After that, weed out structurals that are inside strings and find invalid string characters. + // + uint64_t unescaped_1 = in_1.lteq(0x1F); + utf8_checker.check_next_input(in_1); + this->structural_indexes.write_indexes(idx-64, this->prev_structurals); // Output *last* iteration's structurals to ParsedJson + this->prev_structurals = structurals_1 & ~string_1; + this->unescaped_chars_error |= unescaped_1 & string_1; + + uint64_t unescaped_2 = in_2.lteq(0x1F); + utf8_checker.check_next_input(in_2); + this->structural_indexes.write_indexes(idx, this->prev_structurals); // Output *last* iteration's structurals to ParsedJson + this->prev_structurals = structurals_2 & ~string_2; + this->unescaped_chars_error |= unescaped_2 & string_2; +} + +// +// Find the important bits of JSON in a 64-byte chunk, and add them to structural_indexes. +// +template<> +really_inline void json_structural_scanner::scan_step<64>(const uint8_t *buf, const size_t idx, utf8_checker &utf8_checker) { + // + // Load up bytes into SIMD registers + // + simd::simd8x64 in_1(buf); + + // + // Find the strings and potential structurals (operators / primitives). + // + // This will include false structurals that are *inside* strings--we'll filter strings out + // before we return. + // + uint64_t string_1 = this->find_strings(in_1); + uint64_t structurals_1 = this->find_potential_structurals(in_1); + + // + // Do miscellaneous work while the processor is busy calculating strings and structurals. + // + // After that, weed out structurals that are inside strings and find invalid string characters. + // + uint64_t unescaped_1 = in_1.lteq(0x1F); + utf8_checker.check_next_input(in_1); + this->structural_indexes.write_indexes(idx-64, this->prev_structurals); // Output *last* iteration's structurals to ParsedJson + this->prev_structurals = structurals_1 & ~string_1; + this->unescaped_chars_error |= unescaped_1 & string_1; +} + +template +really_inline void json_structural_scanner::scan(const uint8_t *buf, const size_t len, utf8_checker &utf8_checker) { + size_t lenminusstep = len < STEP_SIZE ? 0 : len - STEP_SIZE; + size_t idx = 0; + + for (; idx < lenminusstep; idx += STEP_SIZE) { + this->scan_step(&buf[idx], idx, utf8_checker); + } + + /* If we have a final chunk of less than 64 bytes, pad it to 64 with + * spaces before processing it (otherwise, we risk invalidating the UTF-8 + * checks). */ + if (likely(idx < len)) { + uint8_t tmp_buf[STEP_SIZE]; + memset(tmp_buf, 0x20, STEP_SIZE); + memcpy(tmp_buf, buf + idx, len - idx); + this->scan_step(&tmp_buf[0], idx, utf8_checker); + idx += STEP_SIZE; + } + + /* finally, flatten out the remaining structurals from the last iteration */ + this->structural_indexes.write_indexes(idx-64, this->prev_structurals); +} + +template int find_structural_bits(const uint8_t *buf, size_t len, simdjson::ParsedJson &pj, bool streaming) { if (unlikely(len > pj.byte_capacity)) { std::cerr << "Your ParsedJson object only supports documents up to " @@ -38611,7 +38852,7 @@ int find_structural_bits(const uint8_t *buf, size_t len, simdjson::ParsedJson &p } utf8_checker utf8_checker{}; json_structural_scanner scanner{pj.structural_indexes}; - scanner.scan(buf, len, utf8_checker); + scanner.scan(buf, len, utf8_checker); simdjson::ErrorValues error = scanner.detect_errors_on_eof(); if (!streaming && unlikely(error != simdjson::SUCCESS)) { @@ -38637,6 +38878,8 @@ int find_structural_bits(const uint8_t *buf, size_t len, simdjson::ParsedJson &p return utf8_checker.errors(); } +} // namespace stage1 + } // namespace haswell UNTARGET_REGION @@ -38645,7 +38888,7 @@ namespace simdjson { template <> int find_structural_bits(const uint8_t *buf, size_t len, simdjson::ParsedJson &pj, bool streaming) { - return haswell::find_structural_bits(buf, len, pj, streaming); + return haswell::stage1::find_structural_bits<128>(buf, len, pj, streaming); } } // namespace simdjson @@ -38861,7 +39104,7 @@ struct utf8_checker { // We assume the file in which it is included already includes // "simdjson/stage1_find_marks.h" (this simplifies amalgation) -static const size_t STEP_SIZE = 128; +namespace stage1 { class bit_indexer { public: @@ -38932,81 +39175,12 @@ public: json_structural_scanner(uint32_t *_structural_indexes) : structural_indexes{_structural_indexes} {} - // return a bitvector indicating where we have characters that end an odd-length - // sequence of backslashes (and thus change the behavior of the next character - // to follow). A even-length sequence of backslashes, and, for that matter, the - // largest even-length prefix of our odd-length sequence of backslashes, simply - // modify the behavior of the backslashes themselves. - // We also update the prev_iter_ends_odd_backslash reference parameter to - // indicate whether we end an iteration on an odd-length sequence of - // backslashes, which modifies our subsequent search for odd-length - // sequences of backslashes in an obvious way. - really_inline uint64_t follows_odd_sequence_of(const uint64_t match, uint64_t &overflow) { - const uint64_t even_bits = 0x5555555555555555ULL; - const uint64_t odd_bits = ~even_bits; - uint64_t start_edges = match & ~(match << 1); - /* flip lowest if we have an odd-length run at the end of the prior - * iteration */ - uint64_t even_start_mask = even_bits ^ overflow; - uint64_t even_starts = start_edges & even_start_mask; - uint64_t odd_starts = start_edges & ~even_start_mask; - uint64_t even_carries = match + even_starts; - - uint64_t odd_carries; - /* must record the carry-out of our odd-carries out of bit 63; this - * indicates whether the sense of any edge going to the next iteration - * should be flipped */ - bool new_overflow = add_overflow(match, odd_starts, &odd_carries); - - odd_carries |= overflow; /* push in bit zero as a - * potential end if we had an - * odd-numbered run at the - * end of the previous - * iteration */ - overflow = new_overflow ? 0x1ULL : 0x0ULL; - uint64_t even_carry_ends = even_carries & ~match; - uint64_t odd_carry_ends = odd_carries & ~match; - uint64_t even_start_odd_end = even_carry_ends & odd_bits; - uint64_t odd_start_even_end = odd_carry_ends & even_bits; - uint64_t odd_ends = even_start_odd_end | odd_start_even_end; - return odd_ends; - } - // - // Check if the current character immediately follows a matching character. + // Finish the scan and return any errors. // - // For example, this checks for quotes with backslashes in front of them: + // This may detect errors as well, such as unclosed string and certain UTF-8 errors. // - // const uint64_t backslashed_quote = in.eq('"') & immediately_follows(in.eq('\'), prev_backslash); - // - really_inline uint64_t follows(const uint64_t match, uint64_t &overflow) { - const uint64_t result = match << 1 | overflow; - overflow = match >> 63; - return result; - } - - // - // Check if the current character follows a matching character, with possible "filler" between. - // For example, this checks for empty curly braces, e.g. - // - // in.eq('}') & follows(in.eq('['), in.eq(' '), prev_empty_array) // { * } - // - really_inline uint64_t follows(const uint64_t match, const uint64_t filler, uint64_t &overflow) { - uint64_t follows_match = follows(match, overflow); - uint64_t result; - overflow |= add_overflow(follows_match, filler, &result); - return result; - } - - really_inline ErrorValues detect_errors_on_eof() { - if (prev_in_string) { - return UNCLOSED_STRING; - } - if (unescaped_chars_error) { - return UNESCAPED_CHARS; - } - return SUCCESS; - } + really_inline ErrorValues detect_errors_on_eof(); // // Return a mask of all string characters plus end quotes. @@ -39016,28 +39190,7 @@ public: // // Backslash sequences outside of quotes will be detected in stage 2. // - really_inline uint64_t find_strings(const simd::simd8x64 in) { - const uint64_t backslash = in.eq('\\'); - const uint64_t escaped = follows_odd_sequence_of(backslash, prev_escaped); - const uint64_t quote = in.eq('"') & ~escaped; - // prefix_xor flips on bits inside the string (and flips off the end quote). - const uint64_t in_string = prefix_xor(quote) ^ prev_in_string; - /* right shift of a signed value expected to be well-defined and standard - * compliant as of C++20, - * John Regher from Utah U. says this is fine code */ - prev_in_string = static_cast(static_cast(in_string) >> 63); - // Use ^ to turn the beginning quote off, and the end quote on. - return in_string ^ quote; - } - - really_inline uint64_t invalid_string_bytes(const uint64_t unescaped, const uint64_t quote_mask) { - /* All Unicode characters may be placed within the - * quotation marks, except for the characters that MUST be escaped: - * quotation mark, reverse solidus, and the control characters (U+0000 - * through U+001F). - * https://tools.ietf.org/html/rfc8259 */ - return quote_mask & unescaped; - } + really_inline uint64_t find_strings(const simd::simd8x64 in); // // Determine which characters are *structural*: @@ -39055,101 +39208,262 @@ public: // contents of a string the same as content outside. Errors and structurals inside the string or on // the trailing quote will need to be removed later when the correct string information is known. // - really_inline uint64_t find_potential_structurals(const simd::simd8x64 in) { - // These use SIMD so let's kick them off before running the regular 64-bit stuff ... - uint64_t whitespace, op; - find_whitespace_and_operators(in, whitespace, op); - - // Detect the start of a run of primitive characters. Includes numbers, booleans, and strings ("). - // Everything except whitespace, braces, colon and comma. - const uint64_t primitive = ~(op | whitespace); - const uint64_t follows_primitive = follows(primitive, prev_primitive); - const uint64_t start_primitive = primitive & ~follows_primitive; - - // Return final structurals - return op | start_primitive; - } + really_inline uint64_t find_potential_structurals(const simd::simd8x64 in); // - // Find the important bits of JSON in a 128-byte chunk, and add them to structural_indexes. + // Find the important bits of JSON in a STEP_SIZE-byte chunk, and add them to structural_indexes. // - // PERF NOTES: - // We pipe 2 inputs through these stages: - // 1. Load JSON into registers. This takes a long time and is highly parallelizable, so we load - // 2 inputs' worth at once so that by the time step 2 is looking for them input, it's available. - // 2. Scan the JSON for critical data: strings, primitives and operators. This is the critical path. - // The output of step 1 depends entirely on this information. These functions don't quite use - // up enough CPU: the second half of the functions is highly serial, only using 1 execution core - // at a time. The second input's scans has some dependency on the first ones finishing it, but - // they can make a lot of progress before they need that information. - // 3. Step 1 doesn't use enough capacity, so we run some extra stuff while we're waiting for that - // to finish: utf-8 checks and generating the output from the last iteration. - // - // The reason we run 2 inputs at a time, is steps 2 and 3 are *still* not enough to soak up all - // available capacity with just one input. Running 2 at a time seems to give the CPU a good enough - // workout. + template + really_inline void scan_step(const uint8_t *buf, const size_t idx, utf8_checker &utf8_checker); + // - really_inline void scan_step(const uint8_t *buf, const size_t idx, utf8_checker &utf8_checker) { - // - // Load up all 128 bytes into SIMD registers - // - simd::simd8x64 in_1(buf); - simd::simd8x64 in_2(buf+64); - - // - // Find the strings and potential structurals (operators / primitives). - // - // This will include false structurals that are *inside* strings--we'll filter strings out - // before we return. - // - uint64_t string_1 = this->find_strings(in_1); - uint64_t structurals_1 = this->find_potential_structurals(in_1); - uint64_t string_2 = this->find_strings(in_2); - uint64_t structurals_2 = this->find_potential_structurals(in_2); - - // - // Do miscellaneous work while the processor is busy calculating strings and structurals. - // - // After that, weed out structurals that are inside strings and find invalid string characters. - // - uint64_t unescaped_1 = in_1.lteq(0x1F); - utf8_checker.check_next_input(in_1); - this->structural_indexes.write_indexes(idx-64, prev_structurals); // Output *last* iteration's structurals to ParsedJson - this->prev_structurals = structurals_1 & ~string_1; - this->unescaped_chars_error |= unescaped_1 & string_1; - - uint64_t unescaped_2 = in_2.lteq(0x1F); - utf8_checker.check_next_input(in_2); - this->structural_indexes.write_indexes(idx, prev_structurals); // Output *last* iteration's structurals to ParsedJson - this->prev_structurals = structurals_2 & ~string_2; - this->unescaped_chars_error |= unescaped_2 & string_2; - } - - really_inline void scan(const uint8_t *buf, const size_t len, utf8_checker &utf8_checker) { - size_t lenminusstep = len < STEP_SIZE ? 0 : len - STEP_SIZE; - size_t idx = 0; - - for (; idx < lenminusstep; idx += STEP_SIZE) { - this->scan_step(&buf[idx], idx, utf8_checker); - } - - /* If we have a final chunk of less than 64 bytes, pad it to 64 with - * spaces before processing it (otherwise, we risk invalidating the UTF-8 - * checks). */ - if (likely(idx < len)) { - uint8_t tmp_buf[STEP_SIZE]; - memset(tmp_buf, 0x20, STEP_SIZE); - memcpy(tmp_buf, buf + idx, len - idx); - this->scan_step(&tmp_buf[0], idx, utf8_checker); - idx += STEP_SIZE; - } - - /* finally, flatten out the remaining structurals from the last iteration */ - this->structural_indexes.write_indexes(idx-64, this->prev_structurals); - } - + // Parse the entire input in STEP_SIZE-byte chunks. + // + template + really_inline void scan(const uint8_t *buf, const size_t len, utf8_checker &utf8_checker); }; +// return a bitvector indicating where we have characters that end an odd-length +// sequence of backslashes (and thus change the behavior of the next character +// to follow). A even-length sequence of backslashes, and, for that matter, the +// largest even-length prefix of our odd-length sequence of backslashes, simply +// modify the behavior of the backslashes themselves. +// We also update the prev_iter_ends_odd_backslash reference parameter to +// indicate whether we end an iteration on an odd-length sequence of +// backslashes, which modifies our subsequent search for odd-length +// sequences of backslashes in an obvious way. +really_inline uint64_t follows_odd_sequence_of(const uint64_t match, uint64_t &overflow) { + const uint64_t even_bits = 0x5555555555555555ULL; + const uint64_t odd_bits = ~even_bits; + uint64_t start_edges = match & ~(match << 1); + /* flip lowest if we have an odd-length run at the end of the prior + * iteration */ + uint64_t even_start_mask = even_bits ^ overflow; + uint64_t even_starts = start_edges & even_start_mask; + uint64_t odd_starts = start_edges & ~even_start_mask; + uint64_t even_carries = match + even_starts; + + uint64_t odd_carries; + /* must record the carry-out of our odd-carries out of bit 63; this + * indicates whether the sense of any edge going to the next iteration + * should be flipped */ + bool new_overflow = add_overflow(match, odd_starts, &odd_carries); + + odd_carries |= overflow; /* push in bit zero as a + * potential end if we had an + * odd-numbered run at the + * end of the previous + * iteration */ + overflow = new_overflow ? 0x1ULL : 0x0ULL; + uint64_t even_carry_ends = even_carries & ~match; + uint64_t odd_carry_ends = odd_carries & ~match; + uint64_t even_start_odd_end = even_carry_ends & odd_bits; + uint64_t odd_start_even_end = odd_carry_ends & even_bits; + uint64_t odd_ends = even_start_odd_end | odd_start_even_end; + return odd_ends; +} + +// +// Check if the current character immediately follows a matching character. +// +// For example, this checks for quotes with backslashes in front of them: +// +// const uint64_t backslashed_quote = in.eq('"') & immediately_follows(in.eq('\'), prev_backslash); +// +really_inline uint64_t follows(const uint64_t match, uint64_t &overflow) { + const uint64_t result = match << 1 | overflow; + overflow = match >> 63; + return result; +} + +// +// Check if the current character follows a matching character, with possible "filler" between. +// For example, this checks for empty curly braces, e.g. +// +// in.eq('}') & follows(in.eq('['), in.eq(' '), prev_empty_array) // { * } +// +really_inline uint64_t follows(const uint64_t match, const uint64_t filler, uint64_t &overflow) { + uint64_t follows_match = follows(match, overflow); + uint64_t result; + overflow |= add_overflow(follows_match, filler, &result); + return result; +} + +really_inline ErrorValues json_structural_scanner::detect_errors_on_eof() { + if (prev_in_string) { + return UNCLOSED_STRING; + } + if (unescaped_chars_error) { + return UNESCAPED_CHARS; + } + return SUCCESS; +} + +// +// Return a mask of all string characters plus end quotes. +// +// prev_escaped is overflow saying whether the next character is escaped. +// prev_in_string is overflow saying whether we're still in a string. +// +// Backslash sequences outside of quotes will be detected in stage 2. +// +really_inline uint64_t json_structural_scanner::find_strings(const simd::simd8x64 in) { + const uint64_t backslash = in.eq('\\'); + const uint64_t escaped = follows_odd_sequence_of(backslash, prev_escaped); + const uint64_t quote = in.eq('"') & ~escaped; + // prefix_xor flips on bits inside the string (and flips off the end quote). + const uint64_t in_string = prefix_xor(quote) ^ prev_in_string; + /* right shift of a signed value expected to be well-defined and standard + * compliant as of C++20, + * John Regher from Utah U. says this is fine code */ + prev_in_string = static_cast(static_cast(in_string) >> 63); + // Use ^ to turn the beginning quote off, and the end quote on. + return in_string ^ quote; +} + +// +// Determine which characters are *structural*: +// - braces: [] and {} +// - the start of primitives (123, true, false, null) +// - the start of invalid non-whitespace (+, &, ture, UTF-8) +// +// Also detects value sequence errors: +// - two values with no separator between ("hello" "world") +// - separators with no values ([1,] [1,,]and [,2]) +// +// This method will find all of the above whether it is in a string or not. +// +// To reduce dependency on the expensive "what is in a string" computation, this method treats the +// contents of a string the same as content outside. Errors and structurals inside the string or on +// the trailing quote will need to be removed later when the correct string information is known. +// +really_inline uint64_t json_structural_scanner::find_potential_structurals(const simd::simd8x64 in) { + // These use SIMD so let's kick them off before running the regular 64-bit stuff ... + uint64_t whitespace, op; + find_whitespace_and_operators(in, whitespace, op); + + // Detect the start of a run of primitive characters. Includes numbers, booleans, and strings ("). + // Everything except whitespace, braces, colon and comma. + const uint64_t primitive = ~(op | whitespace); + const uint64_t follows_primitive = follows(primitive, prev_primitive); + const uint64_t start_primitive = primitive & ~follows_primitive; + + // Return final structurals + return op | start_primitive; +} + +// +// Find the important bits of JSON in a 128-byte chunk, and add them to structural_indexes. +// +// PERF NOTES: +// We pipe 2 inputs through these stages: +// 1. Load JSON into registers. This takes a long time and is highly parallelizable, so we load +// 2 inputs' worth at once so that by the time step 2 is looking for them input, it's available. +// 2. Scan the JSON for critical data: strings, primitives and operators. This is the critical path. +// The output of step 1 depends entirely on this information. These functions don't quite use +// up enough CPU: the second half of the functions is highly serial, only using 1 execution core +// at a time. The second input's scans has some dependency on the first ones finishing it, but +// they can make a lot of progress before they need that information. +// 3. Step 1 doesn't use enough capacity, so we run some extra stuff while we're waiting for that +// to finish: utf-8 checks and generating the output from the last iteration. +// +// The reason we run 2 inputs at a time, is steps 2 and 3 are *still* not enough to soak up all +// available capacity with just one input. Running 2 at a time seems to give the CPU a good enough +// workout. +// +template<> +really_inline void json_structural_scanner::scan_step<128>(const uint8_t *buf, const size_t idx, utf8_checker &utf8_checker) { + // + // Load up all 128 bytes into SIMD registers + // + simd::simd8x64 in_1(buf); + simd::simd8x64 in_2(buf+64); + + // + // Find the strings and potential structurals (operators / primitives). + // + // This will include false structurals that are *inside* strings--we'll filter strings out + // before we return. + // + uint64_t string_1 = this->find_strings(in_1); + uint64_t structurals_1 = this->find_potential_structurals(in_1); + uint64_t string_2 = this->find_strings(in_2); + uint64_t structurals_2 = this->find_potential_structurals(in_2); + + // + // Do miscellaneous work while the processor is busy calculating strings and structurals. + // + // After that, weed out structurals that are inside strings and find invalid string characters. + // + uint64_t unescaped_1 = in_1.lteq(0x1F); + utf8_checker.check_next_input(in_1); + this->structural_indexes.write_indexes(idx-64, this->prev_structurals); // Output *last* iteration's structurals to ParsedJson + this->prev_structurals = structurals_1 & ~string_1; + this->unescaped_chars_error |= unescaped_1 & string_1; + + uint64_t unescaped_2 = in_2.lteq(0x1F); + utf8_checker.check_next_input(in_2); + this->structural_indexes.write_indexes(idx, this->prev_structurals); // Output *last* iteration's structurals to ParsedJson + this->prev_structurals = structurals_2 & ~string_2; + this->unescaped_chars_error |= unescaped_2 & string_2; +} + +// +// Find the important bits of JSON in a 64-byte chunk, and add them to structural_indexes. +// +template<> +really_inline void json_structural_scanner::scan_step<64>(const uint8_t *buf, const size_t idx, utf8_checker &utf8_checker) { + // + // Load up bytes into SIMD registers + // + simd::simd8x64 in_1(buf); + + // + // Find the strings and potential structurals (operators / primitives). + // + // This will include false structurals that are *inside* strings--we'll filter strings out + // before we return. + // + uint64_t string_1 = this->find_strings(in_1); + uint64_t structurals_1 = this->find_potential_structurals(in_1); + + // + // Do miscellaneous work while the processor is busy calculating strings and structurals. + // + // After that, weed out structurals that are inside strings and find invalid string characters. + // + uint64_t unescaped_1 = in_1.lteq(0x1F); + utf8_checker.check_next_input(in_1); + this->structural_indexes.write_indexes(idx-64, this->prev_structurals); // Output *last* iteration's structurals to ParsedJson + this->prev_structurals = structurals_1 & ~string_1; + this->unescaped_chars_error |= unescaped_1 & string_1; +} + +template +really_inline void json_structural_scanner::scan(const uint8_t *buf, const size_t len, utf8_checker &utf8_checker) { + size_t lenminusstep = len < STEP_SIZE ? 0 : len - STEP_SIZE; + size_t idx = 0; + + for (; idx < lenminusstep; idx += STEP_SIZE) { + this->scan_step(&buf[idx], idx, utf8_checker); + } + + /* If we have a final chunk of less than 64 bytes, pad it to 64 with + * spaces before processing it (otherwise, we risk invalidating the UTF-8 + * checks). */ + if (likely(idx < len)) { + uint8_t tmp_buf[STEP_SIZE]; + memset(tmp_buf, 0x20, STEP_SIZE); + memcpy(tmp_buf, buf + idx, len - idx); + this->scan_step(&tmp_buf[0], idx, utf8_checker); + idx += STEP_SIZE; + } + + /* finally, flatten out the remaining structurals from the last iteration */ + this->structural_indexes.write_indexes(idx-64, this->prev_structurals); +} + +template int find_structural_bits(const uint8_t *buf, size_t len, simdjson::ParsedJson &pj, bool streaming) { if (unlikely(len > pj.byte_capacity)) { std::cerr << "Your ParsedJson object only supports documents up to " @@ -39159,7 +39473,7 @@ int find_structural_bits(const uint8_t *buf, size_t len, simdjson::ParsedJson &p } utf8_checker utf8_checker{}; json_structural_scanner scanner{pj.structural_indexes}; - scanner.scan(buf, len, utf8_checker); + scanner.scan(buf, len, utf8_checker); simdjson::ErrorValues error = scanner.detect_errors_on_eof(); if (!streaming && unlikely(error != simdjson::SUCCESS)) { @@ -39185,6 +39499,8 @@ int find_structural_bits(const uint8_t *buf, size_t len, simdjson::ParsedJson &p return utf8_checker.errors(); } +} // namespace stage1 + } // namespace westmere UNTARGET_REGION @@ -39193,7 +39509,7 @@ namespace simdjson { template <> int find_structural_bits(const uint8_t *buf, size_t len, simdjson::ParsedJson &pj, bool streaming) { - return westmere::find_structural_bits(buf, len, pj, streaming); + return westmere::stage1::find_structural_bits<64>(buf, len, pj, streaming); } } // namespace simdjson @@ -43047,6 +43363,36 @@ ParsedJson::ParsedJson(ParsedJson &&p) p.current_string_buf_loc = nullptr; } +ParsedJson &ParsedJson::operator=(ParsedJson &&p) { + byte_capacity = p.byte_capacity; + p.byte_capacity = 0; + depth_capacity = p.depth_capacity; + p.depth_capacity = 0; + tape_capacity = p.tape_capacity; + p.tape_capacity = 0; + string_capacity = p.string_capacity; + p.string_capacity = 0; + current_loc = p.current_loc; + p.current_loc = 0; + n_structural_indexes = p.n_structural_indexes; + p.n_structural_indexes = 0; + structural_indexes = p.structural_indexes; + p.structural_indexes = nullptr; + tape = p.tape; + p.tape = nullptr; + containing_scope_offset = p.containing_scope_offset; + p.containing_scope_offset = nullptr; + ret_address = p.ret_address; + p.ret_address = nullptr; + string_buf = p.string_buf; + p.string_buf = nullptr; + current_string_buf_loc = p.current_string_buf_loc; + p.current_string_buf_loc = nullptr; + valid = p.valid; + p.valid = false; + return *this; +} + WARN_UNUSED bool ParsedJson::allocate_capacity(size_t len, size_t max_depth) { if (max_depth <= 0) { diff --git a/singleheader/simdjson.h b/singleheader/simdjson.h index 58722c71d..288967885 100755 --- a/singleheader/simdjson.h +++ b/singleheader/simdjson.h @@ -1,4 +1,4 @@ -/* auto-generated on Thu 07 Nov 2019 05:05:37 PM EST. Do not edit! */ +/* auto-generated on Wed Nov 20 11:15:43 EST 2019. Do not edit! */ /* begin file include/simdjson/simdjson_version.h */ // /include/simdjson/simdjson_version.h automatically generated by release.py, // do not change by hand @@ -59,6 +59,11 @@ enum { #define TARGET_WESTMERE TARGET_REGION("sse4.2,pclmul") #define TARGET_ARM64 +// Is threading enabled? +#if defined(BOOST_HAS_THREADS) || defined(_REENTRANT) || defined(_MT) +#define SIMDJSON_THREADS_ENABLED 1 +#endif + #ifdef _MSC_VER #include #else @@ -705,6 +710,7 @@ const std::string &error_message(const int); #define SIMDJSON_PADDING_STRING_H #include #include +#include namespace simdjson { // low-level function to allocate memory with padding so we can read passed the @@ -743,6 +749,15 @@ public: : viable_size(o.viable_size), data_ptr(o.data_ptr) { o.data_ptr = nullptr; // we take ownership } + + padded_string &operator=(padded_string &&o) { + data_ptr = o.data_ptr; + viable_size = o.viable_size; + o.data_ptr = nullptr; // we take ownership + o.viable_size = 0; + return *this; + } + void swap(padded_string &o) { size_t tmp_viable_size = viable_size; char *tmp_data_ptr = data_ptr; @@ -859,6 +874,7 @@ public: ParsedJson(); ~ParsedJson(); ParsedJson(ParsedJson &&p); + ParsedJson &operator=(ParsedJson &&o); // if needed, allocate memory so that the object is able to process JSON // documents having up to len bytes and max_depth "depth" @@ -2013,6 +2029,7 @@ inline ParsedJson build_parsed_json(const padded_string &s) { #include +#include namespace simdjson { @@ -2042,8 +2059,17 @@ namespace simdjson { bool error_on_last_attempt{false}; bool load_next_batch{true}; size_t current_buffer_loc{0}; + size_t last_json_buffer_loc{0}; + size_t thread_current_buffer_loc{0}; size_t n_parsed_docs{0}; size_t n_bytes_parsed{0}; + + std::thread stage_1_thread; + simdjson::ParsedJson pj_thread; + +#ifdef SIMDJSON_THREADS_ENABLED + size_t find_last_json(const ParsedJson &pj); +#endif }; } diff --git a/src/jsonstream.cpp b/src/jsonstream.cpp index 99964129b..d6c5dee08 100755 --- a/src/jsonstream.cpp +++ b/src/jsonstream.cpp @@ -3,15 +3,14 @@ #include using namespace simdjson; - void find_the_best_supported_implementation(); + typedef int (*stage1_functype)(const char *buf, size_t len, ParsedJson &pj, bool streaming); typedef int (*stage2_functype)(const char *buf, size_t len, ParsedJson &pj, size_t &next_json); stage1_functype best_stage1; stage2_functype best_stage2; - JsonStream::JsonStream(const char *buf, size_t len, size_t batchSize) : _buf(buf), _len(len), _batch_size(batchSize) { find_the_best_supported_implementation(); @@ -30,11 +29,10 @@ void JsonStream::set_new_buffer(const char *buf, size_t len) { } int JsonStream::json_parse(ParsedJson &pj) { - //return json_parse_ptr.load(std::memory_order_relaxed)(buf, len, batch_size, pj, realloc_if_needed); - if (pj.byte_capacity == 0) { const bool allocok = pj.allocate_capacity(_batch_size, _batch_size); - if (!allocok) { + const bool allocok_thread = pj_thread.allocate_capacity(_batch_size, _batch_size); + if (!allocok || !allocok_thread) { std::cerr << "can't allocate memory" << std::endl; return false; } @@ -42,31 +40,64 @@ int JsonStream::json_parse(ParsedJson &pj) { else if (pj.byte_capacity < _batch_size) { return simdjson::CAPACITY; } - - //Quick heuristic to see if it's worth parsing the remaining data in the batch - if(!load_next_batch && n_bytes_parsed > 0) { - const auto remaining_data = _batch_size - current_buffer_loc; - const auto avg_doc_len = (float) n_bytes_parsed / n_parsed_docs; - - if(remaining_data < avg_doc_len) - load_next_batch = true; - } +#ifdef SIMDJSON_THREADS_ENABLED + if(current_buffer_loc == last_json_buffer_loc) + load_next_batch = true; +#endif if (load_next_batch){ +#ifdef SIMDJSON_THREADS_ENABLED + //First time loading + if(!stage_1_thread.joinable()){ + _buf = &_buf[current_buffer_loc]; + _len -= current_buffer_loc; + n_bytes_parsed += current_buffer_loc; + _batch_size = std::min(_batch_size, _len); + int stage1_is_ok = (*best_stage1)(_buf, _batch_size, pj, true); + + if (stage1_is_ok != simdjson::SUCCESS) { + pj.error_code = stage1_is_ok; + return pj.error_code; + } + } + + //the second thread is running or done. + else{ + stage_1_thread.join(); + std::swap(pj.structural_indexes, pj_thread.structural_indexes); + pj.n_structural_indexes = pj_thread.n_structural_indexes; + + _buf = &_buf[last_json_buffer_loc]; + _len -= last_json_buffer_loc; + n_bytes_parsed += last_json_buffer_loc; + last_json_buffer_loc = 0; //because we want to use it in the if above. + } + + if(_len-_batch_size > 0) { + last_json_buffer_loc = find_last_json(pj); + _batch_size = std::min(_batch_size, _len - last_json_buffer_loc); + if(_batch_size>0) + stage_1_thread = std::thread( + static_cast(*best_stage1), + &_buf[last_json_buffer_loc], _batch_size, + std::ref(pj_thread), + true); + + } +#else _buf = &_buf[current_buffer_loc]; _len -= current_buffer_loc; n_bytes_parsed += current_buffer_loc; _batch_size = std::min(_batch_size, _len); - int stage1_is_ok = (*best_stage1)(_buf, _batch_size, pj, true); if (stage1_is_ok != simdjson::SUCCESS) { pj.error_code = stage1_is_ok; return pj.error_code; } - +#endif load_next_batch = false; //If we loaded a perfect amount of documents last time, we need to skip the first element, @@ -89,7 +120,9 @@ int JsonStream::json_parse(ParsedJson &pj) { load_next_batch = true; } - else current_buffer_loc = pj.structural_indexes[next_json]; + else { + current_buffer_loc = pj.structural_indexes[next_json]; + } } //TODO: have a more precise error check //Give it two chances for now. We assume the error is because the json was not loaded completely in this batch. @@ -99,10 +132,51 @@ int JsonStream::json_parse(ParsedJson &pj) { error_on_last_attempt = true; res = json_parse(pj); } - return res; } +#ifdef SIMDJSON_THREADS_ENABLED +size_t JsonStream::find_last_json(const ParsedJson &pj) { + auto last_i = pj.n_structural_indexes - 1; + if (pj.structural_indexes[last_i] == _batch_size) + last_i = pj.n_structural_indexes - 2; + auto arr_cnt = 0; + auto obj_cnt = 0; + for (auto i = last_i; i > 0; i--) { + auto idxb = pj.structural_indexes[i]; + switch (_buf[idxb]) { + case ':': + case ',': + continue; + case '}': + obj_cnt--; + continue; + case ']': + arr_cnt--; + continue; + case '{': + obj_cnt++; + break; + case '[': + arr_cnt++; + break; + } + auto idxa = pj.structural_indexes[i - 1]; + switch (_buf[idxa]) { + case '{': + case '[': + case ':': + case ',': + continue; + } + if (!arr_cnt && !obj_cnt) + return pj.structural_indexes[last_i+1]; + return idxb; + } + return 0; +} +#endif + size_t JsonStream::get_current_buffer_loc() const { return current_buffer_loc; } @@ -115,18 +189,16 @@ size_t JsonStream::get_n_bytes_parsed() const { return n_bytes_parsed; } - //// TODO: generalize this set of functions. We don't want to have a copy in jsonparser.cpp void find_the_best_supported_implementation() { + uint32_t supports = detect_supported_architectures(); + // Order from best to worst (within architecture) +#ifdef IS_X86_64 constexpr uint32_t haswell_flags = instruction_set::AVX2 | instruction_set::PCLMULQDQ | instruction_set::BMI1 | instruction_set::BMI2; constexpr uint32_t westmere_flags = instruction_set::SSE42 | instruction_set::PCLMULQDQ; - - uint32_t supports = detect_supported_architectures(); - // Order from best to worst (within architecture) -#ifdef IS_X86_64 if ((haswell_flags & supports) == haswell_flags) { best_stage1 = simdjson::find_structural_bits; best_stage2 = simdjson::unified_machine; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 80dcf2744..b0c8c4685 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -10,6 +10,7 @@ add_cpp_test(jsonstream_test) add_cpp_test(pointercheck) add_cpp_test(integer_tests) +target_link_libraries(jsonstream_test Threads::Threads) ## This causes problems # add_executable(singleheader ./singleheadertest.cpp ${PROJECT_SOURCE_DIR}/singleheader/simdjson.cpp) # target_compile_definitions(singleheader PRIVATE JSON_TEST_PATH="${PROJECT_SOURCE_DIR}/jsonexamples/twitter.json")