diff --git a/README.md b/README.md index 163e1a624..8a6554db2 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,45 @@ -# simdjson : Parsing gigabytes of JSON per second +# simdjson : Parsing gigabytes of JSON per second + + +JSON is everywhere on the Internet. Servers spend a *lot* of time parsing it. We need a fresh approach. simdjson uses commonly available SIMD instructions and microparallel algorithms to parse JSON 2.5x faster than anything else out there. + +* **Ludicrous Speed:** Over 2.5x faster than other production-grade JSON parsers. +* **Delightfully Easy:** First-class, easy to use API. +* **Complete Validation:** Full JSON and UTF-8 validation, with no compromises. +* **Rock-Solid Reliability:** From memory allocation to error handling, simdjson's design avoids surprises. + +This library is part of the [Awesome Modern C++](https://awesomecpp.com) list. + [![Build Status](https://cloud.drone.io/api/badges/simdjson/simdjson/status.svg)](https://cloud.drone.io/simdjson/simdjson) [![CircleCI](https://circleci.com/gh/simdjson/simdjson.svg?style=svg)](https://circleci.com/gh/simdjson/simdjson) [![Build status](https://ci.appveyor.com/api/projects/status/ae77wp5v3lebmu6n/branch/master?svg=true)](https://ci.appveyor.com/project/lemire/simdjson-jmmti/branch/master) [![][license img]][license] [![Fuzzing Status](https://oss-fuzz-build-logs.storage.googleapis.com/badges/simdjson.svg)](https://bugs.chromium.org/p/oss-fuzz/issues/list?sort=-opened&can=1&q=proj:simdjson) +## Quick Start -## A C++ library to see how fast we can parse JSON with complete validation. +simdjson is easily consumable with a single .h and .cpp file. -JSON documents are everywhere on the Internet. Servers spend a lot of time parsing these documents. We want to accelerate the parsing of JSON per se using commonly available SIMD instructions as much as possible while doing full validation (including character encoding). This library is part of the [Awesome Modern C++](https://awesomecpp.com) list. +0. Prerequisites: `g++` or `clang++`. +1. Pull [simdjson.h](singleheader/simdjson.h) and [simdjson.cpp](singleheader/simdjson.h) into a directory, along with the sample file [twitter.json](jsonexamples/twitter.json). + ``` + wget https://raw.githubusercontent.com/simdjson/simdjson/master/singleheader/simdjson.h https://raw.githubusercontent.com/simdjson/simdjson/master/singleheader/simdjson.cpp https://raw.githubusercontent.com/simdjson/simdjson/master/jsonexamples/twitter.json + ``` +2. Create `parser.cpp`: - + ```c++ + #include "simdjson.h" + int main(void) { + simdjson::document::parser parser; + simdjson::document& tweets = parser.load("twitter.json"); + std::cout << tweets["search_metadata"]["count"] << " results." << std::endl; + } + ``` +3. `g++ -o parser parser.cpp` (or clang++) +4. `./parser` + ``` + 100 results. + ``` ## Real-world usage diff --git a/singleheader/amalgamation_demo.cpp b/singleheader/amalgamation_demo.cpp index 759dc41c3..4979e0f79 100755 --- a/singleheader/amalgamation_demo.cpp +++ b/singleheader/amalgamation_demo.cpp @@ -1,4 +1,4 @@ -/* auto-generated on Thu Mar 5 10:30:07 PST 2020. Do not edit! */ +/* auto-generated on Fri Mar 20 11:47:31 PDT 2020. Do not edit! */ #include #include "simdjson.h" @@ -8,37 +8,31 @@ int main(int argc, char *argv[]) { std::cerr << "Please specify at least one file name. " << std::endl; } const char * filename = argv[1]; - simdjson::padded_string p = simdjson::get_corpus(filename); - auto [doc, error] = simdjson::document::parse(p); // do the parsing + simdjson::document::parser parser; + auto [doc, error] = parser.load(filename); // do the parsing if (error) { - std::cout << "document::parse failed" << std::endl; + std::cout << "parse failed" << std::endl; std::cout << "error code: " << error << std::endl; std::cout << error << std::endl; } else { - std::cout << "document::parse valid" << std::endl; + std::cout << "parse valid" << std::endl; } if(argc == 2) { return EXIT_SUCCESS; } - //JsonStream + // parse_many const char * filename2 = argv[2]; - simdjson::padded_string p2 = simdjson::get_corpus(filename2); - simdjson::document::parser parser; - simdjson::JsonStream js{p2}; - int parse_res = simdjson::SUCCESS_AND_HAS_MORE; - - while (parse_res == simdjson::SUCCESS_AND_HAS_MORE) { - parse_res = js.json_parse(parser); + for (auto result : parser.load_many(filename2)) { + error = result.error(); } - - if( ! parser.is_valid()) { - std::cout << "JsonStream not valid" << std::endl; + if (error) { + std::cout << "parse_many failed" << std::endl; + std::cout << "error code: " << error << std::endl; + std::cout << error << std::endl; } else { - std::cout << "JsonStream valid" << std::endl; + std::cout << "parse_many valid" << std::endl; } - - return EXIT_SUCCESS; } diff --git a/singleheader/simdjson.cpp b/singleheader/simdjson.cpp index ba25e97a6..f096537e4 100644 --- a/singleheader/simdjson.cpp +++ b/singleheader/simdjson.cpp @@ -1,4 +1,4 @@ -/* auto-generated on Thu Mar 5 10:30:07 PST 2020. Do not edit! */ +/* auto-generated on Fri Mar 20 11:47:31 PDT 2020. Do not edit! */ #include "simdjson.h" /* used for http://dmalloc.com/ Dmalloc - Debug Malloc Library */ @@ -7,67 +7,7 @@ #endif /* begin file src/simdjson.cpp */ -/* begin file src/error.cpp */ -#include - -namespace simdjson { - -const std::map error_strings = { - {SUCCESS, "No error"}, - {SUCCESS_AND_HAS_MORE, "No error and buffer still has more data"}, - {CAPACITY, "This parser can't support a document that big"}, - {MEMALLOC, "Error allocating memory, we're most likely out of memory"}, - {TAPE_ERROR, "Something went wrong while writing to the tape"}, - {STRING_ERROR, "Problem while parsing a string"}, - {T_ATOM_ERROR, "Problem while parsing an atom starting with the letter 't'"}, - {F_ATOM_ERROR, "Problem while parsing an atom starting with the letter 'f'"}, - {N_ATOM_ERROR, "Problem while parsing an atom starting with the letter 'n'"}, - {NUMBER_ERROR, "Problem while parsing a number"}, - {UTF8_ERROR, "The input is not valid UTF-8"}, - {UNINITIALIZED, "Uninitialized"}, - {EMPTY, "Empty: no JSON found"}, - {UNESCAPED_CHARS, "Within strings, some characters must be escaped, we" - " found unescaped characters"}, - {UNCLOSED_STRING, "A string is opened, but never closed."}, - {UNSUPPORTED_ARCHITECTURE, "simdjson does not have an implementation" - " supported by this CPU architecture (perhaps" - " it's a non-SIMD CPU?)."}, - {INCORRECT_TYPE, "The JSON element does not have the requested type."}, - {NUMBER_OUT_OF_RANGE, "The JSON number is too large or too small to fit within the requested type."}, - {NO_SUCH_FIELD, "The JSON field referenced does not exist in this object."}, - {UNEXPECTED_ERROR, "Unexpected error, consider reporting this problem as" - " you may have found a bug in simdjson"}, -}; - -// string returned when the error code is not recognized -const std::string unexpected_error_msg {"Unexpected error"}; - -// returns a string matching the error code -const std::string &error_message(error_code code) noexcept { - auto keyvalue = error_strings.find(code); - if(keyvalue == error_strings.end()) { - return unexpected_error_msg; - } - return keyvalue->second; -} - -} // namespace simdjson -/* end file src/error.cpp */ /* begin file src/implementation.cpp */ -#include - -// Static array of known implementations. We're hoping these get baked into the executable -// without requiring a static initializer. - -#ifdef IS_X86_64 - -/* begin file src/haswell/implementation.h */ -#ifndef SIMDJSON_HASWELL_IMPLEMENTATION_H -#define SIMDJSON_HASWELL_IMPLEMENTATION_H - - -#ifdef IS_X86_64 - /* begin file src/isadetection.h */ /* From https://github.com/endorno/pytorch/blob/master/torch/lib/TH/generic/simd/simd.h @@ -223,6 +163,17 @@ static inline uint32_t detect_supported_architectures() { #endif // SIMDJSON_ISADETECTION_H /* end file src/isadetection.h */ +#include + +// Static array of known implementations. We're hoping these get baked into the executable +// without requiring a static initializer. + +#if SIMDJSON_IMPLEMENTATION_HASWELL +/* begin file src/haswell/implementation.h */ +#ifndef SIMDJSON_HASWELL_IMPLEMENTATION_H +#define SIMDJSON_HASWELL_IMPLEMENTATION_H + +/* isadetection.h already included: #include "isadetection.h" */ namespace simdjson::haswell { @@ -241,17 +192,16 @@ public: } // namespace simdjson::haswell -#endif // IS_X86_64 - #endif // SIMDJSON_HASWELL_IMPLEMENTATION_H -/* end file src/isadetection.h */ +/* end file src/haswell/implementation.h */ +namespace simdjson::internal { const haswell::implementation haswell_singleton{}; } +#endif // SIMDJSON_IMPLEMENTATION_HASWELL + +#if SIMDJSON_IMPLEMENTATION_WESTMERE /* begin file src/westmere/implementation.h */ #ifndef SIMDJSON_WESTMERE_IMPLEMENTATION_H #define SIMDJSON_WESTMERE_IMPLEMENTATION_H - -#ifdef IS_X86_64 - /* isadetection.h already included: #include "isadetection.h" */ namespace simdjson::westmere { @@ -267,28 +217,16 @@ public: } // namespace simdjson::westmere -#endif // IS_X86_64 - #endif // SIMDJSON_WESTMERE_IMPLEMENTATION_H /* end file src/westmere/implementation.h */ +namespace simdjson::internal { const westmere::implementation westmere_singleton{}; } +#endif // SIMDJSON_IMPLEMENTATION_WESTMERE -namespace simdjson::internal { -const haswell::implementation haswell_singleton{}; -const westmere::implementation westmere_singleton{}; -constexpr const std::initializer_list available_implementation_pointers { &haswell_singleton, &westmere_singleton }; -} - -#endif - -#ifdef IS_ARM64 - +#if SIMDJSON_IMPLEMENTATION_ARM64 /* begin file src/arm64/implementation.h */ #ifndef SIMDJSON_ARM64_IMPLEMENTATION_H #define SIMDJSON_ARM64_IMPLEMENTATION_H - -#ifdef IS_ARM64 - /* isadetection.h already included: #include "isadetection.h" */ namespace simdjson::arm64 { @@ -304,20 +242,56 @@ public: } // namespace simdjson::arm64 -#endif // IS_ARM64 - #endif // SIMDJSON_ARM64_IMPLEMENTATION_H /* end file src/arm64/implementation.h */ +namespace simdjson::internal { const arm64::implementation arm64_singleton{}; } +#endif // SIMDJSON_IMPLEMENTATION_ARM64 + +#if SIMDJSON_IMPLEMENTATION_FALLBACK +/* begin file src/fallback/implementation.h */ +#ifndef SIMDJSON_FALLBACK_IMPLEMENTATION_H +#define SIMDJSON_FALLBACK_IMPLEMENTATION_H + +/* isadetection.h already included: #include "isadetection.h" */ + +namespace simdjson::fallback { + +class implementation final : public simdjson::implementation { +public: + really_inline implementation() : simdjson::implementation( + "fallback", + "Generic fallback implementation", + 0 + ) {} + WARN_UNUSED error_code parse(const uint8_t *buf, size_t len, document::parser &parser) const noexcept final; + WARN_UNUSED error_code stage1(const uint8_t *buf, size_t len, document::parser &parser, bool streaming) const noexcept final; + WARN_UNUSED error_code stage2(const uint8_t *buf, size_t len, document::parser &parser) const noexcept final; + WARN_UNUSED error_code stage2(const uint8_t *buf, size_t len, document::parser &parser, size_t &next_json) const noexcept final; +}; + +} // namespace simdjson::fallback + +#endif // SIMDJSON_FALLBACK_IMPLEMENTATION_H +/* end file src/fallback/implementation.h */ +namespace simdjson::internal { const fallback::implementation fallback_singleton{}; } +#endif // SIMDJSON_IMPLEMENTATION_FALLBACK namespace simdjson::internal { -const arm64::implementation arm64_singleton{}; -constexpr const std::initializer_list available_implementation_pointers { &arm64_singleton }; -} +constexpr const std::initializer_list available_implementation_pointers { +#if SIMDJSON_IMPLEMENTATION_HASWELL + &haswell_singleton, #endif - - -namespace simdjson::internal { +#if SIMDJSON_IMPLEMENTATION_WESTMERE + &westmere_singleton, +#endif +#if SIMDJSON_IMPLEMENTATION_ARM64 + &arm64_singleton, +#endif +#if SIMDJSON_IMPLEMENTATION_FALLBACK + &fallback_singleton, +#endif +}; // available_implementation_pointers // So we can return UNSUPPORTED_ARCHITECTURE from the parser when there is no support class unsupported_implementation final : public implementation { @@ -363,45 +337,8 @@ const implementation *detect_best_supported_implementation_on_first_use::set_bes return active_implementation = available_implementations.detect_best_supported(); } -} // namespace simdjson -/* end file src/arm64/implementation.h */ -/* begin file src/jsonioutil.cpp */ -#include -#include -#include - -namespace simdjson { - -padded_string get_corpus(const std::string &filename) { - std::FILE *fp = std::fopen(filename.c_str(), "rb"); - if (fp != nullptr) { - if(std::fseek(fp, 0, SEEK_END) < 0) { - std::fclose(fp); - throw std::runtime_error("cannot seek in the file"); - } - long llen = std::ftell(fp); - if((llen < 0) || (llen == LONG_MAX)) { - std::fclose(fp); - throw std::runtime_error("cannot tell where we are in the file"); - } - size_t len = (size_t) llen; - padded_string s(len); - if (s.data() == nullptr) { - std::fclose(fp); - throw std::runtime_error("could not allocate memory"); - } - std::rewind(fp); - size_t readb = std::fread(s.data(), 1, len, fp); - std::fclose(fp); - if (readb != len) { - throw std::runtime_error("could not read the data"); - } - return s; - } - throw std::runtime_error("could not load corpus"); -} -} // namespace simdjson -/* end file src/jsonioutil.cpp */ +} // namespace simdjson::internal +/* end file src/fallback/implementation.h */ /* begin file src/jsonminifier.cpp */ #include @@ -1013,32 +950,25 @@ size_t oldjson_minify(const uint8_t *buf, size_t len, uint8_t *out) { #endif /* end file src/simdprune_tables.h */ /* begin file src/stage1_find_marks.cpp */ +#if SIMDJSON_IMPLEMENTATION_ARM64 /* begin file src/arm64/stage1_find_marks.h */ #ifndef SIMDJSON_ARM64_STAGE1_FIND_MARKS_H #define SIMDJSON_ARM64_STAGE1_FIND_MARKS_H - -#ifdef IS_ARM64 - /* begin file src/arm64/bitmask.h */ #ifndef SIMDJSON_ARM64_BITMASK_H #define SIMDJSON_ARM64_BITMASK_H -#ifdef IS_ARM64 - /* begin file src/arm64/intrinsics.h */ #ifndef SIMDJSON_ARM64_INTRINSICS_H #define SIMDJSON_ARM64_INTRINSICS_H -#ifdef IS_ARM64 - // This should be the correct header whether // you use visual studio or other compilers. #include -#endif // IS_ARM64 #endif // SIMDJSON_ARM64_INTRINSICS_H /* end file src/arm64/intrinsics.h */ @@ -1075,16 +1005,12 @@ really_inline uint64_t prefix_xor(uint64_t bitmask) { } // namespace simdjson::arm64 UNTARGET_REGION -#endif // IS_ARM64 #endif /* end file src/arm64/intrinsics.h */ /* begin file src/arm64/simd.h */ #ifndef SIMDJSON_ARM64_SIMD_H #define SIMDJSON_ARM64_SIMD_H - -#ifdef IS_ARM64 - /* arm64/intrinsics.h already included: #include "arm64/intrinsics.h" */ namespace simdjson::arm64::simd { @@ -1427,16 +1353,12 @@ namespace simdjson::arm64::simd { } // namespace simdjson::arm64::simd -#endif // IS_ARM64 #endif // SIMDJSON_ARM64_SIMD_H /* end file src/arm64/simd.h */ /* begin file src/arm64/bitmanipulation.h */ #ifndef SIMDJSON_ARM64_BITMANIPULATION_H #define SIMDJSON_ARM64_BITMANIPULATION_H - -#ifdef IS_ARM64 - /* arm64/intrinsics.h already included: #include "arm64/intrinsics.h" */ namespace simdjson::arm64 { @@ -1515,8 +1437,6 @@ really_inline bool mul_overflow(uint64_t value1, uint64_t value2, uint64_t *resu } // namespace simdjson::arm64 -#endif // IS_ARM64 - #endif // SIMDJSON_ARM64_BITMANIPULATION_H /* end file src/arm64/bitmanipulation.h */ /* arm64/implementation.h already included: #include "arm64/implementation.h" */ @@ -1525,10 +1445,18 @@ namespace simdjson::arm64 { using namespace simd; -really_inline void find_whitespace_and_operators( - const simd::simd8x64 in, - uint64_t &whitespace, uint64_t &op) { +struct json_character_block { + static really_inline json_character_block classify(const simd::simd8x64 in); + really_inline uint64_t whitespace() const { return _whitespace; } + really_inline uint64_t op() const { return _op; } + really_inline uint64_t scalar() { return ~(op() | whitespace()); } + + uint64_t _whitespace; + uint64_t _op; +}; + +really_inline json_character_block json_character_block::classify(const simd::simd8x64 in) { auto v = in.map([&](simd8 chunk) { auto nib_lo = chunk & 0xf; auto nib_hi = chunk.shr<4>(); @@ -1537,8 +1465,9 @@ really_inline void find_whitespace_and_operators( return shuf_lo & shuf_hi; }); - op = v.map([&](simd8 _v) { return _v.any_bits_set(0x7); }).to_bitmask(); - whitespace = v.map([&](simd8 _v) { return _v.any_bits_set(0x18); }).to_bitmask(); + uint64_t op = v.map([&](simd8 _v) { return _v.any_bits_set(0x7); }).to_bitmask(); + uint64_t whitespace = v.map([&](simd8 _v) { return _v.any_bits_set(0x18); }).to_bitmask(); + return { whitespace, op }; } really_inline bool is_ascii(simd8x64 input) { @@ -1567,7 +1496,7 @@ really_inline simd8 must_be_continuation(simd8 prev1, simd8 in); + really_inline error_code finish(bool streaming); + +private: + really_inline uint64_t find_escaped(uint64_t escape); + + // Whether the last iteration was still inside a string (all 1's = true, all 0's = false). + uint64_t prev_in_string = 0ULL; + // Whether the first character of the next iteration is escaped. + uint64_t prev_escaped = 0ULL; +}; + +// +// Finds escaped characters (characters following \). +// +// Handles runs of backslashes like \\\" and \\\\" correctly (yielding 0101 and 01010, respectively). +// +// Does this by: +// - Shift the escape mask to get potentially escaped characters (characters after backslashes). +// - Mask escaped sequences that start on *even* bits with 1010101010 (odd bits are escaped, even bits are not) +// - Mask escaped sequences that start on *odd* bits with 0101010101 (even bits are escaped, odd bits are not) +// +// To distinguish between escaped sequences starting on even/odd bits, it finds the start of all +// escape sequences, filters out the ones that start on even bits, and adds that to the mask of +// escape sequences. This causes the addition to clear out the sequences starting on odd bits (since +// the start bit causes a carry), and leaves even-bit sequences alone. +// +// Example: +// +// text | \\\ | \\\"\\\" \\\" \\"\\" | +// escape | xxx | xx xxx xxx xx xx | Removed overflow backslash; will | it into follows_escape +// odd_starts | x | x x x | escape & ~even_bits & ~follows_escape +// even_seq | c| cxxx c xx c | c = carry bit -- will be masked out later +// invert_mask | | cxxx c xx c| even_seq << 1 +// follows_escape | xx | x xx xxx xxx xx xx | Includes overflow bit +// escaped | x | x x x x x x x x | +// desired | x | x x x x x x x x | +// text | \\\ | \\\"\\\" \\\" \\"\\" | +// +really_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { + // If there was overflow, pretend the first character isn't a backslash + backslash &= ~prev_escaped; + uint64_t follows_escape = backslash << 1 | prev_escaped; + + // Get sequences starting on even bits by clearing out the odd series using + + const uint64_t even_bits = 0x5555555555555555ULL; + uint64_t odd_sequence_starts = backslash & ~even_bits & ~follows_escape; + uint64_t sequences_starting_on_even_bits; + prev_escaped = add_overflow(odd_sequence_starts, backslash, &sequences_starting_on_even_bits); + uint64_t invert_mask = sequences_starting_on_even_bits << 1; // The mask we want to return is the *escaped* bits, not escapes. + + // Mask every other backslashed character as an escaped character + // Flip the mask for sequences that start on even bits, to correct them + return (even_bits ^ invert_mask) & follows_escape; +} + +// +// Return a mask of all string characters plus end quotes. +// +// prev_escaped is overflow saying whether the next character is escaped. +// prev_in_string is overflow saying whether we're still in a string. +// +// Backslash sequences outside of quotes will be detected in stage 2. +// +really_inline json_string_block json_string_scanner::next(const simd::simd8x64 in) { + const uint64_t backslash = in.eq('\\'); + const uint64_t escaped = find_escaped(backslash); + const uint64_t quote = in.eq('"') & ~escaped; + // prefix_xor flips on bits inside the string (and flips off the end quote). + // Then we xor with prev_in_string: if we were in a string already, its effect is flipped + // (characters inside strings are outside, and characters outside strings are inside). + const uint64_t in_string = prefix_xor(quote) ^ prev_in_string; + // right shift of a signed value expected to be well-defined and standard + // compliant as of C++20, John Regher from Utah U. says this is fine code + prev_in_string = static_cast(static_cast(in_string) >> 63); + // Use ^ to turn the beginning quote off, and the end quote on. + return { + backslash, + escaped, + quote, + in_string + }; +} + +really_inline error_code json_string_scanner::finish(bool streaming) { + if (prev_in_string and (not streaming)) { + return UNCLOSED_STRING; + } + return SUCCESS; +} + +} // namespace stage1 +/* end file src/generic/json_string_scanner.h */ +/* begin file src/generic/json_scanner.h */ +namespace stage1 { + +/** + * A block of scanned json, with information on operators and scalars. + */ +struct json_block { +public: + // the start of structurals that are not inside strings + really_inline uint64_t structural_start() { return potential_structural_start() & ~_string.string_tail(); } + + // operators plus scalar starts like 123, true and "abc" + really_inline uint64_t potential_structural_start() { return _characters.op() | potential_scalar_start(); } + // the start of non-operator runs, like 123, true and "abc" + really_inline uint64_t potential_scalar_start() { return _characters.scalar() & ~follows_potential_scalar(); } + // whether the given character is immediately after a non-operator like 123, true or " + really_inline uint64_t follows_potential_scalar() { return _follows_potential_scalar; } + // Return a mask of whether the given characters are inside a string (only works on non-quotes) + really_inline uint64_t non_quote_inside_string(uint64_t mask) { return _string.non_quote_inside_string(mask); } + // string and escape characters + json_string_block _string; + // whitespace, operators, scalars + json_character_block _characters; + // whether the previous character was a scalar + uint64_t _follows_potential_scalar; +}; + +/** + * Scans JSON for important bits: operators, strings, and scalars. + * + * The scanner starts by calculating two distinct things: + * - string characters (taking \" into account) + * - operators ([]{},:) and scalars (runs of non-operators like 123, true and "abc") + * + * To minimize data dependency (a key component of the scanner's speed), it finds these in parallel: + * in particular, the operator/scalar bit will find plenty of things that are actually part of + * strings. When we're done, json_block will fuse the two together by masking out tokens that are + * part of a string. + */ +class json_scanner { +public: + really_inline json_block next(const simd::simd8x64 in); + really_inline error_code finish(bool streaming); + +private: + // Whether the last character of the previous iteration is part of a scalar token + // (anything except whitespace or an operator). + uint64_t prev_scalar = 0ULL; + json_string_scanner string_scanner; +}; + + +// +// Check if the current character immediately follows a matching character. +// +// For example, this checks for quotes with backslashes in front of them: +// +// const uint64_t backslashed_quote = in.eq('"') & immediately_follows(in.eq('\'), prev_backslash); +// +really_inline uint64_t follows(const uint64_t match, uint64_t &overflow) { + const uint64_t result = match << 1 | overflow; + overflow = match >> 63; + return result; +} + +// +// Check if the current character follows a matching character, with possible "filler" between. +// For example, this checks for empty curly braces, e.g. +// +// in.eq('}') & follows(in.eq('['), in.eq(' '), prev_empty_array) // { * } +// +really_inline uint64_t follows(const uint64_t match, const uint64_t filler, uint64_t &overflow) { + uint64_t follows_match = follows(match, overflow); + uint64_t result; + overflow |= uint64_t(add_overflow(follows_match, filler, &result)); + return result; +} + +really_inline json_block json_scanner::next(const simd::simd8x64 in) { + json_string_block strings = string_scanner.next(in); + json_character_block characters = json_character_block::classify(in); + uint64_t follows_scalar = follows(characters.scalar(), prev_scalar); + return { + strings, + characters, + follows_scalar + }; +} + +really_inline error_code json_scanner::finish(bool streaming) { + return string_scanner.finish(streaming); +} + +} // namespace stage1 +/* end file src/generic/json_scanner.h */ +/* begin file src/generic/json_structural_indexer.h */ // This file contains the common code every implementation uses in stage1 // It is intended to be included multiple times and compiled multiple times // We assume the file in which it is included already includes @@ -1997,14 +2148,14 @@ class bit_indexer { public: uint32_t *tail; - bit_indexer(uint32_t *index_buf) : tail(index_buf) {} + really_inline bit_indexer(uint32_t *index_buf) : tail(index_buf) {} // flatten out values in 'bits' assuming that they are are to have values of idx // plus their position in the bitvector, and store these indexes at // base_ptr[base] incrementing base as we go // will potentially store extra values beyond end of valid bits, so base_ptr // needs to be large enough to handle this - really_inline void write_indexes(uint32_t idx, uint64_t bits) { + really_inline void write(uint32_t idx, uint64_t bits) { // In some instances, the next branch is expensive because it is mispredicted. // Unfortunately, in other cases, // it helps tremendously. @@ -2043,74 +2194,6 @@ public: } }; -class json_structural_scanner { -public: - // Whether the first character of the next iteration is escaped. - uint64_t prev_escaped = 0ULL; - // Whether the last iteration was still inside a string (all 1's = true, all 0's = false). - uint64_t prev_in_string = 0ULL; - // Whether the last character of the previous iteration is a primitive value character - // (anything except whitespace, braces, comma or colon). - uint64_t prev_primitive = 0ULL; - // Mask of structural characters from the last iteration. - // Kept around for performance reasons, so we can call flatten_bits to soak up some unused - // CPU capacity while the next iteration is busy with an expensive clmul in compute_quote_mask. - uint64_t prev_structurals = 0; - // Errors with unescaped characters in strings (ASCII codepoints < 0x20) - uint64_t unescaped_chars_error = 0; - bit_indexer structural_indexes; - - json_structural_scanner(uint32_t *_structural_indexes) : structural_indexes{_structural_indexes} {} - - // - // Finish the scan and return any errors. - // - // This may detect errors as well, such as unclosed string and certain UTF-8 errors. - // if streaming is set to true, an unclosed string is allowed. - // - really_inline error_code detect_errors_on_eof(bool streaming = false); - - // - // Return a mask of all string characters plus end quotes. - // - // prev_escaped is overflow saying whether the next character is escaped. - // prev_in_string is overflow saying whether we're still in a string. - // - // Backslash sequences outside of quotes will be detected in stage 2. - // - really_inline uint64_t find_strings(const simd::simd8x64 in); - - // - // Determine which characters are *structural*: - // - braces: [] and {} - // - the start of primitives (123, true, false, null) - // - the start of invalid non-whitespace (+, &, ture, UTF-8) - // - // Also detects value sequence errors: - // - two values with no separator between ("hello" "world") - // - separators with no values ([1,] [1,,]and [,2]) - // - // This method will find all of the above whether it is in a string or not. - // - // To reduce dependency on the expensive "what is in a string" computation, this method treats the - // contents of a string the same as content outside. Errors and structurals inside the string or on - // the trailing quote will need to be removed later when the correct string information is known. - // - really_inline uint64_t find_potential_structurals(const simd::simd8x64 in); - - // - // Find the important bits of JSON in a STEP_SIZE-byte chunk, and add them to structural_indexes. - // - template - really_inline void scan_step(const uint8_t *buf, const size_t idx, utf8_checker &utf8_checker); - - // - // Parse the entire input in STEP_SIZE-byte chunks. - // - template - really_inline void scan(const uint8_t *buf, const size_t len, utf8_checker &utf8_checker); -}; - // Routines to print masks and text for debugging bitmask operations UNUSED static char * format_input_text(const simd8x64 in) { static char *buf = (char*)malloc(sizeof(simd8x64) + 1); @@ -2131,267 +2214,74 @@ UNUSED static char * format_mask(uint64_t mask) { return buf; } -// -// Finds escaped characters (characters following \). -// -// Handles runs of backslashes like \\\" and \\\\" correctly (yielding 0101 and 01010, respectively). -// -// Does this by: -// - Shift the escape mask to get potentially escaped characters (characters after backslashes). -// - Mask escaped sequences that start on *even* bits with 1010101010 (odd bits are escaped, even bits are not) -// - Mask escaped sequences that start on *odd* bits with 0101010101 (even bits are escaped, odd bits are not) -// -// To distinguish between escaped sequences starting on even/odd bits, it finds the start of all -// escape sequences, filters out the ones that start on even bits, and adds that to the mask of -// escape sequences. This causes the addition to clear out the sequences starting on odd bits (since -// the start bit causes a carry), and leaves even-bit sequences alone. -// -// Example: -// -// text | \\\ | \\\"\\\" \\\" \\"\\" | -// escape | xxx | xx xxx xxx xx xx | Removed overflow backslash; will | it into follows_escape -// odd_starts | x | x x x | escape & ~even_bits & ~follows_escape -// even_seq | c| cxxx c xx c | c = carry bit -- will be masked out later -// invert_mask | | cxxx c xx c| even_seq << 1 -// follows_escape | xx | x xx xxx xxx xx xx | Includes overflow bit -// escaped | x | x x x x x x x x | -// desired | x | x x x x x x x x | -// text | \\\ | \\\"\\\" \\\" \\"\\" | -// -really_inline uint64_t find_escaped(uint64_t escape, uint64_t &escaped_overflow) { - // If there was overflow, pretend the first character isn't a backslash - escape &= ~escaped_overflow; - uint64_t follows_escape = escape << 1 | escaped_overflow; - - // Get sequences starting on even bits by clearing out the odd series using + - const uint64_t even_bits = 0x5555555555555555ULL; - uint64_t odd_sequence_starts = escape & ~even_bits & ~follows_escape; - uint64_t sequences_starting_on_even_bits; - escaped_overflow = add_overflow(odd_sequence_starts, escape, &sequences_starting_on_even_bits); - uint64_t invert_mask = sequences_starting_on_even_bits << 1; // The mask we want to return is the *escaped* bits, not escapes. - - // Mask every other backslashed character as an escaped character - // Flip the mask for sequences that start on even bits, to correct them - return (even_bits ^ invert_mask) & follows_escape; -} - -// -// Check if the current character immediately follows a matching character. -// -// For example, this checks for quotes with backslashes in front of them: -// -// const uint64_t backslashed_quote = in.eq('"') & immediately_follows(in.eq('\'), prev_backslash); -// -really_inline uint64_t follows(const uint64_t match, uint64_t &overflow) { - const uint64_t result = match << 1 | overflow; - overflow = match >> 63; - return result; -} - -// -// Check if the current character follows a matching character, with possible "filler" between. -// For example, this checks for empty curly braces, e.g. -// -// in.eq('}') & follows(in.eq('['), in.eq(' '), prev_empty_array) // { * } -// -really_inline uint64_t follows(const uint64_t match, const uint64_t filler, uint64_t &overflow) { - uint64_t follows_match = follows(match, overflow); - uint64_t result; - overflow |= uint64_t(add_overflow(follows_match, filler, &result)); - return result; -} - -really_inline error_code json_structural_scanner::detect_errors_on_eof(bool streaming) { - if ((prev_in_string) and (not streaming)) { - return UNCLOSED_STRING; +// Walks through a buffer in block-sized increments, loading the last part with spaces +template +struct buf_block_reader { +public: + really_inline buf_block_reader(const uint8_t *_buf, size_t _len) : buf{_buf}, len{_len}, lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {} + really_inline size_t block_index() { return idx; } + really_inline bool has_full_block() const { + return idx < lenminusstep; } + really_inline const uint8_t *full_block() const { + return &buf[idx]; + } + really_inline bool has_remainder() const { + return idx < len; + } + really_inline void get_remainder(uint8_t *tmp_buf) const { + memset(tmp_buf, 0x20, STEP_SIZE); + memcpy(tmp_buf, buf + idx, len - idx); + } + really_inline void advance() { + idx += STEP_SIZE; + } +private: + const uint8_t *buf; + const size_t len; + const size_t lenminusstep; + size_t idx; +}; + +class json_structural_indexer { +public: + template + static error_code index(const uint8_t *buf, size_t len, document::parser &parser, bool streaming) noexcept; + +private: + really_inline json_structural_indexer(uint32_t *structural_indexes) : indexer{structural_indexes} {} + template + really_inline void index_step(const uint8_t *block, buf_block_reader &reader) noexcept; + really_inline void next(simd::simd8x64 in, json_block block, size_t idx); + really_inline error_code finish(document::parser &parser, size_t idx, size_t len, bool streaming); + + json_scanner scanner; + utf8_checker checker{}; + bit_indexer indexer; + uint64_t prev_structurals = 0; + uint64_t unescaped_chars_error = 0; +}; + +really_inline void json_structural_indexer::next(simd::simd8x64 in, json_block block, size_t idx) { + uint64_t unescaped = in.lteq(0x1F); + checker.check_next_input(in); + indexer.write(idx-64, prev_structurals); // Output *last* iteration's structurals to the parser + prev_structurals = block.structural_start(); + unescaped_chars_error |= block.non_quote_inside_string(unescaped); +} + +really_inline error_code json_structural_indexer::finish(document::parser &parser, size_t idx, size_t len, bool streaming) { + // Write out the final iteration's structurals + indexer.write(idx-64, prev_structurals); + + error_code error = scanner.finish(streaming); + if (unlikely(error != SUCCESS)) { return error; } + if (unescaped_chars_error) { return UNESCAPED_CHARS; } - return SUCCESS; -} -// -// Return a mask of all string characters plus end quotes. -// -// prev_escaped is overflow saying whether the next character is escaped. -// prev_in_string is overflow saying whether we're still in a string. -// -// Backslash sequences outside of quotes will be detected in stage 2. -// -really_inline uint64_t json_structural_scanner::find_strings(const simd::simd8x64 in) { - const uint64_t backslash = in.eq('\\'); - const uint64_t escaped = find_escaped(backslash, prev_escaped); - const uint64_t quote = in.eq('"') & ~escaped; - // prefix_xor flips on bits inside the string (and flips off the end quote). - const uint64_t in_string = prefix_xor(quote) ^ prev_in_string; - /* right shift of a signed value expected to be well-defined and standard - * compliant as of C++20, - * John Regher from Utah U. says this is fine code */ - prev_in_string = static_cast(static_cast(in_string) >> 63); - // Use ^ to turn the beginning quote off, and the end quote on. - return in_string ^ quote; -} - -// -// Determine which characters are *structural*: -// - braces: [] and {} -// - the start of primitives (123, true, false, null) -// - the start of invalid non-whitespace (+, &, ture, UTF-8) -// -// Also detects value sequence errors: -// - two values with no separator between ("hello" "world") -// - separators with no values ([1,] [1,,]and [,2]) -// -// This method will find all of the above whether it is in a string or not. -// -// To reduce dependency on the expensive "what is in a string" computation, this method treats the -// contents of a string the same as content outside. Errors and structurals inside the string or on -// the trailing quote will need to be removed later when the correct string information is known. -// -really_inline uint64_t json_structural_scanner::find_potential_structurals(const simd::simd8x64 in) { - // These use SIMD so let's kick them off before running the regular 64-bit stuff ... - uint64_t whitespace, op; - find_whitespace_and_operators(in, whitespace, op); - - // Detect the start of a run of primitive characters. Includes numbers, booleans, and strings ("). - // Everything except whitespace, braces, colon and comma. - const uint64_t primitive = ~(op | whitespace); - const uint64_t follows_primitive = follows(primitive, prev_primitive); - const uint64_t start_primitive = primitive & ~follows_primitive; - - // Return final structurals - return op | start_primitive; -} - -// -// Find the important bits of JSON in a 128-byte chunk, and add them to structural_indexes. -// -// PERF NOTES: -// We pipe 2 inputs through these stages: -// 1. Load JSON into registers. This takes a long time and is highly parallelizable, so we load -// 2 inputs' worth at once so that by the time step 2 is looking for them input, it's available. -// 2. Scan the JSON for critical data: strings, primitives and operators. This is the critical path. -// The output of step 1 depends entirely on this information. These functions don't quite use -// up enough CPU: the second half of the functions is highly serial, only using 1 execution core -// at a time. The second input's scans has some dependency on the first ones finishing it, but -// they can make a lot of progress before they need that information. -// 3. Step 1 doesn't use enough capacity, so we run some extra stuff while we're waiting for that -// to finish: utf-8 checks and generating the output from the last iteration. -// -// The reason we run 2 inputs at a time, is steps 2 and 3 are *still* not enough to soak up all -// available capacity with just one input. Running 2 at a time seems to give the CPU a good enough -// workout. -// -template<> -really_inline void json_structural_scanner::scan_step<128>(const uint8_t *buf, const size_t idx, utf8_checker &utf8_checker) { - // - // Load up all 128 bytes into SIMD registers - // - simd::simd8x64 in_1(buf); - simd::simd8x64 in_2(buf+64); - - // - // Find the strings and potential structurals (operators / primitives). - // - // This will include false structurals that are *inside* strings--we'll filter strings out - // before we return. - // - uint64_t string_1 = this->find_strings(in_1); - uint64_t structurals_1 = this->find_potential_structurals(in_1); - uint64_t string_2 = this->find_strings(in_2); - uint64_t structurals_2 = this->find_potential_structurals(in_2); - - // - // Do miscellaneous work while the processor is busy calculating strings and structurals. - // - // After that, weed out structurals that are inside strings and find invalid string characters. - // - uint64_t unescaped_1 = in_1.lteq(0x1F); - utf8_checker.check_next_input(in_1); - this->structural_indexes.write_indexes(idx-64, this->prev_structurals); // Output *last* iteration's structurals to the parser - this->prev_structurals = structurals_1 & ~string_1; - this->unescaped_chars_error |= unescaped_1 & string_1; - - uint64_t unescaped_2 = in_2.lteq(0x1F); - utf8_checker.check_next_input(in_2); - this->structural_indexes.write_indexes(idx, this->prev_structurals); // Output *last* iteration's structurals to the parser - this->prev_structurals = structurals_2 & ~string_2; - this->unescaped_chars_error |= unescaped_2 & string_2; -} - -// -// Find the important bits of JSON in a 64-byte chunk, and add them to structural_indexes. -// -template<> -really_inline void json_structural_scanner::scan_step<64>(const uint8_t *buf, const size_t idx, utf8_checker &utf8_checker) { - // - // Load up bytes into SIMD registers - // - simd::simd8x64 in_1(buf); - - // - // Find the strings and potential structurals (operators / primitives). - // - // This will include false structurals that are *inside* strings--we'll filter strings out - // before we return. - // - uint64_t string_1 = this->find_strings(in_1); - uint64_t structurals_1 = this->find_potential_structurals(in_1); - - // - // Do miscellaneous work while the processor is busy calculating strings and structurals. - // - // After that, weed out structurals that are inside strings and find invalid string characters. - // - uint64_t unescaped_1 = in_1.lteq(0x1F); - utf8_checker.check_next_input(in_1); - this->structural_indexes.write_indexes(idx-64, this->prev_structurals); // Output *last* iteration's structurals to ParsedJson - this->prev_structurals = structurals_1 & ~string_1; - this->unescaped_chars_error |= unescaped_1 & string_1; -} - -template -really_inline void json_structural_scanner::scan(const uint8_t *buf, const size_t len, utf8_checker &utf8_checker) { - size_t lenminusstep = len < STEP_SIZE ? 0 : len - STEP_SIZE; - size_t idx = 0; - - for (; idx < lenminusstep; idx += STEP_SIZE) { - this->scan_step(&buf[idx], idx, utf8_checker); - } - - /* If we have a final chunk of less than STEP_SIZE bytes, pad it to STEP_SIZE with - * spaces before processing it (otherwise, we risk invalidating the UTF-8 - * checks). */ - if (likely(idx < len)) { - uint8_t tmp_buf[STEP_SIZE]; - memset(tmp_buf, 0x20, STEP_SIZE); - memcpy(tmp_buf, buf + idx, len - idx); - this->scan_step(&tmp_buf[0], idx, utf8_checker); - idx += STEP_SIZE; - } - - /* finally, flatten out the remaining structurals from the last iteration */ - this->structural_indexes.write_indexes(idx-64, this->prev_structurals); -} - -// Setting the streaming parameter to true allows the find_structural_bits to tolerate unclosed strings. -// The caller should still ensure that the input is valid UTF-8. If you are processing substrings, -// you may want to call on a function like trimmed_length_safe_utf8. -template -error_code find_structural_bits(const uint8_t *buf, size_t len, document::parser &parser, bool streaming) { - if (unlikely(len > parser.capacity())) { - return CAPACITY; - } - utf8_checker utf8_checker{}; - json_structural_scanner scanner{parser.structural_indexes.get()}; - scanner.scan(buf, len, utf8_checker); - // we might tolerate an unclosed string if streaming is true - error_code error = scanner.detect_errors_on_eof(streaming); - if (unlikely(error != SUCCESS)) { - return error; - } - parser.n_structural_indexes = scanner.structural_indexes.tail - parser.structural_indexes.get(); + parser.n_structural_indexes = indexer.tail - parser.structural_indexes.get(); /* a valid JSON file cannot have zero structural indexes - we should have * found something */ if (unlikely(parser.n_structural_indexes == 0u)) { @@ -2407,51 +2297,262 @@ error_code find_structural_bits(const uint8_t *buf, size_t len, document::parser } /* make it safe to dereference one beyond this array */ parser.structural_indexes[parser.n_structural_indexes] = 0; - return utf8_checker.errors(); + return checker.errors(); +} + +template<> +really_inline void json_structural_indexer::index_step<128>(const uint8_t *block, buf_block_reader<128> &reader) noexcept { + simd::simd8x64 in_1(block); + simd::simd8x64 in_2(block+64); + json_block block_1 = scanner.next(in_1); + json_block block_2 = scanner.next(in_2); + this->next(in_1, block_1, reader.block_index()); + this->next(in_2, block_2, reader.block_index()+64); + reader.advance(); +} + +template<> +really_inline void json_structural_indexer::index_step<64>(const uint8_t *block, buf_block_reader<64> &reader) noexcept { + simd::simd8x64 in_1(block); + json_block block_1 = scanner.next(in_1); + this->next(in_1, block_1, reader.block_index()); + reader.advance(); +} + +// +// Find the important bits of JSON in a 128-byte chunk, and add them to structural_indexes. +// +// PERF NOTES: +// We pipe 2 inputs through these stages: +// 1. Load JSON into registers. This takes a long time and is highly parallelizable, so we load +// 2 inputs' worth at once so that by the time step 2 is looking for them input, it's available. +// 2. Scan the JSON for critical data: strings, scalars and operators. This is the critical path. +// The output of step 1 depends entirely on this information. These functions don't quite use +// up enough CPU: the second half of the functions is highly serial, only using 1 execution core +// at a time. The second input's scans has some dependency on the first ones finishing it, but +// they can make a lot of progress before they need that information. +// 3. Step 1 doesn't use enough capacity, so we run some extra stuff while we're waiting for that +// to finish: utf-8 checks and generating the output from the last iteration. +// +// The reason we run 2 inputs at a time, is steps 2 and 3 are *still* not enough to soak up all +// available capacity with just one input. Running 2 at a time seems to give the CPU a good enough +// workout. +// +// Setting the streaming parameter to true allows the find_structural_bits to tolerate unclosed strings. +// The caller should still ensure that the input is valid UTF-8. If you are processing substrings, +// you may want to call on a function like trimmed_length_safe_utf8. +template +error_code json_structural_indexer::index(const uint8_t *buf, size_t len, document::parser &parser, bool streaming) noexcept { + if (unlikely(len > parser.capacity())) { return CAPACITY; } + + buf_block_reader reader(buf, len); + json_structural_indexer indexer(parser.structural_indexes.get()); + while (reader.has_full_block()) { + indexer.index_step(reader.full_block(), reader); + } + + if (likely(reader.has_remainder())) { + uint8_t block[STEP_SIZE]; + reader.get_remainder(block); + indexer.index_step(block, reader); + } + + return indexer.finish(parser, reader.block_index(), len, streaming); } } // namespace stage1 -/* end file src/generic/stage1_find_marks.h */ +/* end file src/generic/json_structural_indexer.h */ WARN_UNUSED error_code implementation::stage1(const uint8_t *buf, size_t len, document::parser &parser, bool streaming) const noexcept { - return arm64::stage1::find_structural_bits<64>(buf, len, parser, streaming); + return arm64::stage1::json_structural_indexer::index<64>(buf, len, parser, streaming); } } // namespace simdjson::arm64 -#endif // IS_ARM64 - #endif // SIMDJSON_ARM64_STAGE1_FIND_MARKS_H -/* end file src/generic/stage1_find_marks.h */ +/* end file src/generic/json_structural_indexer.h */ +#endif +#if SIMDJSON_IMPLEMENTATION_FALLBACK +/* begin file src/fallback/stage1_find_marks.h */ +#ifndef SIMDJSON_FALLBACK_STAGE1_FIND_MARKS_H +#define SIMDJSON_FALLBACK_STAGE1_FIND_MARKS_H + +/* fallback/implementation.h already included: #include "fallback/implementation.h" */ + +namespace simdjson::fallback::stage1 { + +class structural_scanner { +public: + +really_inline structural_scanner(const uint8_t *_buf, uint32_t _len, document::parser &_doc_parser, bool _streaming) + : buf{_buf}, next_structural_index{_doc_parser.structural_indexes.get()}, doc_parser{_doc_parser}, idx{0}, len{_len}, error{SUCCESS}, streaming{_streaming} {} + +really_inline void add_structural() { + *next_structural_index = idx; + next_structural_index++; +} + +really_inline bool is_continuation(uint8_t c) { + return (c & 0b11000000) == 0b10000000; +} + +really_inline void validate_utf8_character() { + // Continuation + if (unlikely((buf[idx] & 0b01000000) == 0)) { + // extra continuation + error = UTF8_ERROR; + idx++; + return; + } + + // 2-byte + if ((buf[idx] & 0b00100000) == 0) { + // missing continuation + if (unlikely(idx+1 > len || !is_continuation(buf[idx+1]))) { error = UTF8_ERROR; idx++; return; } + // overlong: 1100000_ 10______ + if (buf[idx] <= 0b11000001) { error = UTF8_ERROR; } + idx += 2; + return; + } + + // 3-byte + if ((buf[idx] & 0b00010000) == 0) { + // missing continuation + if (unlikely(idx+2 > len || !is_continuation(buf[idx+1]) || !is_continuation(buf[idx+2]))) { error = UTF8_ERROR; idx++; return; } + // overlong: 11100000 100_____ ________ + if (buf[idx] == 0b11100000 && buf[idx+1] <= 0b10011111) { error = UTF8_ERROR; } + // surrogates: U+D800-U+DFFF 11101101 101_____ + if (buf[idx] == 0b11101101 && buf[idx+1] >= 0b10100000) { error = UTF8_ERROR; } + idx += 3; + return; + } + + // 4-byte + // missing continuation + if (unlikely(idx+3 > len || !is_continuation(buf[idx+1]) || !is_continuation(buf[idx+2]) || !is_continuation(buf[idx+3]))) { error = UTF8_ERROR; idx++; return; } + // overlong: 11110000 1000____ ________ ________ + if (buf[idx] == 0b11110000 && buf[idx+1] <= 0b10001111) { error = UTF8_ERROR; } + // too large: > U+10FFFF: + // 11110100 (1001|101_)____ + // 1111(1___|011_|0101) 10______ + // also includes 5, 6, 7 and 8 byte characters: + // 11111___ + if (buf[idx] == 0b11110100 && buf[idx+1] >= 0b10010000) { error = UTF8_ERROR; } + if (buf[idx] >= 0b11110101) { error = UTF8_ERROR; } + idx += 4; +} + +really_inline void validate_string() { + idx++; // skip first quote + while (idx < len && buf[idx] != '"') { + if (buf[idx] == '\\') { + idx += 2; + } else if (unlikely(buf[idx] & 0b10000000)) { + validate_utf8_character(); + } else { + if (buf[idx] < 0x20) { error = UNESCAPED_CHARS; } + idx++; + } + } + if (idx >= len && !streaming) { error = UNCLOSED_STRING; } +} + +really_inline bool is_whitespace_or_operator(uint8_t c) { + switch (c) { + case '{': case '}': case '[': case ']': case ',': case ':': + case ' ': case '\r': case '\n': case '\t': + return true; + default: + return false; + } +} + +// +// Parse the entire input in STEP_SIZE-byte chunks. +// +really_inline error_code scan() { + for (;idx parser.capacity())) { + return CAPACITY; + } + stage1::structural_scanner scanner(buf, len, parser, streaming); + return scanner.scan(); +} + +} // namespace simdjson::fallback + +#endif // SIMDJSON_FALLBACK_STAGE1_FIND_MARKS_H +/* end file src/fallback/stage1_find_marks.h */ +#endif +#if SIMDJSON_IMPLEMENTATION_HASWELL /* begin file src/haswell/stage1_find_marks.h */ #ifndef SIMDJSON_HASWELL_STAGE1_FIND_MARKS_H #define SIMDJSON_HASWELL_STAGE1_FIND_MARKS_H -#ifdef IS_X86_64 - /* begin file src/haswell/bitmask.h */ #ifndef SIMDJSON_HASWELL_BITMASK_H #define SIMDJSON_HASWELL_BITMASK_H -#ifdef IS_X86_64 - /* begin file src/haswell/intrinsics.h */ #ifndef SIMDJSON_HASWELL_INTRINSICS_H #define SIMDJSON_HASWELL_INTRINSICS_H -#ifdef IS_X86_64 - #ifdef _MSC_VER #include // visual studio #else #include // elsewhere #endif // _MSC_VER -#endif // IS_X86_64 - #endif // SIMDJSON_HASWELL_INTRINSICS_H /* end file src/haswell/intrinsics.h */ @@ -2474,17 +2575,12 @@ really_inline uint64_t prefix_xor(const uint64_t bitmask) { } // namespace simdjson::haswell UNTARGET_REGION -#endif // IS_X86_64 - #endif // SIMDJSON_HASWELL_BITMASK_H /* end file src/haswell/intrinsics.h */ /* begin file src/haswell/simd.h */ #ifndef SIMDJSON_HASWELL_SIMD_H #define SIMDJSON_HASWELL_SIMD_H - -#ifdef IS_X86_64 - /* haswell/intrinsics.h already included: #include "haswell/intrinsics.h" */ TARGET_HASWELL @@ -2791,8 +2887,6 @@ namespace simdjson::haswell::simd { } // namespace simdjson::haswell::simd UNTARGET_REGION -#endif // IS_X86_64 - #endif // SIMDJSON_HASWELL_SIMD_H /* end file src/haswell/simd.h */ /* begin file src/haswell/bitmanipulation.h */ @@ -2800,8 +2894,6 @@ UNTARGET_REGION #define SIMDJSON_HASWELL_BITMANIPULATION_H -#ifdef IS_X86_64 - /* haswell/intrinsics.h already included: #include "haswell/intrinsics.h" */ TARGET_HASWELL @@ -2873,8 +2965,6 @@ really_inline bool mul_overflow(uint64_t value1, uint64_t value2, }// namespace simdjson::haswell UNTARGET_REGION -#endif // IS_X86_64 - #endif // SIMDJSON_HASWELL_BITMANIPULATION_H /* end file src/haswell/bitmanipulation.h */ /* haswell/implementation.h already included: #include "haswell/implementation.h" */ @@ -2884,21 +2974,32 @@ namespace simdjson::haswell { using namespace simd; -really_inline void find_whitespace_and_operators(simd8x64 in, uint64_t &whitespace, uint64_t &op) { +struct json_character_block { + static really_inline json_character_block classify(const simd::simd8x64 in); + really_inline uint64_t whitespace() const { return _whitespace; } + really_inline uint64_t op() const { return _op; } + really_inline uint64_t scalar() { return ~(op() | whitespace()); } + + uint64_t _whitespace; + uint64_t _op; +}; + +really_inline json_character_block json_character_block::classify(const simd::simd8x64 in) { // These lookups rely on the fact that anything < 127 will match the lower 4 bits, which is why // we can't use the generic lookup_16. auto whitespace_table = simd8::repeat_16(' ', 100, 100, 100, 17, 100, 113, 2, 100, '\t', '\n', 112, 100, '\r', 100, 100); auto op_table = simd8::repeat_16(',', '}', 0, 0, 0xc0u, 0, 0, 0, 0, 0, 0, 0, 0, 0, ':', '{'); - whitespace = in.map([&](simd8 _in) { + uint64_t whitespace = in.map([&](simd8 _in) { return _in == simd8(_mm256_shuffle_epi8(whitespace_table, _in)); }).to_bitmask(); - op = in.map([&](simd8 _in) { + uint64_t op = in.map([&](simd8 _in) { // | 32 handles the fact that { } and [ ] are exactly 32 bytes apart return (_in | 32) == simd8(_mm256_shuffle_epi8(op_table, _in-',')); }).to_bitmask(); + return { whitespace, op }; } really_inline bool is_ascii(simd8x64 input) { @@ -2923,7 +3024,7 @@ really_inline simd8 must_be_continuation(simd8 prev1, simd8 in); + really_inline error_code finish(bool streaming); + +private: + really_inline uint64_t find_escaped(uint64_t escape); + + // Whether the last iteration was still inside a string (all 1's = true, all 0's = false). + uint64_t prev_in_string = 0ULL; + // Whether the first character of the next iteration is escaped. + uint64_t prev_escaped = 0ULL; +}; + +// +// Finds escaped characters (characters following \). +// +// Handles runs of backslashes like \\\" and \\\\" correctly (yielding 0101 and 01010, respectively). +// +// Does this by: +// - Shift the escape mask to get potentially escaped characters (characters after backslashes). +// - Mask escaped sequences that start on *even* bits with 1010101010 (odd bits are escaped, even bits are not) +// - Mask escaped sequences that start on *odd* bits with 0101010101 (even bits are escaped, odd bits are not) +// +// To distinguish between escaped sequences starting on even/odd bits, it finds the start of all +// escape sequences, filters out the ones that start on even bits, and adds that to the mask of +// escape sequences. This causes the addition to clear out the sequences starting on odd bits (since +// the start bit causes a carry), and leaves even-bit sequences alone. +// +// Example: +// +// text | \\\ | \\\"\\\" \\\" \\"\\" | +// escape | xxx | xx xxx xxx xx xx | Removed overflow backslash; will | it into follows_escape +// odd_starts | x | x x x | escape & ~even_bits & ~follows_escape +// even_seq | c| cxxx c xx c | c = carry bit -- will be masked out later +// invert_mask | | cxxx c xx c| even_seq << 1 +// follows_escape | xx | x xx xxx xxx xx xx | Includes overflow bit +// escaped | x | x x x x x x x x | +// desired | x | x x x x x x x x | +// text | \\\ | \\\"\\\" \\\" \\"\\" | +// +really_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { + // If there was overflow, pretend the first character isn't a backslash + backslash &= ~prev_escaped; + uint64_t follows_escape = backslash << 1 | prev_escaped; + + // Get sequences starting on even bits by clearing out the odd series using + + const uint64_t even_bits = 0x5555555555555555ULL; + uint64_t odd_sequence_starts = backslash & ~even_bits & ~follows_escape; + uint64_t sequences_starting_on_even_bits; + prev_escaped = add_overflow(odd_sequence_starts, backslash, &sequences_starting_on_even_bits); + uint64_t invert_mask = sequences_starting_on_even_bits << 1; // The mask we want to return is the *escaped* bits, not escapes. + + // Mask every other backslashed character as an escaped character + // Flip the mask for sequences that start on even bits, to correct them + return (even_bits ^ invert_mask) & follows_escape; +} + +// +// Return a mask of all string characters plus end quotes. +// +// prev_escaped is overflow saying whether the next character is escaped. +// prev_in_string is overflow saying whether we're still in a string. +// +// Backslash sequences outside of quotes will be detected in stage 2. +// +really_inline json_string_block json_string_scanner::next(const simd::simd8x64 in) { + const uint64_t backslash = in.eq('\\'); + const uint64_t escaped = find_escaped(backslash); + const uint64_t quote = in.eq('"') & ~escaped; + // prefix_xor flips on bits inside the string (and flips off the end quote). + // Then we xor with prev_in_string: if we were in a string already, its effect is flipped + // (characters inside strings are outside, and characters outside strings are inside). + const uint64_t in_string = prefix_xor(quote) ^ prev_in_string; + // right shift of a signed value expected to be well-defined and standard + // compliant as of C++20, John Regher from Utah U. says this is fine code + prev_in_string = static_cast(static_cast(in_string) >> 63); + // Use ^ to turn the beginning quote off, and the end quote on. + return { + backslash, + escaped, + quote, + in_string + }; +} + +really_inline error_code json_string_scanner::finish(bool streaming) { + if (prev_in_string and (not streaming)) { + return UNCLOSED_STRING; + } + return SUCCESS; +} + +} // namespace stage1 +/* end file src/generic/json_string_scanner.h */ +/* begin file src/generic/json_scanner.h */ +namespace stage1 { + +/** + * A block of scanned json, with information on operators and scalars. + */ +struct json_block { +public: + // the start of structurals that are not inside strings + really_inline uint64_t structural_start() { return potential_structural_start() & ~_string.string_tail(); } + + // operators plus scalar starts like 123, true and "abc" + really_inline uint64_t potential_structural_start() { return _characters.op() | potential_scalar_start(); } + // the start of non-operator runs, like 123, true and "abc" + really_inline uint64_t potential_scalar_start() { return _characters.scalar() & ~follows_potential_scalar(); } + // whether the given character is immediately after a non-operator like 123, true or " + really_inline uint64_t follows_potential_scalar() { return _follows_potential_scalar; } + // Return a mask of whether the given characters are inside a string (only works on non-quotes) + really_inline uint64_t non_quote_inside_string(uint64_t mask) { return _string.non_quote_inside_string(mask); } + // string and escape characters + json_string_block _string; + // whitespace, operators, scalars + json_character_block _characters; + // whether the previous character was a scalar + uint64_t _follows_potential_scalar; +}; + +/** + * Scans JSON for important bits: operators, strings, and scalars. + * + * The scanner starts by calculating two distinct things: + * - string characters (taking \" into account) + * - operators ([]{},:) and scalars (runs of non-operators like 123, true and "abc") + * + * To minimize data dependency (a key component of the scanner's speed), it finds these in parallel: + * in particular, the operator/scalar bit will find plenty of things that are actually part of + * strings. When we're done, json_block will fuse the two together by masking out tokens that are + * part of a string. + */ +class json_scanner { +public: + really_inline json_block next(const simd::simd8x64 in); + really_inline error_code finish(bool streaming); + +private: + // Whether the last character of the previous iteration is part of a scalar token + // (anything except whitespace or an operator). + uint64_t prev_scalar = 0ULL; + json_string_scanner string_scanner; +}; + + +// +// Check if the current character immediately follows a matching character. +// +// For example, this checks for quotes with backslashes in front of them: +// +// const uint64_t backslashed_quote = in.eq('"') & immediately_follows(in.eq('\'), prev_backslash); +// +really_inline uint64_t follows(const uint64_t match, uint64_t &overflow) { + const uint64_t result = match << 1 | overflow; + overflow = match >> 63; + return result; +} + +// +// Check if the current character follows a matching character, with possible "filler" between. +// For example, this checks for empty curly braces, e.g. +// +// in.eq('}') & follows(in.eq('['), in.eq(' '), prev_empty_array) // { * } +// +really_inline uint64_t follows(const uint64_t match, const uint64_t filler, uint64_t &overflow) { + uint64_t follows_match = follows(match, overflow); + uint64_t result; + overflow |= uint64_t(add_overflow(follows_match, filler, &result)); + return result; +} + +really_inline json_block json_scanner::next(const simd::simd8x64 in) { + json_string_block strings = string_scanner.next(in); + json_character_block characters = json_character_block::classify(in); + uint64_t follows_scalar = follows(characters.scalar(), prev_scalar); + return { + strings, + characters, + follows_scalar + }; +} + +really_inline error_code json_scanner::finish(bool streaming) { + return string_scanner.finish(streaming); +} + +} // namespace stage1 +/* end file src/generic/json_scanner.h */ +/* begin file src/generic/json_structural_indexer.h */ // This file contains the common code every implementation uses in stage1 // It is intended to be included multiple times and compiled multiple times // We assume the file in which it is included already includes @@ -3353,14 +3676,14 @@ class bit_indexer { public: uint32_t *tail; - bit_indexer(uint32_t *index_buf) : tail(index_buf) {} + really_inline bit_indexer(uint32_t *index_buf) : tail(index_buf) {} // flatten out values in 'bits' assuming that they are are to have values of idx // plus their position in the bitvector, and store these indexes at // base_ptr[base] incrementing base as we go // will potentially store extra values beyond end of valid bits, so base_ptr // needs to be large enough to handle this - really_inline void write_indexes(uint32_t idx, uint64_t bits) { + really_inline void write(uint32_t idx, uint64_t bits) { // In some instances, the next branch is expensive because it is mispredicted. // Unfortunately, in other cases, // it helps tremendously. @@ -3399,74 +3722,6 @@ public: } }; -class json_structural_scanner { -public: - // Whether the first character of the next iteration is escaped. - uint64_t prev_escaped = 0ULL; - // Whether the last iteration was still inside a string (all 1's = true, all 0's = false). - uint64_t prev_in_string = 0ULL; - // Whether the last character of the previous iteration is a primitive value character - // (anything except whitespace, braces, comma or colon). - uint64_t prev_primitive = 0ULL; - // Mask of structural characters from the last iteration. - // Kept around for performance reasons, so we can call flatten_bits to soak up some unused - // CPU capacity while the next iteration is busy with an expensive clmul in compute_quote_mask. - uint64_t prev_structurals = 0; - // Errors with unescaped characters in strings (ASCII codepoints < 0x20) - uint64_t unescaped_chars_error = 0; - bit_indexer structural_indexes; - - json_structural_scanner(uint32_t *_structural_indexes) : structural_indexes{_structural_indexes} {} - - // - // Finish the scan and return any errors. - // - // This may detect errors as well, such as unclosed string and certain UTF-8 errors. - // if streaming is set to true, an unclosed string is allowed. - // - really_inline error_code detect_errors_on_eof(bool streaming = false); - - // - // Return a mask of all string characters plus end quotes. - // - // prev_escaped is overflow saying whether the next character is escaped. - // prev_in_string is overflow saying whether we're still in a string. - // - // Backslash sequences outside of quotes will be detected in stage 2. - // - really_inline uint64_t find_strings(const simd::simd8x64 in); - - // - // Determine which characters are *structural*: - // - braces: [] and {} - // - the start of primitives (123, true, false, null) - // - the start of invalid non-whitespace (+, &, ture, UTF-8) - // - // Also detects value sequence errors: - // - two values with no separator between ("hello" "world") - // - separators with no values ([1,] [1,,]and [,2]) - // - // This method will find all of the above whether it is in a string or not. - // - // To reduce dependency on the expensive "what is in a string" computation, this method treats the - // contents of a string the same as content outside. Errors and structurals inside the string or on - // the trailing quote will need to be removed later when the correct string information is known. - // - really_inline uint64_t find_potential_structurals(const simd::simd8x64 in); - - // - // Find the important bits of JSON in a STEP_SIZE-byte chunk, and add them to structural_indexes. - // - template - really_inline void scan_step(const uint8_t *buf, const size_t idx, utf8_checker &utf8_checker); - - // - // Parse the entire input in STEP_SIZE-byte chunks. - // - template - really_inline void scan(const uint8_t *buf, const size_t len, utf8_checker &utf8_checker); -}; - // Routines to print masks and text for debugging bitmask operations UNUSED static char * format_input_text(const simd8x64 in) { static char *buf = (char*)malloc(sizeof(simd8x64) + 1); @@ -3487,267 +3742,74 @@ UNUSED static char * format_mask(uint64_t mask) { return buf; } -// -// Finds escaped characters (characters following \). -// -// Handles runs of backslashes like \\\" and \\\\" correctly (yielding 0101 and 01010, respectively). -// -// Does this by: -// - Shift the escape mask to get potentially escaped characters (characters after backslashes). -// - Mask escaped sequences that start on *even* bits with 1010101010 (odd bits are escaped, even bits are not) -// - Mask escaped sequences that start on *odd* bits with 0101010101 (even bits are escaped, odd bits are not) -// -// To distinguish between escaped sequences starting on even/odd bits, it finds the start of all -// escape sequences, filters out the ones that start on even bits, and adds that to the mask of -// escape sequences. This causes the addition to clear out the sequences starting on odd bits (since -// the start bit causes a carry), and leaves even-bit sequences alone. -// -// Example: -// -// text | \\\ | \\\"\\\" \\\" \\"\\" | -// escape | xxx | xx xxx xxx xx xx | Removed overflow backslash; will | it into follows_escape -// odd_starts | x | x x x | escape & ~even_bits & ~follows_escape -// even_seq | c| cxxx c xx c | c = carry bit -- will be masked out later -// invert_mask | | cxxx c xx c| even_seq << 1 -// follows_escape | xx | x xx xxx xxx xx xx | Includes overflow bit -// escaped | x | x x x x x x x x | -// desired | x | x x x x x x x x | -// text | \\\ | \\\"\\\" \\\" \\"\\" | -// -really_inline uint64_t find_escaped(uint64_t escape, uint64_t &escaped_overflow) { - // If there was overflow, pretend the first character isn't a backslash - escape &= ~escaped_overflow; - uint64_t follows_escape = escape << 1 | escaped_overflow; - - // Get sequences starting on even bits by clearing out the odd series using + - const uint64_t even_bits = 0x5555555555555555ULL; - uint64_t odd_sequence_starts = escape & ~even_bits & ~follows_escape; - uint64_t sequences_starting_on_even_bits; - escaped_overflow = add_overflow(odd_sequence_starts, escape, &sequences_starting_on_even_bits); - uint64_t invert_mask = sequences_starting_on_even_bits << 1; // The mask we want to return is the *escaped* bits, not escapes. - - // Mask every other backslashed character as an escaped character - // Flip the mask for sequences that start on even bits, to correct them - return (even_bits ^ invert_mask) & follows_escape; -} - -// -// Check if the current character immediately follows a matching character. -// -// For example, this checks for quotes with backslashes in front of them: -// -// const uint64_t backslashed_quote = in.eq('"') & immediately_follows(in.eq('\'), prev_backslash); -// -really_inline uint64_t follows(const uint64_t match, uint64_t &overflow) { - const uint64_t result = match << 1 | overflow; - overflow = match >> 63; - return result; -} - -// -// Check if the current character follows a matching character, with possible "filler" between. -// For example, this checks for empty curly braces, e.g. -// -// in.eq('}') & follows(in.eq('['), in.eq(' '), prev_empty_array) // { * } -// -really_inline uint64_t follows(const uint64_t match, const uint64_t filler, uint64_t &overflow) { - uint64_t follows_match = follows(match, overflow); - uint64_t result; - overflow |= uint64_t(add_overflow(follows_match, filler, &result)); - return result; -} - -really_inline error_code json_structural_scanner::detect_errors_on_eof(bool streaming) { - if ((prev_in_string) and (not streaming)) { - return UNCLOSED_STRING; +// Walks through a buffer in block-sized increments, loading the last part with spaces +template +struct buf_block_reader { +public: + really_inline buf_block_reader(const uint8_t *_buf, size_t _len) : buf{_buf}, len{_len}, lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {} + really_inline size_t block_index() { return idx; } + really_inline bool has_full_block() const { + return idx < lenminusstep; } + really_inline const uint8_t *full_block() const { + return &buf[idx]; + } + really_inline bool has_remainder() const { + return idx < len; + } + really_inline void get_remainder(uint8_t *tmp_buf) const { + memset(tmp_buf, 0x20, STEP_SIZE); + memcpy(tmp_buf, buf + idx, len - idx); + } + really_inline void advance() { + idx += STEP_SIZE; + } +private: + const uint8_t *buf; + const size_t len; + const size_t lenminusstep; + size_t idx; +}; + +class json_structural_indexer { +public: + template + static error_code index(const uint8_t *buf, size_t len, document::parser &parser, bool streaming) noexcept; + +private: + really_inline json_structural_indexer(uint32_t *structural_indexes) : indexer{structural_indexes} {} + template + really_inline void index_step(const uint8_t *block, buf_block_reader &reader) noexcept; + really_inline void next(simd::simd8x64 in, json_block block, size_t idx); + really_inline error_code finish(document::parser &parser, size_t idx, size_t len, bool streaming); + + json_scanner scanner; + utf8_checker checker{}; + bit_indexer indexer; + uint64_t prev_structurals = 0; + uint64_t unescaped_chars_error = 0; +}; + +really_inline void json_structural_indexer::next(simd::simd8x64 in, json_block block, size_t idx) { + uint64_t unescaped = in.lteq(0x1F); + checker.check_next_input(in); + indexer.write(idx-64, prev_structurals); // Output *last* iteration's structurals to the parser + prev_structurals = block.structural_start(); + unescaped_chars_error |= block.non_quote_inside_string(unescaped); +} + +really_inline error_code json_structural_indexer::finish(document::parser &parser, size_t idx, size_t len, bool streaming) { + // Write out the final iteration's structurals + indexer.write(idx-64, prev_structurals); + + error_code error = scanner.finish(streaming); + if (unlikely(error != SUCCESS)) { return error; } + if (unescaped_chars_error) { return UNESCAPED_CHARS; } - return SUCCESS; -} -// -// Return a mask of all string characters plus end quotes. -// -// prev_escaped is overflow saying whether the next character is escaped. -// prev_in_string is overflow saying whether we're still in a string. -// -// Backslash sequences outside of quotes will be detected in stage 2. -// -really_inline uint64_t json_structural_scanner::find_strings(const simd::simd8x64 in) { - const uint64_t backslash = in.eq('\\'); - const uint64_t escaped = find_escaped(backslash, prev_escaped); - const uint64_t quote = in.eq('"') & ~escaped; - // prefix_xor flips on bits inside the string (and flips off the end quote). - const uint64_t in_string = prefix_xor(quote) ^ prev_in_string; - /* right shift of a signed value expected to be well-defined and standard - * compliant as of C++20, - * John Regher from Utah U. says this is fine code */ - prev_in_string = static_cast(static_cast(in_string) >> 63); - // Use ^ to turn the beginning quote off, and the end quote on. - return in_string ^ quote; -} - -// -// Determine which characters are *structural*: -// - braces: [] and {} -// - the start of primitives (123, true, false, null) -// - the start of invalid non-whitespace (+, &, ture, UTF-8) -// -// Also detects value sequence errors: -// - two values with no separator between ("hello" "world") -// - separators with no values ([1,] [1,,]and [,2]) -// -// This method will find all of the above whether it is in a string or not. -// -// To reduce dependency on the expensive "what is in a string" computation, this method treats the -// contents of a string the same as content outside. Errors and structurals inside the string or on -// the trailing quote will need to be removed later when the correct string information is known. -// -really_inline uint64_t json_structural_scanner::find_potential_structurals(const simd::simd8x64 in) { - // These use SIMD so let's kick them off before running the regular 64-bit stuff ... - uint64_t whitespace, op; - find_whitespace_and_operators(in, whitespace, op); - - // Detect the start of a run of primitive characters. Includes numbers, booleans, and strings ("). - // Everything except whitespace, braces, colon and comma. - const uint64_t primitive = ~(op | whitespace); - const uint64_t follows_primitive = follows(primitive, prev_primitive); - const uint64_t start_primitive = primitive & ~follows_primitive; - - // Return final structurals - return op | start_primitive; -} - -// -// Find the important bits of JSON in a 128-byte chunk, and add them to structural_indexes. -// -// PERF NOTES: -// We pipe 2 inputs through these stages: -// 1. Load JSON into registers. This takes a long time and is highly parallelizable, so we load -// 2 inputs' worth at once so that by the time step 2 is looking for them input, it's available. -// 2. Scan the JSON for critical data: strings, primitives and operators. This is the critical path. -// The output of step 1 depends entirely on this information. These functions don't quite use -// up enough CPU: the second half of the functions is highly serial, only using 1 execution core -// at a time. The second input's scans has some dependency on the first ones finishing it, but -// they can make a lot of progress before they need that information. -// 3. Step 1 doesn't use enough capacity, so we run some extra stuff while we're waiting for that -// to finish: utf-8 checks and generating the output from the last iteration. -// -// The reason we run 2 inputs at a time, is steps 2 and 3 are *still* not enough to soak up all -// available capacity with just one input. Running 2 at a time seems to give the CPU a good enough -// workout. -// -template<> -really_inline void json_structural_scanner::scan_step<128>(const uint8_t *buf, const size_t idx, utf8_checker &utf8_checker) { - // - // Load up all 128 bytes into SIMD registers - // - simd::simd8x64 in_1(buf); - simd::simd8x64 in_2(buf+64); - - // - // Find the strings and potential structurals (operators / primitives). - // - // This will include false structurals that are *inside* strings--we'll filter strings out - // before we return. - // - uint64_t string_1 = this->find_strings(in_1); - uint64_t structurals_1 = this->find_potential_structurals(in_1); - uint64_t string_2 = this->find_strings(in_2); - uint64_t structurals_2 = this->find_potential_structurals(in_2); - - // - // Do miscellaneous work while the processor is busy calculating strings and structurals. - // - // After that, weed out structurals that are inside strings and find invalid string characters. - // - uint64_t unescaped_1 = in_1.lteq(0x1F); - utf8_checker.check_next_input(in_1); - this->structural_indexes.write_indexes(idx-64, this->prev_structurals); // Output *last* iteration's structurals to the parser - this->prev_structurals = structurals_1 & ~string_1; - this->unescaped_chars_error |= unescaped_1 & string_1; - - uint64_t unescaped_2 = in_2.lteq(0x1F); - utf8_checker.check_next_input(in_2); - this->structural_indexes.write_indexes(idx, this->prev_structurals); // Output *last* iteration's structurals to the parser - this->prev_structurals = structurals_2 & ~string_2; - this->unescaped_chars_error |= unescaped_2 & string_2; -} - -// -// Find the important bits of JSON in a 64-byte chunk, and add them to structural_indexes. -// -template<> -really_inline void json_structural_scanner::scan_step<64>(const uint8_t *buf, const size_t idx, utf8_checker &utf8_checker) { - // - // Load up bytes into SIMD registers - // - simd::simd8x64 in_1(buf); - - // - // Find the strings and potential structurals (operators / primitives). - // - // This will include false structurals that are *inside* strings--we'll filter strings out - // before we return. - // - uint64_t string_1 = this->find_strings(in_1); - uint64_t structurals_1 = this->find_potential_structurals(in_1); - - // - // Do miscellaneous work while the processor is busy calculating strings and structurals. - // - // After that, weed out structurals that are inside strings and find invalid string characters. - // - uint64_t unescaped_1 = in_1.lteq(0x1F); - utf8_checker.check_next_input(in_1); - this->structural_indexes.write_indexes(idx-64, this->prev_structurals); // Output *last* iteration's structurals to ParsedJson - this->prev_structurals = structurals_1 & ~string_1; - this->unescaped_chars_error |= unescaped_1 & string_1; -} - -template -really_inline void json_structural_scanner::scan(const uint8_t *buf, const size_t len, utf8_checker &utf8_checker) { - size_t lenminusstep = len < STEP_SIZE ? 0 : len - STEP_SIZE; - size_t idx = 0; - - for (; idx < lenminusstep; idx += STEP_SIZE) { - this->scan_step(&buf[idx], idx, utf8_checker); - } - - /* If we have a final chunk of less than STEP_SIZE bytes, pad it to STEP_SIZE with - * spaces before processing it (otherwise, we risk invalidating the UTF-8 - * checks). */ - if (likely(idx < len)) { - uint8_t tmp_buf[STEP_SIZE]; - memset(tmp_buf, 0x20, STEP_SIZE); - memcpy(tmp_buf, buf + idx, len - idx); - this->scan_step(&tmp_buf[0], idx, utf8_checker); - idx += STEP_SIZE; - } - - /* finally, flatten out the remaining structurals from the last iteration */ - this->structural_indexes.write_indexes(idx-64, this->prev_structurals); -} - -// Setting the streaming parameter to true allows the find_structural_bits to tolerate unclosed strings. -// The caller should still ensure that the input is valid UTF-8. If you are processing substrings, -// you may want to call on a function like trimmed_length_safe_utf8. -template -error_code find_structural_bits(const uint8_t *buf, size_t len, document::parser &parser, bool streaming) { - if (unlikely(len > parser.capacity())) { - return CAPACITY; - } - utf8_checker utf8_checker{}; - json_structural_scanner scanner{parser.structural_indexes.get()}; - scanner.scan(buf, len, utf8_checker); - // we might tolerate an unclosed string if streaming is true - error_code error = scanner.detect_errors_on_eof(streaming); - if (unlikely(error != SUCCESS)) { - return error; - } - parser.n_structural_indexes = scanner.structural_indexes.tail - parser.structural_indexes.get(); + parser.n_structural_indexes = indexer.tail - parser.structural_indexes.get(); /* a valid JSON file cannot have zero structural indexes - we should have * found something */ if (unlikely(parser.n_structural_indexes == 0u)) { @@ -3763,48 +3825,100 @@ error_code find_structural_bits(const uint8_t *buf, size_t len, document::parser } /* make it safe to dereference one beyond this array */ parser.structural_indexes[parser.n_structural_indexes] = 0; - return utf8_checker.errors(); + return checker.errors(); +} + +template<> +really_inline void json_structural_indexer::index_step<128>(const uint8_t *block, buf_block_reader<128> &reader) noexcept { + simd::simd8x64 in_1(block); + simd::simd8x64 in_2(block+64); + json_block block_1 = scanner.next(in_1); + json_block block_2 = scanner.next(in_2); + this->next(in_1, block_1, reader.block_index()); + this->next(in_2, block_2, reader.block_index()+64); + reader.advance(); +} + +template<> +really_inline void json_structural_indexer::index_step<64>(const uint8_t *block, buf_block_reader<64> &reader) noexcept { + simd::simd8x64 in_1(block); + json_block block_1 = scanner.next(in_1); + this->next(in_1, block_1, reader.block_index()); + reader.advance(); +} + +// +// Find the important bits of JSON in a 128-byte chunk, and add them to structural_indexes. +// +// PERF NOTES: +// We pipe 2 inputs through these stages: +// 1. Load JSON into registers. This takes a long time and is highly parallelizable, so we load +// 2 inputs' worth at once so that by the time step 2 is looking for them input, it's available. +// 2. Scan the JSON for critical data: strings, scalars and operators. This is the critical path. +// The output of step 1 depends entirely on this information. These functions don't quite use +// up enough CPU: the second half of the functions is highly serial, only using 1 execution core +// at a time. The second input's scans has some dependency on the first ones finishing it, but +// they can make a lot of progress before they need that information. +// 3. Step 1 doesn't use enough capacity, so we run some extra stuff while we're waiting for that +// to finish: utf-8 checks and generating the output from the last iteration. +// +// The reason we run 2 inputs at a time, is steps 2 and 3 are *still* not enough to soak up all +// available capacity with just one input. Running 2 at a time seems to give the CPU a good enough +// workout. +// +// Setting the streaming parameter to true allows the find_structural_bits to tolerate unclosed strings. +// The caller should still ensure that the input is valid UTF-8. If you are processing substrings, +// you may want to call on a function like trimmed_length_safe_utf8. +template +error_code json_structural_indexer::index(const uint8_t *buf, size_t len, document::parser &parser, bool streaming) noexcept { + if (unlikely(len > parser.capacity())) { return CAPACITY; } + + buf_block_reader reader(buf, len); + json_structural_indexer indexer(parser.structural_indexes.get()); + while (reader.has_full_block()) { + indexer.index_step(reader.full_block(), reader); + } + + if (likely(reader.has_remainder())) { + uint8_t block[STEP_SIZE]; + reader.get_remainder(block); + indexer.index_step(block, reader); + } + + return indexer.finish(parser, reader.block_index(), len, streaming); } } // namespace stage1 -/* end file src/generic/stage1_find_marks.h */ +/* end file src/generic/json_structural_indexer.h */ WARN_UNUSED error_code implementation::stage1(const uint8_t *buf, size_t len, document::parser &parser, bool streaming) const noexcept { - return haswell::stage1::find_structural_bits<128>(buf, len, parser, streaming); + return haswell::stage1::json_structural_indexer::index<128>(buf, len, parser, streaming); } } // namespace simdjson::haswell UNTARGET_REGION -#endif // IS_X86_64 - #endif // SIMDJSON_HASWELL_STAGE1_FIND_MARKS_H -/* end file src/generic/stage1_find_marks.h */ +/* end file src/generic/json_structural_indexer.h */ +#endif +#if SIMDJSON_IMPLEMENTATION_WESTMERE /* begin file src/westmere/stage1_find_marks.h */ #ifndef SIMDJSON_WESTMERE_STAGE1_FIND_MARKS_H #define SIMDJSON_WESTMERE_STAGE1_FIND_MARKS_H - -#ifdef IS_X86_64 - /* begin file src/westmere/bitmask.h */ #ifndef SIMDJSON_WESTMERE_BITMASK_H #define SIMDJSON_WESTMERE_BITMASK_H - -#ifdef IS_X86_64 - /* begin file src/westmere/intrinsics.h */ #ifndef SIMDJSON_WESTMERE_INTRINSICS_H #define SIMDJSON_WESTMERE_INTRINSICS_H -#ifdef IS_X86_64 #ifdef _MSC_VER #include // visual studio #else #include // elsewhere #endif // _MSC_VER -#endif // IS_X86_64 #endif // SIMDJSON_WESTMERE_INTRINSICS_H /* end file src/westmere/intrinsics.h */ @@ -3828,17 +3942,12 @@ really_inline uint64_t prefix_xor(const uint64_t bitmask) { } // namespace simdjson::westmere UNTARGET_REGION -#endif // IS_X86_64 - #endif // SIMDJSON_WESTMERE_BITMASK_H /* end file src/westmere/intrinsics.h */ /* begin file src/westmere/simd.h */ #ifndef SIMDJSON_WESTMERE_SIMD_H #define SIMDJSON_WESTMERE_SIMD_H - -#ifdef IS_X86_64 - /* westmere/intrinsics.h already included: #include "westmere/intrinsics.h" */ TARGET_WESTMERE @@ -4145,16 +4254,12 @@ namespace simdjson::westmere::simd { } // namespace simdjson::westmere::simd UNTARGET_REGION -#endif // IS_X86_64 #endif // SIMDJSON_WESTMERE_SIMD_INPUT_H /* end file src/westmere/simd.h */ /* begin file src/westmere/bitmanipulation.h */ #ifndef SIMDJSON_WESTMERE_BITMANIPULATION_H #define SIMDJSON_WESTMERE_BITMANIPULATION_H - -#ifdef IS_X86_64 - /* westmere/intrinsics.h already included: #include "westmere/intrinsics.h" */ TARGET_WESTMERE @@ -4234,11 +4339,9 @@ really_inline bool mul_overflow(uint64_t value1, uint64_t value2, #endif } -}// namespace simdjson::westmere +} // namespace simdjson::westmere UNTARGET_REGION -#endif // IS_X86_64 - #endif // SIMDJSON_WESTMERE_BITMANIPULATION_H /* end file src/westmere/bitmanipulation.h */ /* westmere/implementation.h already included: #include "westmere/implementation.h" */ @@ -4248,23 +4351,32 @@ namespace simdjson::westmere { using namespace simd; -really_inline void find_whitespace_and_operators( - const simd8x64 in, - uint64_t &whitespace, uint64_t &op) { +struct json_character_block { + static really_inline json_character_block classify(const simd::simd8x64 in); + really_inline uint64_t whitespace() const { return _whitespace; } + really_inline uint64_t op() const { return _op; } + really_inline uint64_t scalar() { return ~(op() | whitespace()); } + + uint64_t _whitespace; + uint64_t _op; +}; + +really_inline json_character_block json_character_block::classify(const simd::simd8x64 in) { // These lookups rely on the fact that anything < 127 will match the lower 4 bits, which is why // we can't use the generic lookup_16. auto whitespace_table = simd8::repeat_16(' ', 100, 100, 100, 17, 100, 113, 2, 100, '\t', '\n', 112, 100, '\r', 100, 100); auto op_table = simd8::repeat_16(',', '}', 0, 0, 0xc0u, 0, 0, 0, 0, 0, 0, 0, 0, 0, ':', '{'); - whitespace = in.map([&](simd8 _in) { + uint64_t whitespace = in.map([&](simd8 _in) { return _in == simd8(_mm_shuffle_epi8(whitespace_table, _in)); }).to_bitmask(); - op = in.map([&](simd8 _in) { + uint64_t op = in.map([&](simd8 _in) { // | 32 handles the fact that { } and [ ] are exactly 32 bytes apart return (_in | 32) == simd8(_mm_shuffle_epi8(op_table, _in-',')); }).to_bitmask(); + return { whitespace, op }; } really_inline bool is_ascii(simd8x64 input) { @@ -4289,7 +4401,7 @@ really_inline simd8 must_be_continuation(simd8 prev1, simd8 in); + really_inline error_code finish(bool streaming); + +private: + really_inline uint64_t find_escaped(uint64_t escape); + + // Whether the last iteration was still inside a string (all 1's = true, all 0's = false). + uint64_t prev_in_string = 0ULL; + // Whether the first character of the next iteration is escaped. + uint64_t prev_escaped = 0ULL; +}; + +// +// Finds escaped characters (characters following \). +// +// Handles runs of backslashes like \\\" and \\\\" correctly (yielding 0101 and 01010, respectively). +// +// Does this by: +// - Shift the escape mask to get potentially escaped characters (characters after backslashes). +// - Mask escaped sequences that start on *even* bits with 1010101010 (odd bits are escaped, even bits are not) +// - Mask escaped sequences that start on *odd* bits with 0101010101 (even bits are escaped, odd bits are not) +// +// To distinguish between escaped sequences starting on even/odd bits, it finds the start of all +// escape sequences, filters out the ones that start on even bits, and adds that to the mask of +// escape sequences. This causes the addition to clear out the sequences starting on odd bits (since +// the start bit causes a carry), and leaves even-bit sequences alone. +// +// Example: +// +// text | \\\ | \\\"\\\" \\\" \\"\\" | +// escape | xxx | xx xxx xxx xx xx | Removed overflow backslash; will | it into follows_escape +// odd_starts | x | x x x | escape & ~even_bits & ~follows_escape +// even_seq | c| cxxx c xx c | c = carry bit -- will be masked out later +// invert_mask | | cxxx c xx c| even_seq << 1 +// follows_escape | xx | x xx xxx xxx xx xx | Includes overflow bit +// escaped | x | x x x x x x x x | +// desired | x | x x x x x x x x | +// text | \\\ | \\\"\\\" \\\" \\"\\" | +// +really_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { + // If there was overflow, pretend the first character isn't a backslash + backslash &= ~prev_escaped; + uint64_t follows_escape = backslash << 1 | prev_escaped; + + // Get sequences starting on even bits by clearing out the odd series using + + const uint64_t even_bits = 0x5555555555555555ULL; + uint64_t odd_sequence_starts = backslash & ~even_bits & ~follows_escape; + uint64_t sequences_starting_on_even_bits; + prev_escaped = add_overflow(odd_sequence_starts, backslash, &sequences_starting_on_even_bits); + uint64_t invert_mask = sequences_starting_on_even_bits << 1; // The mask we want to return is the *escaped* bits, not escapes. + + // Mask every other backslashed character as an escaped character + // Flip the mask for sequences that start on even bits, to correct them + return (even_bits ^ invert_mask) & follows_escape; +} + +// +// Return a mask of all string characters plus end quotes. +// +// prev_escaped is overflow saying whether the next character is escaped. +// prev_in_string is overflow saying whether we're still in a string. +// +// Backslash sequences outside of quotes will be detected in stage 2. +// +really_inline json_string_block json_string_scanner::next(const simd::simd8x64 in) { + const uint64_t backslash = in.eq('\\'); + const uint64_t escaped = find_escaped(backslash); + const uint64_t quote = in.eq('"') & ~escaped; + // prefix_xor flips on bits inside the string (and flips off the end quote). + // Then we xor with prev_in_string: if we were in a string already, its effect is flipped + // (characters inside strings are outside, and characters outside strings are inside). + const uint64_t in_string = prefix_xor(quote) ^ prev_in_string; + // right shift of a signed value expected to be well-defined and standard + // compliant as of C++20, John Regher from Utah U. says this is fine code + prev_in_string = static_cast(static_cast(in_string) >> 63); + // Use ^ to turn the beginning quote off, and the end quote on. + return { + backslash, + escaped, + quote, + in_string + }; +} + +really_inline error_code json_string_scanner::finish(bool streaming) { + if (prev_in_string and (not streaming)) { + return UNCLOSED_STRING; + } + return SUCCESS; +} + +} // namespace stage1 +/* end file src/generic/json_string_scanner.h */ +/* begin file src/generic/json_scanner.h */ +namespace stage1 { + +/** + * A block of scanned json, with information on operators and scalars. + */ +struct json_block { +public: + // the start of structurals that are not inside strings + really_inline uint64_t structural_start() { return potential_structural_start() & ~_string.string_tail(); } + + // operators plus scalar starts like 123, true and "abc" + really_inline uint64_t potential_structural_start() { return _characters.op() | potential_scalar_start(); } + // the start of non-operator runs, like 123, true and "abc" + really_inline uint64_t potential_scalar_start() { return _characters.scalar() & ~follows_potential_scalar(); } + // whether the given character is immediately after a non-operator like 123, true or " + really_inline uint64_t follows_potential_scalar() { return _follows_potential_scalar; } + // Return a mask of whether the given characters are inside a string (only works on non-quotes) + really_inline uint64_t non_quote_inside_string(uint64_t mask) { return _string.non_quote_inside_string(mask); } + // string and escape characters + json_string_block _string; + // whitespace, operators, scalars + json_character_block _characters; + // whether the previous character was a scalar + uint64_t _follows_potential_scalar; +}; + +/** + * Scans JSON for important bits: operators, strings, and scalars. + * + * The scanner starts by calculating two distinct things: + * - string characters (taking \" into account) + * - operators ([]{},:) and scalars (runs of non-operators like 123, true and "abc") + * + * To minimize data dependency (a key component of the scanner's speed), it finds these in parallel: + * in particular, the operator/scalar bit will find plenty of things that are actually part of + * strings. When we're done, json_block will fuse the two together by masking out tokens that are + * part of a string. + */ +class json_scanner { +public: + really_inline json_block next(const simd::simd8x64 in); + really_inline error_code finish(bool streaming); + +private: + // Whether the last character of the previous iteration is part of a scalar token + // (anything except whitespace or an operator). + uint64_t prev_scalar = 0ULL; + json_string_scanner string_scanner; +}; + + +// +// Check if the current character immediately follows a matching character. +// +// For example, this checks for quotes with backslashes in front of them: +// +// const uint64_t backslashed_quote = in.eq('"') & immediately_follows(in.eq('\'), prev_backslash); +// +really_inline uint64_t follows(const uint64_t match, uint64_t &overflow) { + const uint64_t result = match << 1 | overflow; + overflow = match >> 63; + return result; +} + +// +// Check if the current character follows a matching character, with possible "filler" between. +// For example, this checks for empty curly braces, e.g. +// +// in.eq('}') & follows(in.eq('['), in.eq(' '), prev_empty_array) // { * } +// +really_inline uint64_t follows(const uint64_t match, const uint64_t filler, uint64_t &overflow) { + uint64_t follows_match = follows(match, overflow); + uint64_t result; + overflow |= uint64_t(add_overflow(follows_match, filler, &result)); + return result; +} + +really_inline json_block json_scanner::next(const simd::simd8x64 in) { + json_string_block strings = string_scanner.next(in); + json_character_block characters = json_character_block::classify(in); + uint64_t follows_scalar = follows(characters.scalar(), prev_scalar); + return { + strings, + characters, + follows_scalar + }; +} + +really_inline error_code json_scanner::finish(bool streaming) { + return string_scanner.finish(streaming); +} + +} // namespace stage1 +/* end file src/generic/json_scanner.h */ +/* begin file src/generic/json_structural_indexer.h */ // This file contains the common code every implementation uses in stage1 // It is intended to be included multiple times and compiled multiple times // We assume the file in which it is included already includes @@ -4719,14 +5053,14 @@ class bit_indexer { public: uint32_t *tail; - bit_indexer(uint32_t *index_buf) : tail(index_buf) {} + really_inline bit_indexer(uint32_t *index_buf) : tail(index_buf) {} // flatten out values in 'bits' assuming that they are are to have values of idx // plus their position in the bitvector, and store these indexes at // base_ptr[base] incrementing base as we go // will potentially store extra values beyond end of valid bits, so base_ptr // needs to be large enough to handle this - really_inline void write_indexes(uint32_t idx, uint64_t bits) { + really_inline void write(uint32_t idx, uint64_t bits) { // In some instances, the next branch is expensive because it is mispredicted. // Unfortunately, in other cases, // it helps tremendously. @@ -4765,74 +5099,6 @@ public: } }; -class json_structural_scanner { -public: - // Whether the first character of the next iteration is escaped. - uint64_t prev_escaped = 0ULL; - // Whether the last iteration was still inside a string (all 1's = true, all 0's = false). - uint64_t prev_in_string = 0ULL; - // Whether the last character of the previous iteration is a primitive value character - // (anything except whitespace, braces, comma or colon). - uint64_t prev_primitive = 0ULL; - // Mask of structural characters from the last iteration. - // Kept around for performance reasons, so we can call flatten_bits to soak up some unused - // CPU capacity while the next iteration is busy with an expensive clmul in compute_quote_mask. - uint64_t prev_structurals = 0; - // Errors with unescaped characters in strings (ASCII codepoints < 0x20) - uint64_t unescaped_chars_error = 0; - bit_indexer structural_indexes; - - json_structural_scanner(uint32_t *_structural_indexes) : structural_indexes{_structural_indexes} {} - - // - // Finish the scan and return any errors. - // - // This may detect errors as well, such as unclosed string and certain UTF-8 errors. - // if streaming is set to true, an unclosed string is allowed. - // - really_inline error_code detect_errors_on_eof(bool streaming = false); - - // - // Return a mask of all string characters plus end quotes. - // - // prev_escaped is overflow saying whether the next character is escaped. - // prev_in_string is overflow saying whether we're still in a string. - // - // Backslash sequences outside of quotes will be detected in stage 2. - // - really_inline uint64_t find_strings(const simd::simd8x64 in); - - // - // Determine which characters are *structural*: - // - braces: [] and {} - // - the start of primitives (123, true, false, null) - // - the start of invalid non-whitespace (+, &, ture, UTF-8) - // - // Also detects value sequence errors: - // - two values with no separator between ("hello" "world") - // - separators with no values ([1,] [1,,]and [,2]) - // - // This method will find all of the above whether it is in a string or not. - // - // To reduce dependency on the expensive "what is in a string" computation, this method treats the - // contents of a string the same as content outside. Errors and structurals inside the string or on - // the trailing quote will need to be removed later when the correct string information is known. - // - really_inline uint64_t find_potential_structurals(const simd::simd8x64 in); - - // - // Find the important bits of JSON in a STEP_SIZE-byte chunk, and add them to structural_indexes. - // - template - really_inline void scan_step(const uint8_t *buf, const size_t idx, utf8_checker &utf8_checker); - - // - // Parse the entire input in STEP_SIZE-byte chunks. - // - template - really_inline void scan(const uint8_t *buf, const size_t len, utf8_checker &utf8_checker); -}; - // Routines to print masks and text for debugging bitmask operations UNUSED static char * format_input_text(const simd8x64 in) { static char *buf = (char*)malloc(sizeof(simd8x64) + 1); @@ -4853,267 +5119,74 @@ UNUSED static char * format_mask(uint64_t mask) { return buf; } -// -// Finds escaped characters (characters following \). -// -// Handles runs of backslashes like \\\" and \\\\" correctly (yielding 0101 and 01010, respectively). -// -// Does this by: -// - Shift the escape mask to get potentially escaped characters (characters after backslashes). -// - Mask escaped sequences that start on *even* bits with 1010101010 (odd bits are escaped, even bits are not) -// - Mask escaped sequences that start on *odd* bits with 0101010101 (even bits are escaped, odd bits are not) -// -// To distinguish between escaped sequences starting on even/odd bits, it finds the start of all -// escape sequences, filters out the ones that start on even bits, and adds that to the mask of -// escape sequences. This causes the addition to clear out the sequences starting on odd bits (since -// the start bit causes a carry), and leaves even-bit sequences alone. -// -// Example: -// -// text | \\\ | \\\"\\\" \\\" \\"\\" | -// escape | xxx | xx xxx xxx xx xx | Removed overflow backslash; will | it into follows_escape -// odd_starts | x | x x x | escape & ~even_bits & ~follows_escape -// even_seq | c| cxxx c xx c | c = carry bit -- will be masked out later -// invert_mask | | cxxx c xx c| even_seq << 1 -// follows_escape | xx | x xx xxx xxx xx xx | Includes overflow bit -// escaped | x | x x x x x x x x | -// desired | x | x x x x x x x x | -// text | \\\ | \\\"\\\" \\\" \\"\\" | -// -really_inline uint64_t find_escaped(uint64_t escape, uint64_t &escaped_overflow) { - // If there was overflow, pretend the first character isn't a backslash - escape &= ~escaped_overflow; - uint64_t follows_escape = escape << 1 | escaped_overflow; - - // Get sequences starting on even bits by clearing out the odd series using + - const uint64_t even_bits = 0x5555555555555555ULL; - uint64_t odd_sequence_starts = escape & ~even_bits & ~follows_escape; - uint64_t sequences_starting_on_even_bits; - escaped_overflow = add_overflow(odd_sequence_starts, escape, &sequences_starting_on_even_bits); - uint64_t invert_mask = sequences_starting_on_even_bits << 1; // The mask we want to return is the *escaped* bits, not escapes. - - // Mask every other backslashed character as an escaped character - // Flip the mask for sequences that start on even bits, to correct them - return (even_bits ^ invert_mask) & follows_escape; -} - -// -// Check if the current character immediately follows a matching character. -// -// For example, this checks for quotes with backslashes in front of them: -// -// const uint64_t backslashed_quote = in.eq('"') & immediately_follows(in.eq('\'), prev_backslash); -// -really_inline uint64_t follows(const uint64_t match, uint64_t &overflow) { - const uint64_t result = match << 1 | overflow; - overflow = match >> 63; - return result; -} - -// -// Check if the current character follows a matching character, with possible "filler" between. -// For example, this checks for empty curly braces, e.g. -// -// in.eq('}') & follows(in.eq('['), in.eq(' '), prev_empty_array) // { * } -// -really_inline uint64_t follows(const uint64_t match, const uint64_t filler, uint64_t &overflow) { - uint64_t follows_match = follows(match, overflow); - uint64_t result; - overflow |= uint64_t(add_overflow(follows_match, filler, &result)); - return result; -} - -really_inline error_code json_structural_scanner::detect_errors_on_eof(bool streaming) { - if ((prev_in_string) and (not streaming)) { - return UNCLOSED_STRING; +// Walks through a buffer in block-sized increments, loading the last part with spaces +template +struct buf_block_reader { +public: + really_inline buf_block_reader(const uint8_t *_buf, size_t _len) : buf{_buf}, len{_len}, lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {} + really_inline size_t block_index() { return idx; } + really_inline bool has_full_block() const { + return idx < lenminusstep; } + really_inline const uint8_t *full_block() const { + return &buf[idx]; + } + really_inline bool has_remainder() const { + return idx < len; + } + really_inline void get_remainder(uint8_t *tmp_buf) const { + memset(tmp_buf, 0x20, STEP_SIZE); + memcpy(tmp_buf, buf + idx, len - idx); + } + really_inline void advance() { + idx += STEP_SIZE; + } +private: + const uint8_t *buf; + const size_t len; + const size_t lenminusstep; + size_t idx; +}; + +class json_structural_indexer { +public: + template + static error_code index(const uint8_t *buf, size_t len, document::parser &parser, bool streaming) noexcept; + +private: + really_inline json_structural_indexer(uint32_t *structural_indexes) : indexer{structural_indexes} {} + template + really_inline void index_step(const uint8_t *block, buf_block_reader &reader) noexcept; + really_inline void next(simd::simd8x64 in, json_block block, size_t idx); + really_inline error_code finish(document::parser &parser, size_t idx, size_t len, bool streaming); + + json_scanner scanner; + utf8_checker checker{}; + bit_indexer indexer; + uint64_t prev_structurals = 0; + uint64_t unescaped_chars_error = 0; +}; + +really_inline void json_structural_indexer::next(simd::simd8x64 in, json_block block, size_t idx) { + uint64_t unescaped = in.lteq(0x1F); + checker.check_next_input(in); + indexer.write(idx-64, prev_structurals); // Output *last* iteration's structurals to the parser + prev_structurals = block.structural_start(); + unescaped_chars_error |= block.non_quote_inside_string(unescaped); +} + +really_inline error_code json_structural_indexer::finish(document::parser &parser, size_t idx, size_t len, bool streaming) { + // Write out the final iteration's structurals + indexer.write(idx-64, prev_structurals); + + error_code error = scanner.finish(streaming); + if (unlikely(error != SUCCESS)) { return error; } + if (unescaped_chars_error) { return UNESCAPED_CHARS; } - return SUCCESS; -} -// -// Return a mask of all string characters plus end quotes. -// -// prev_escaped is overflow saying whether the next character is escaped. -// prev_in_string is overflow saying whether we're still in a string. -// -// Backslash sequences outside of quotes will be detected in stage 2. -// -really_inline uint64_t json_structural_scanner::find_strings(const simd::simd8x64 in) { - const uint64_t backslash = in.eq('\\'); - const uint64_t escaped = find_escaped(backslash, prev_escaped); - const uint64_t quote = in.eq('"') & ~escaped; - // prefix_xor flips on bits inside the string (and flips off the end quote). - const uint64_t in_string = prefix_xor(quote) ^ prev_in_string; - /* right shift of a signed value expected to be well-defined and standard - * compliant as of C++20, - * John Regher from Utah U. says this is fine code */ - prev_in_string = static_cast(static_cast(in_string) >> 63); - // Use ^ to turn the beginning quote off, and the end quote on. - return in_string ^ quote; -} - -// -// Determine which characters are *structural*: -// - braces: [] and {} -// - the start of primitives (123, true, false, null) -// - the start of invalid non-whitespace (+, &, ture, UTF-8) -// -// Also detects value sequence errors: -// - two values with no separator between ("hello" "world") -// - separators with no values ([1,] [1,,]and [,2]) -// -// This method will find all of the above whether it is in a string or not. -// -// To reduce dependency on the expensive "what is in a string" computation, this method treats the -// contents of a string the same as content outside. Errors and structurals inside the string or on -// the trailing quote will need to be removed later when the correct string information is known. -// -really_inline uint64_t json_structural_scanner::find_potential_structurals(const simd::simd8x64 in) { - // These use SIMD so let's kick them off before running the regular 64-bit stuff ... - uint64_t whitespace, op; - find_whitespace_and_operators(in, whitespace, op); - - // Detect the start of a run of primitive characters. Includes numbers, booleans, and strings ("). - // Everything except whitespace, braces, colon and comma. - const uint64_t primitive = ~(op | whitespace); - const uint64_t follows_primitive = follows(primitive, prev_primitive); - const uint64_t start_primitive = primitive & ~follows_primitive; - - // Return final structurals - return op | start_primitive; -} - -// -// Find the important bits of JSON in a 128-byte chunk, and add them to structural_indexes. -// -// PERF NOTES: -// We pipe 2 inputs through these stages: -// 1. Load JSON into registers. This takes a long time and is highly parallelizable, so we load -// 2 inputs' worth at once so that by the time step 2 is looking for them input, it's available. -// 2. Scan the JSON for critical data: strings, primitives and operators. This is the critical path. -// The output of step 1 depends entirely on this information. These functions don't quite use -// up enough CPU: the second half of the functions is highly serial, only using 1 execution core -// at a time. The second input's scans has some dependency on the first ones finishing it, but -// they can make a lot of progress before they need that information. -// 3. Step 1 doesn't use enough capacity, so we run some extra stuff while we're waiting for that -// to finish: utf-8 checks and generating the output from the last iteration. -// -// The reason we run 2 inputs at a time, is steps 2 and 3 are *still* not enough to soak up all -// available capacity with just one input. Running 2 at a time seems to give the CPU a good enough -// workout. -// -template<> -really_inline void json_structural_scanner::scan_step<128>(const uint8_t *buf, const size_t idx, utf8_checker &utf8_checker) { - // - // Load up all 128 bytes into SIMD registers - // - simd::simd8x64 in_1(buf); - simd::simd8x64 in_2(buf+64); - - // - // Find the strings and potential structurals (operators / primitives). - // - // This will include false structurals that are *inside* strings--we'll filter strings out - // before we return. - // - uint64_t string_1 = this->find_strings(in_1); - uint64_t structurals_1 = this->find_potential_structurals(in_1); - uint64_t string_2 = this->find_strings(in_2); - uint64_t structurals_2 = this->find_potential_structurals(in_2); - - // - // Do miscellaneous work while the processor is busy calculating strings and structurals. - // - // After that, weed out structurals that are inside strings and find invalid string characters. - // - uint64_t unescaped_1 = in_1.lteq(0x1F); - utf8_checker.check_next_input(in_1); - this->structural_indexes.write_indexes(idx-64, this->prev_structurals); // Output *last* iteration's structurals to the parser - this->prev_structurals = structurals_1 & ~string_1; - this->unescaped_chars_error |= unescaped_1 & string_1; - - uint64_t unescaped_2 = in_2.lteq(0x1F); - utf8_checker.check_next_input(in_2); - this->structural_indexes.write_indexes(idx, this->prev_structurals); // Output *last* iteration's structurals to the parser - this->prev_structurals = structurals_2 & ~string_2; - this->unescaped_chars_error |= unescaped_2 & string_2; -} - -// -// Find the important bits of JSON in a 64-byte chunk, and add them to structural_indexes. -// -template<> -really_inline void json_structural_scanner::scan_step<64>(const uint8_t *buf, const size_t idx, utf8_checker &utf8_checker) { - // - // Load up bytes into SIMD registers - // - simd::simd8x64 in_1(buf); - - // - // Find the strings and potential structurals (operators / primitives). - // - // This will include false structurals that are *inside* strings--we'll filter strings out - // before we return. - // - uint64_t string_1 = this->find_strings(in_1); - uint64_t structurals_1 = this->find_potential_structurals(in_1); - - // - // Do miscellaneous work while the processor is busy calculating strings and structurals. - // - // After that, weed out structurals that are inside strings and find invalid string characters. - // - uint64_t unescaped_1 = in_1.lteq(0x1F); - utf8_checker.check_next_input(in_1); - this->structural_indexes.write_indexes(idx-64, this->prev_structurals); // Output *last* iteration's structurals to ParsedJson - this->prev_structurals = structurals_1 & ~string_1; - this->unescaped_chars_error |= unescaped_1 & string_1; -} - -template -really_inline void json_structural_scanner::scan(const uint8_t *buf, const size_t len, utf8_checker &utf8_checker) { - size_t lenminusstep = len < STEP_SIZE ? 0 : len - STEP_SIZE; - size_t idx = 0; - - for (; idx < lenminusstep; idx += STEP_SIZE) { - this->scan_step(&buf[idx], idx, utf8_checker); - } - - /* If we have a final chunk of less than STEP_SIZE bytes, pad it to STEP_SIZE with - * spaces before processing it (otherwise, we risk invalidating the UTF-8 - * checks). */ - if (likely(idx < len)) { - uint8_t tmp_buf[STEP_SIZE]; - memset(tmp_buf, 0x20, STEP_SIZE); - memcpy(tmp_buf, buf + idx, len - idx); - this->scan_step(&tmp_buf[0], idx, utf8_checker); - idx += STEP_SIZE; - } - - /* finally, flatten out the remaining structurals from the last iteration */ - this->structural_indexes.write_indexes(idx-64, this->prev_structurals); -} - -// Setting the streaming parameter to true allows the find_structural_bits to tolerate unclosed strings. -// The caller should still ensure that the input is valid UTF-8. If you are processing substrings, -// you may want to call on a function like trimmed_length_safe_utf8. -template -error_code find_structural_bits(const uint8_t *buf, size_t len, document::parser &parser, bool streaming) { - if (unlikely(len > parser.capacity())) { - return CAPACITY; - } - utf8_checker utf8_checker{}; - json_structural_scanner scanner{parser.structural_indexes.get()}; - scanner.scan(buf, len, utf8_checker); - // we might tolerate an unclosed string if streaming is true - error_code error = scanner.detect_errors_on_eof(streaming); - if (unlikely(error != SUCCESS)) { - return error; - } - parser.n_structural_indexes = scanner.structural_indexes.tail - parser.structural_indexes.get(); + parser.n_structural_indexes = indexer.tail - parser.structural_indexes.get(); /* a valid JSON file cannot have zero structural indexes - we should have * found something */ if (unlikely(parser.n_structural_indexes == 0u)) { @@ -5129,23 +5202,83 @@ error_code find_structural_bits(const uint8_t *buf, size_t len, document::parser } /* make it safe to dereference one beyond this array */ parser.structural_indexes[parser.n_structural_indexes] = 0; - return utf8_checker.errors(); + return checker.errors(); +} + +template<> +really_inline void json_structural_indexer::index_step<128>(const uint8_t *block, buf_block_reader<128> &reader) noexcept { + simd::simd8x64 in_1(block); + simd::simd8x64 in_2(block+64); + json_block block_1 = scanner.next(in_1); + json_block block_2 = scanner.next(in_2); + this->next(in_1, block_1, reader.block_index()); + this->next(in_2, block_2, reader.block_index()+64); + reader.advance(); +} + +template<> +really_inline void json_structural_indexer::index_step<64>(const uint8_t *block, buf_block_reader<64> &reader) noexcept { + simd::simd8x64 in_1(block); + json_block block_1 = scanner.next(in_1); + this->next(in_1, block_1, reader.block_index()); + reader.advance(); +} + +// +// Find the important bits of JSON in a 128-byte chunk, and add them to structural_indexes. +// +// PERF NOTES: +// We pipe 2 inputs through these stages: +// 1. Load JSON into registers. This takes a long time and is highly parallelizable, so we load +// 2 inputs' worth at once so that by the time step 2 is looking for them input, it's available. +// 2. Scan the JSON for critical data: strings, scalars and operators. This is the critical path. +// The output of step 1 depends entirely on this information. These functions don't quite use +// up enough CPU: the second half of the functions is highly serial, only using 1 execution core +// at a time. The second input's scans has some dependency on the first ones finishing it, but +// they can make a lot of progress before they need that information. +// 3. Step 1 doesn't use enough capacity, so we run some extra stuff while we're waiting for that +// to finish: utf-8 checks and generating the output from the last iteration. +// +// The reason we run 2 inputs at a time, is steps 2 and 3 are *still* not enough to soak up all +// available capacity with just one input. Running 2 at a time seems to give the CPU a good enough +// workout. +// +// Setting the streaming parameter to true allows the find_structural_bits to tolerate unclosed strings. +// The caller should still ensure that the input is valid UTF-8. If you are processing substrings, +// you may want to call on a function like trimmed_length_safe_utf8. +template +error_code json_structural_indexer::index(const uint8_t *buf, size_t len, document::parser &parser, bool streaming) noexcept { + if (unlikely(len > parser.capacity())) { return CAPACITY; } + + buf_block_reader reader(buf, len); + json_structural_indexer indexer(parser.structural_indexes.get()); + while (reader.has_full_block()) { + indexer.index_step(reader.full_block(), reader); + } + + if (likely(reader.has_remainder())) { + uint8_t block[STEP_SIZE]; + reader.get_remainder(block); + indexer.index_step(block, reader); + } + + return indexer.finish(parser, reader.block_index(), len, streaming); } } // namespace stage1 -/* end file src/generic/stage1_find_marks.h */ +/* end file src/generic/json_structural_indexer.h */ WARN_UNUSED error_code implementation::stage1(const uint8_t *buf, size_t len, document::parser &parser, bool streaming) const noexcept { - return westmere::stage1::find_structural_bits<64>(buf, len, parser, streaming); + return westmere::stage1::json_structural_indexer::index<64>(buf, len, parser, streaming); } } // namespace simdjson::westmere UNTARGET_REGION -#endif // IS_X86_64 #endif // SIMDJSON_WESTMERE_STAGE1_FIND_MARKS_H -/* end file src/generic/stage1_find_marks.h */ -/* end file src/generic/stage1_find_marks.h */ +/* end file src/generic/json_structural_indexer.h */ +#endif +/* end file src/generic/json_structural_indexer.h */ /* begin file src/stage2_build_tape.cpp */ #include #include @@ -5457,6 +5590,1014 @@ inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { // will return 0 when the code point was too large. return 0; // bad r } + + +//// +// The following code is used in number parsing. It is not +// properly "char utils" stuff, but we move it here so that +// it does not get copied multiple times in the binaries (once +// per instructin set). +/// + + +constexpr int FASTFLOAT_SMALLEST_POWER = -325; +constexpr int FASTFLOAT_LARGEST_POWER = 308; + +struct value128 { + uint64_t low; + uint64_t high; +}; + +really_inline value128 full_multiplication(uint64_t value1, uint64_t value2) { + value128 answer; +#ifdef _MSC_VER + // todo: this might fail under visual studio for ARM + answer.low = _umul128(value1, value2, &answer.high); +#else + __uint128_t r = ((__uint128_t)value1) * value2; + answer.low = r; + answer.high = r >> 64; +#endif + return answer; +} + +// Precomputed powers of ten from 10^0 to 10^22. These +// can be represented exactly using the double type. +static const double power_of_ten[] = { + 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, + 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, 1e20, 1e21, 1e22}; + +// the mantissas of powers of ten from -308 to 308, extended out to sixty four +// bits +// This struct will likely get padded to 16 bytes. +typedef struct { + uint64_t mantissa; + int32_t exp; +} components; + +// The array power_of_ten_components contain the powers of ten approximated +// as a 64-bit mantissa, with an exponent part. It goes from 10^ +// FASTFLOAT_SMALLEST_POWER to +// 10^FASTFLOAT_LARGEST_POWER (inclusively). The mantissa is truncated, and +// never rounded up. +// Uses about 10KB. +static const components power_of_ten_components[] = { + {0xa5ced43b7e3e9188L, 7}, {0xcf42894a5dce35eaL, 10}, + {0x818995ce7aa0e1b2L, 14}, {0xa1ebfb4219491a1fL, 17}, + {0xca66fa129f9b60a6L, 20}, {0xfd00b897478238d0L, 23}, + {0x9e20735e8cb16382L, 27}, {0xc5a890362fddbc62L, 30}, + {0xf712b443bbd52b7bL, 33}, {0x9a6bb0aa55653b2dL, 37}, + {0xc1069cd4eabe89f8L, 40}, {0xf148440a256e2c76L, 43}, + {0x96cd2a865764dbcaL, 47}, {0xbc807527ed3e12bcL, 50}, + {0xeba09271e88d976bL, 53}, {0x93445b8731587ea3L, 57}, + {0xb8157268fdae9e4cL, 60}, {0xe61acf033d1a45dfL, 63}, + {0x8fd0c16206306babL, 67}, {0xb3c4f1ba87bc8696L, 70}, + {0xe0b62e2929aba83cL, 73}, {0x8c71dcd9ba0b4925L, 77}, + {0xaf8e5410288e1b6fL, 80}, {0xdb71e91432b1a24aL, 83}, + {0x892731ac9faf056eL, 87}, {0xab70fe17c79ac6caL, 90}, + {0xd64d3d9db981787dL, 93}, {0x85f0468293f0eb4eL, 97}, + {0xa76c582338ed2621L, 100}, {0xd1476e2c07286faaL, 103}, + {0x82cca4db847945caL, 107}, {0xa37fce126597973cL, 110}, + {0xcc5fc196fefd7d0cL, 113}, {0xff77b1fcbebcdc4fL, 116}, + {0x9faacf3df73609b1L, 120}, {0xc795830d75038c1dL, 123}, + {0xf97ae3d0d2446f25L, 126}, {0x9becce62836ac577L, 130}, + {0xc2e801fb244576d5L, 133}, {0xf3a20279ed56d48aL, 136}, + {0x9845418c345644d6L, 140}, {0xbe5691ef416bd60cL, 143}, + {0xedec366b11c6cb8fL, 146}, {0x94b3a202eb1c3f39L, 150}, + {0xb9e08a83a5e34f07L, 153}, {0xe858ad248f5c22c9L, 156}, + {0x91376c36d99995beL, 160}, {0xb58547448ffffb2dL, 163}, + {0xe2e69915b3fff9f9L, 166}, {0x8dd01fad907ffc3bL, 170}, + {0xb1442798f49ffb4aL, 173}, {0xdd95317f31c7fa1dL, 176}, + {0x8a7d3eef7f1cfc52L, 180}, {0xad1c8eab5ee43b66L, 183}, + {0xd863b256369d4a40L, 186}, {0x873e4f75e2224e68L, 190}, + {0xa90de3535aaae202L, 193}, {0xd3515c2831559a83L, 196}, + {0x8412d9991ed58091L, 200}, {0xa5178fff668ae0b6L, 203}, + {0xce5d73ff402d98e3L, 206}, {0x80fa687f881c7f8eL, 210}, + {0xa139029f6a239f72L, 213}, {0xc987434744ac874eL, 216}, + {0xfbe9141915d7a922L, 219}, {0x9d71ac8fada6c9b5L, 223}, + {0xc4ce17b399107c22L, 226}, {0xf6019da07f549b2bL, 229}, + {0x99c102844f94e0fbL, 233}, {0xc0314325637a1939L, 236}, + {0xf03d93eebc589f88L, 239}, {0x96267c7535b763b5L, 243}, + {0xbbb01b9283253ca2L, 246}, {0xea9c227723ee8bcbL, 249}, + {0x92a1958a7675175fL, 253}, {0xb749faed14125d36L, 256}, + {0xe51c79a85916f484L, 259}, {0x8f31cc0937ae58d2L, 263}, + {0xb2fe3f0b8599ef07L, 266}, {0xdfbdcece67006ac9L, 269}, + {0x8bd6a141006042bdL, 273}, {0xaecc49914078536dL, 276}, + {0xda7f5bf590966848L, 279}, {0x888f99797a5e012dL, 283}, + {0xaab37fd7d8f58178L, 286}, {0xd5605fcdcf32e1d6L, 289}, + {0x855c3be0a17fcd26L, 293}, {0xa6b34ad8c9dfc06fL, 296}, + {0xd0601d8efc57b08bL, 299}, {0x823c12795db6ce57L, 303}, + {0xa2cb1717b52481edL, 306}, {0xcb7ddcdda26da268L, 309}, + {0xfe5d54150b090b02L, 312}, {0x9efa548d26e5a6e1L, 316}, + {0xc6b8e9b0709f109aL, 319}, {0xf867241c8cc6d4c0L, 322}, + {0x9b407691d7fc44f8L, 326}, {0xc21094364dfb5636L, 329}, + {0xf294b943e17a2bc4L, 332}, {0x979cf3ca6cec5b5aL, 336}, + {0xbd8430bd08277231L, 339}, {0xece53cec4a314ebdL, 342}, + {0x940f4613ae5ed136L, 346}, {0xb913179899f68584L, 349}, + {0xe757dd7ec07426e5L, 352}, {0x9096ea6f3848984fL, 356}, + {0xb4bca50b065abe63L, 359}, {0xe1ebce4dc7f16dfbL, 362}, + {0x8d3360f09cf6e4bdL, 366}, {0xb080392cc4349decL, 369}, + {0xdca04777f541c567L, 372}, {0x89e42caaf9491b60L, 376}, + {0xac5d37d5b79b6239L, 379}, {0xd77485cb25823ac7L, 382}, + {0x86a8d39ef77164bcL, 386}, {0xa8530886b54dbdebL, 389}, + {0xd267caa862a12d66L, 392}, {0x8380dea93da4bc60L, 396}, + {0xa46116538d0deb78L, 399}, {0xcd795be870516656L, 402}, + {0x806bd9714632dff6L, 406}, {0xa086cfcd97bf97f3L, 409}, + {0xc8a883c0fdaf7df0L, 412}, {0xfad2a4b13d1b5d6cL, 415}, + {0x9cc3a6eec6311a63L, 419}, {0xc3f490aa77bd60fcL, 422}, + {0xf4f1b4d515acb93bL, 425}, {0x991711052d8bf3c5L, 429}, + {0xbf5cd54678eef0b6L, 432}, {0xef340a98172aace4L, 435}, + {0x9580869f0e7aac0eL, 439}, {0xbae0a846d2195712L, 442}, + {0xe998d258869facd7L, 445}, {0x91ff83775423cc06L, 449}, + {0xb67f6455292cbf08L, 452}, {0xe41f3d6a7377eecaL, 455}, + {0x8e938662882af53eL, 459}, {0xb23867fb2a35b28dL, 462}, + {0xdec681f9f4c31f31L, 465}, {0x8b3c113c38f9f37eL, 469}, + {0xae0b158b4738705eL, 472}, {0xd98ddaee19068c76L, 475}, + {0x87f8a8d4cfa417c9L, 479}, {0xa9f6d30a038d1dbcL, 482}, + {0xd47487cc8470652bL, 485}, {0x84c8d4dfd2c63f3bL, 489}, + {0xa5fb0a17c777cf09L, 492}, {0xcf79cc9db955c2ccL, 495}, + {0x81ac1fe293d599bfL, 499}, {0xa21727db38cb002fL, 502}, + {0xca9cf1d206fdc03bL, 505}, {0xfd442e4688bd304aL, 508}, + {0x9e4a9cec15763e2eL, 512}, {0xc5dd44271ad3cdbaL, 515}, + {0xf7549530e188c128L, 518}, {0x9a94dd3e8cf578b9L, 522}, + {0xc13a148e3032d6e7L, 525}, {0xf18899b1bc3f8ca1L, 528}, + {0x96f5600f15a7b7e5L, 532}, {0xbcb2b812db11a5deL, 535}, + {0xebdf661791d60f56L, 538}, {0x936b9fcebb25c995L, 542}, + {0xb84687c269ef3bfbL, 545}, {0xe65829b3046b0afaL, 548}, + {0x8ff71a0fe2c2e6dcL, 552}, {0xb3f4e093db73a093L, 555}, + {0xe0f218b8d25088b8L, 558}, {0x8c974f7383725573L, 562}, + {0xafbd2350644eeacfL, 565}, {0xdbac6c247d62a583L, 568}, + {0x894bc396ce5da772L, 572}, {0xab9eb47c81f5114fL, 575}, + {0xd686619ba27255a2L, 578}, {0x8613fd0145877585L, 582}, + {0xa798fc4196e952e7L, 585}, {0xd17f3b51fca3a7a0L, 588}, + {0x82ef85133de648c4L, 592}, {0xa3ab66580d5fdaf5L, 595}, + {0xcc963fee10b7d1b3L, 598}, {0xffbbcfe994e5c61fL, 601}, + {0x9fd561f1fd0f9bd3L, 605}, {0xc7caba6e7c5382c8L, 608}, + {0xf9bd690a1b68637bL, 611}, {0x9c1661a651213e2dL, 615}, + {0xc31bfa0fe5698db8L, 618}, {0xf3e2f893dec3f126L, 621}, + {0x986ddb5c6b3a76b7L, 625}, {0xbe89523386091465L, 628}, + {0xee2ba6c0678b597fL, 631}, {0x94db483840b717efL, 635}, + {0xba121a4650e4ddebL, 638}, {0xe896a0d7e51e1566L, 641}, + {0x915e2486ef32cd60L, 645}, {0xb5b5ada8aaff80b8L, 648}, + {0xe3231912d5bf60e6L, 651}, {0x8df5efabc5979c8fL, 655}, + {0xb1736b96b6fd83b3L, 658}, {0xddd0467c64bce4a0L, 661}, + {0x8aa22c0dbef60ee4L, 665}, {0xad4ab7112eb3929dL, 668}, + {0xd89d64d57a607744L, 671}, {0x87625f056c7c4a8bL, 675}, + {0xa93af6c6c79b5d2dL, 678}, {0xd389b47879823479L, 681}, + {0x843610cb4bf160cbL, 685}, {0xa54394fe1eedb8feL, 688}, + {0xce947a3da6a9273eL, 691}, {0x811ccc668829b887L, 695}, + {0xa163ff802a3426a8L, 698}, {0xc9bcff6034c13052L, 701}, + {0xfc2c3f3841f17c67L, 704}, {0x9d9ba7832936edc0L, 708}, + {0xc5029163f384a931L, 711}, {0xf64335bcf065d37dL, 714}, + {0x99ea0196163fa42eL, 718}, {0xc06481fb9bcf8d39L, 721}, + {0xf07da27a82c37088L, 724}, {0x964e858c91ba2655L, 728}, + {0xbbe226efb628afeaL, 731}, {0xeadab0aba3b2dbe5L, 734}, + {0x92c8ae6b464fc96fL, 738}, {0xb77ada0617e3bbcbL, 741}, + {0xe55990879ddcaabdL, 744}, {0x8f57fa54c2a9eab6L, 748}, + {0xb32df8e9f3546564L, 751}, {0xdff9772470297ebdL, 754}, + {0x8bfbea76c619ef36L, 758}, {0xaefae51477a06b03L, 761}, + {0xdab99e59958885c4L, 764}, {0x88b402f7fd75539bL, 768}, + {0xaae103b5fcd2a881L, 771}, {0xd59944a37c0752a2L, 774}, + {0x857fcae62d8493a5L, 778}, {0xa6dfbd9fb8e5b88eL, 781}, + {0xd097ad07a71f26b2L, 784}, {0x825ecc24c873782fL, 788}, + {0xa2f67f2dfa90563bL, 791}, {0xcbb41ef979346bcaL, 794}, + {0xfea126b7d78186bcL, 797}, {0x9f24b832e6b0f436L, 801}, + {0xc6ede63fa05d3143L, 804}, {0xf8a95fcf88747d94L, 807}, + {0x9b69dbe1b548ce7cL, 811}, {0xc24452da229b021bL, 814}, + {0xf2d56790ab41c2a2L, 817}, {0x97c560ba6b0919a5L, 821}, + {0xbdb6b8e905cb600fL, 824}, {0xed246723473e3813L, 827}, + {0x9436c0760c86e30bL, 831}, {0xb94470938fa89bceL, 834}, + {0xe7958cb87392c2c2L, 837}, {0x90bd77f3483bb9b9L, 841}, + {0xb4ecd5f01a4aa828L, 844}, {0xe2280b6c20dd5232L, 847}, + {0x8d590723948a535fL, 851}, {0xb0af48ec79ace837L, 854}, + {0xdcdb1b2798182244L, 857}, {0x8a08f0f8bf0f156bL, 861}, + {0xac8b2d36eed2dac5L, 864}, {0xd7adf884aa879177L, 867}, + {0x86ccbb52ea94baeaL, 871}, {0xa87fea27a539e9a5L, 874}, + {0xd29fe4b18e88640eL, 877}, {0x83a3eeeef9153e89L, 881}, + {0xa48ceaaab75a8e2bL, 884}, {0xcdb02555653131b6L, 887}, + {0x808e17555f3ebf11L, 891}, {0xa0b19d2ab70e6ed6L, 894}, + {0xc8de047564d20a8bL, 897}, {0xfb158592be068d2eL, 900}, + {0x9ced737bb6c4183dL, 904}, {0xc428d05aa4751e4cL, 907}, + {0xf53304714d9265dfL, 910}, {0x993fe2c6d07b7fabL, 914}, + {0xbf8fdb78849a5f96L, 917}, {0xef73d256a5c0f77cL, 920}, + {0x95a8637627989aadL, 924}, {0xbb127c53b17ec159L, 927}, + {0xe9d71b689dde71afL, 930}, {0x9226712162ab070dL, 934}, + {0xb6b00d69bb55c8d1L, 937}, {0xe45c10c42a2b3b05L, 940}, + {0x8eb98a7a9a5b04e3L, 944}, {0xb267ed1940f1c61cL, 947}, + {0xdf01e85f912e37a3L, 950}, {0x8b61313bbabce2c6L, 954}, + {0xae397d8aa96c1b77L, 957}, {0xd9c7dced53c72255L, 960}, + {0x881cea14545c7575L, 964}, {0xaa242499697392d2L, 967}, + {0xd4ad2dbfc3d07787L, 970}, {0x84ec3c97da624ab4L, 974}, + {0xa6274bbdd0fadd61L, 977}, {0xcfb11ead453994baL, 980}, + {0x81ceb32c4b43fcf4L, 984}, {0xa2425ff75e14fc31L, 987}, + {0xcad2f7f5359a3b3eL, 990}, {0xfd87b5f28300ca0dL, 993}, + {0x9e74d1b791e07e48L, 997}, {0xc612062576589ddaL, 1000}, + {0xf79687aed3eec551L, 1003}, {0x9abe14cd44753b52L, 1007}, + {0xc16d9a0095928a27L, 1010}, {0xf1c90080baf72cb1L, 1013}, + {0x971da05074da7beeL, 1017}, {0xbce5086492111aeaL, 1020}, + {0xec1e4a7db69561a5L, 1023}, {0x9392ee8e921d5d07L, 1027}, + {0xb877aa3236a4b449L, 1030}, {0xe69594bec44de15bL, 1033}, + {0x901d7cf73ab0acd9L, 1037}, {0xb424dc35095cd80fL, 1040}, + {0xe12e13424bb40e13L, 1043}, {0x8cbccc096f5088cbL, 1047}, + {0xafebff0bcb24aafeL, 1050}, {0xdbe6fecebdedd5beL, 1053}, + {0x89705f4136b4a597L, 1057}, {0xabcc77118461cefcL, 1060}, + {0xd6bf94d5e57a42bcL, 1063}, {0x8637bd05af6c69b5L, 1067}, + {0xa7c5ac471b478423L, 1070}, {0xd1b71758e219652bL, 1073}, + {0x83126e978d4fdf3bL, 1077}, {0xa3d70a3d70a3d70aL, 1080}, + {0xccccccccccccccccL, 1083}, {0x8000000000000000L, 1087}, + {0xa000000000000000L, 1090}, {0xc800000000000000L, 1093}, + {0xfa00000000000000L, 1096}, {0x9c40000000000000L, 1100}, + {0xc350000000000000L, 1103}, {0xf424000000000000L, 1106}, + {0x9896800000000000L, 1110}, {0xbebc200000000000L, 1113}, + {0xee6b280000000000L, 1116}, {0x9502f90000000000L, 1120}, + {0xba43b74000000000L, 1123}, {0xe8d4a51000000000L, 1126}, + {0x9184e72a00000000L, 1130}, {0xb5e620f480000000L, 1133}, + {0xe35fa931a0000000L, 1136}, {0x8e1bc9bf04000000L, 1140}, + {0xb1a2bc2ec5000000L, 1143}, {0xde0b6b3a76400000L, 1146}, + {0x8ac7230489e80000L, 1150}, {0xad78ebc5ac620000L, 1153}, + {0xd8d726b7177a8000L, 1156}, {0x878678326eac9000L, 1160}, + {0xa968163f0a57b400L, 1163}, {0xd3c21bcecceda100L, 1166}, + {0x84595161401484a0L, 1170}, {0xa56fa5b99019a5c8L, 1173}, + {0xcecb8f27f4200f3aL, 1176}, {0x813f3978f8940984L, 1180}, + {0xa18f07d736b90be5L, 1183}, {0xc9f2c9cd04674edeL, 1186}, + {0xfc6f7c4045812296L, 1189}, {0x9dc5ada82b70b59dL, 1193}, + {0xc5371912364ce305L, 1196}, {0xf684df56c3e01bc6L, 1199}, + {0x9a130b963a6c115cL, 1203}, {0xc097ce7bc90715b3L, 1206}, + {0xf0bdc21abb48db20L, 1209}, {0x96769950b50d88f4L, 1213}, + {0xbc143fa4e250eb31L, 1216}, {0xeb194f8e1ae525fdL, 1219}, + {0x92efd1b8d0cf37beL, 1223}, {0xb7abc627050305adL, 1226}, + {0xe596b7b0c643c719L, 1229}, {0x8f7e32ce7bea5c6fL, 1233}, + {0xb35dbf821ae4f38bL, 1236}, {0xe0352f62a19e306eL, 1239}, + {0x8c213d9da502de45L, 1243}, {0xaf298d050e4395d6L, 1246}, + {0xdaf3f04651d47b4cL, 1249}, {0x88d8762bf324cd0fL, 1253}, + {0xab0e93b6efee0053L, 1256}, {0xd5d238a4abe98068L, 1259}, + {0x85a36366eb71f041L, 1263}, {0xa70c3c40a64e6c51L, 1266}, + {0xd0cf4b50cfe20765L, 1269}, {0x82818f1281ed449fL, 1273}, + {0xa321f2d7226895c7L, 1276}, {0xcbea6f8ceb02bb39L, 1279}, + {0xfee50b7025c36a08L, 1282}, {0x9f4f2726179a2245L, 1286}, + {0xc722f0ef9d80aad6L, 1289}, {0xf8ebad2b84e0d58bL, 1292}, + {0x9b934c3b330c8577L, 1296}, {0xc2781f49ffcfa6d5L, 1299}, + {0xf316271c7fc3908aL, 1302}, {0x97edd871cfda3a56L, 1306}, + {0xbde94e8e43d0c8ecL, 1309}, {0xed63a231d4c4fb27L, 1312}, + {0x945e455f24fb1cf8L, 1316}, {0xb975d6b6ee39e436L, 1319}, + {0xe7d34c64a9c85d44L, 1322}, {0x90e40fbeea1d3a4aL, 1326}, + {0xb51d13aea4a488ddL, 1329}, {0xe264589a4dcdab14L, 1332}, + {0x8d7eb76070a08aecL, 1336}, {0xb0de65388cc8ada8L, 1339}, + {0xdd15fe86affad912L, 1342}, {0x8a2dbf142dfcc7abL, 1346}, + {0xacb92ed9397bf996L, 1349}, {0xd7e77a8f87daf7fbL, 1352}, + {0x86f0ac99b4e8dafdL, 1356}, {0xa8acd7c0222311bcL, 1359}, + {0xd2d80db02aabd62bL, 1362}, {0x83c7088e1aab65dbL, 1366}, + {0xa4b8cab1a1563f52L, 1369}, {0xcde6fd5e09abcf26L, 1372}, + {0x80b05e5ac60b6178L, 1376}, {0xa0dc75f1778e39d6L, 1379}, + {0xc913936dd571c84cL, 1382}, {0xfb5878494ace3a5fL, 1385}, + {0x9d174b2dcec0e47bL, 1389}, {0xc45d1df942711d9aL, 1392}, + {0xf5746577930d6500L, 1395}, {0x9968bf6abbe85f20L, 1399}, + {0xbfc2ef456ae276e8L, 1402}, {0xefb3ab16c59b14a2L, 1405}, + {0x95d04aee3b80ece5L, 1409}, {0xbb445da9ca61281fL, 1412}, + {0xea1575143cf97226L, 1415}, {0x924d692ca61be758L, 1419}, + {0xb6e0c377cfa2e12eL, 1422}, {0xe498f455c38b997aL, 1425}, + {0x8edf98b59a373fecL, 1429}, {0xb2977ee300c50fe7L, 1432}, + {0xdf3d5e9bc0f653e1L, 1435}, {0x8b865b215899f46cL, 1439}, + {0xae67f1e9aec07187L, 1442}, {0xda01ee641a708de9L, 1445}, + {0x884134fe908658b2L, 1449}, {0xaa51823e34a7eedeL, 1452}, + {0xd4e5e2cdc1d1ea96L, 1455}, {0x850fadc09923329eL, 1459}, + {0xa6539930bf6bff45L, 1462}, {0xcfe87f7cef46ff16L, 1465}, + {0x81f14fae158c5f6eL, 1469}, {0xa26da3999aef7749L, 1472}, + {0xcb090c8001ab551cL, 1475}, {0xfdcb4fa002162a63L, 1478}, + {0x9e9f11c4014dda7eL, 1482}, {0xc646d63501a1511dL, 1485}, + {0xf7d88bc24209a565L, 1488}, {0x9ae757596946075fL, 1492}, + {0xc1a12d2fc3978937L, 1495}, {0xf209787bb47d6b84L, 1498}, + {0x9745eb4d50ce6332L, 1502}, {0xbd176620a501fbffL, 1505}, + {0xec5d3fa8ce427affL, 1508}, {0x93ba47c980e98cdfL, 1512}, + {0xb8a8d9bbe123f017L, 1515}, {0xe6d3102ad96cec1dL, 1518}, + {0x9043ea1ac7e41392L, 1522}, {0xb454e4a179dd1877L, 1525}, + {0xe16a1dc9d8545e94L, 1528}, {0x8ce2529e2734bb1dL, 1532}, + {0xb01ae745b101e9e4L, 1535}, {0xdc21a1171d42645dL, 1538}, + {0x899504ae72497ebaL, 1542}, {0xabfa45da0edbde69L, 1545}, + {0xd6f8d7509292d603L, 1548}, {0x865b86925b9bc5c2L, 1552}, + {0xa7f26836f282b732L, 1555}, {0xd1ef0244af2364ffL, 1558}, + {0x8335616aed761f1fL, 1562}, {0xa402b9c5a8d3a6e7L, 1565}, + {0xcd036837130890a1L, 1568}, {0x802221226be55a64L, 1572}, + {0xa02aa96b06deb0fdL, 1575}, {0xc83553c5c8965d3dL, 1578}, + {0xfa42a8b73abbf48cL, 1581}, {0x9c69a97284b578d7L, 1585}, + {0xc38413cf25e2d70dL, 1588}, {0xf46518c2ef5b8cd1L, 1591}, + {0x98bf2f79d5993802L, 1595}, {0xbeeefb584aff8603L, 1598}, + {0xeeaaba2e5dbf6784L, 1601}, {0x952ab45cfa97a0b2L, 1605}, + {0xba756174393d88dfL, 1608}, {0xe912b9d1478ceb17L, 1611}, + {0x91abb422ccb812eeL, 1615}, {0xb616a12b7fe617aaL, 1618}, + {0xe39c49765fdf9d94L, 1621}, {0x8e41ade9fbebc27dL, 1625}, + {0xb1d219647ae6b31cL, 1628}, {0xde469fbd99a05fe3L, 1631}, + {0x8aec23d680043beeL, 1635}, {0xada72ccc20054ae9L, 1638}, + {0xd910f7ff28069da4L, 1641}, {0x87aa9aff79042286L, 1645}, + {0xa99541bf57452b28L, 1648}, {0xd3fa922f2d1675f2L, 1651}, + {0x847c9b5d7c2e09b7L, 1655}, {0xa59bc234db398c25L, 1658}, + {0xcf02b2c21207ef2eL, 1661}, {0x8161afb94b44f57dL, 1665}, + {0xa1ba1ba79e1632dcL, 1668}, {0xca28a291859bbf93L, 1671}, + {0xfcb2cb35e702af78L, 1674}, {0x9defbf01b061adabL, 1678}, + {0xc56baec21c7a1916L, 1681}, {0xf6c69a72a3989f5bL, 1684}, + {0x9a3c2087a63f6399L, 1688}, {0xc0cb28a98fcf3c7fL, 1691}, + {0xf0fdf2d3f3c30b9fL, 1694}, {0x969eb7c47859e743L, 1698}, + {0xbc4665b596706114L, 1701}, {0xeb57ff22fc0c7959L, 1704}, + {0x9316ff75dd87cbd8L, 1708}, {0xb7dcbf5354e9beceL, 1711}, + {0xe5d3ef282a242e81L, 1714}, {0x8fa475791a569d10L, 1718}, + {0xb38d92d760ec4455L, 1721}, {0xe070f78d3927556aL, 1724}, + {0x8c469ab843b89562L, 1728}, {0xaf58416654a6babbL, 1731}, + {0xdb2e51bfe9d0696aL, 1734}, {0x88fcf317f22241e2L, 1738}, + {0xab3c2fddeeaad25aL, 1741}, {0xd60b3bd56a5586f1L, 1744}, + {0x85c7056562757456L, 1748}, {0xa738c6bebb12d16cL, 1751}, + {0xd106f86e69d785c7L, 1754}, {0x82a45b450226b39cL, 1758}, + {0xa34d721642b06084L, 1761}, {0xcc20ce9bd35c78a5L, 1764}, + {0xff290242c83396ceL, 1767}, {0x9f79a169bd203e41L, 1771}, + {0xc75809c42c684dd1L, 1774}, {0xf92e0c3537826145L, 1777}, + {0x9bbcc7a142b17ccbL, 1781}, {0xc2abf989935ddbfeL, 1784}, + {0xf356f7ebf83552feL, 1787}, {0x98165af37b2153deL, 1791}, + {0xbe1bf1b059e9a8d6L, 1794}, {0xeda2ee1c7064130cL, 1797}, + {0x9485d4d1c63e8be7L, 1801}, {0xb9a74a0637ce2ee1L, 1804}, + {0xe8111c87c5c1ba99L, 1807}, {0x910ab1d4db9914a0L, 1811}, + {0xb54d5e4a127f59c8L, 1814}, {0xe2a0b5dc971f303aL, 1817}, + {0x8da471a9de737e24L, 1821}, {0xb10d8e1456105dadL, 1824}, + {0xdd50f1996b947518L, 1827}, {0x8a5296ffe33cc92fL, 1831}, + {0xace73cbfdc0bfb7bL, 1834}, {0xd8210befd30efa5aL, 1837}, + {0x8714a775e3e95c78L, 1841}, {0xa8d9d1535ce3b396L, 1844}, + {0xd31045a8341ca07cL, 1847}, {0x83ea2b892091e44dL, 1851}, + {0xa4e4b66b68b65d60L, 1854}, {0xce1de40642e3f4b9L, 1857}, + {0x80d2ae83e9ce78f3L, 1861}, {0xa1075a24e4421730L, 1864}, + {0xc94930ae1d529cfcL, 1867}, {0xfb9b7cd9a4a7443cL, 1870}, + {0x9d412e0806e88aa5L, 1874}, {0xc491798a08a2ad4eL, 1877}, + {0xf5b5d7ec8acb58a2L, 1880}, {0x9991a6f3d6bf1765L, 1884}, + {0xbff610b0cc6edd3fL, 1887}, {0xeff394dcff8a948eL, 1890}, + {0x95f83d0a1fb69cd9L, 1894}, {0xbb764c4ca7a4440fL, 1897}, + {0xea53df5fd18d5513L, 1900}, {0x92746b9be2f8552cL, 1904}, + {0xb7118682dbb66a77L, 1907}, {0xe4d5e82392a40515L, 1910}, + {0x8f05b1163ba6832dL, 1914}, {0xb2c71d5bca9023f8L, 1917}, + {0xdf78e4b2bd342cf6L, 1920}, {0x8bab8eefb6409c1aL, 1924}, + {0xae9672aba3d0c320L, 1927}, {0xda3c0f568cc4f3e8L, 1930}, + {0x8865899617fb1871L, 1934}, {0xaa7eebfb9df9de8dL, 1937}, + {0xd51ea6fa85785631L, 1940}, {0x8533285c936b35deL, 1944}, + {0xa67ff273b8460356L, 1947}, {0xd01fef10a657842cL, 1950}, + {0x8213f56a67f6b29bL, 1954}, {0xa298f2c501f45f42L, 1957}, + {0xcb3f2f7642717713L, 1960}, {0xfe0efb53d30dd4d7L, 1963}, + {0x9ec95d1463e8a506L, 1967}, {0xc67bb4597ce2ce48L, 1970}, + {0xf81aa16fdc1b81daL, 1973}, {0x9b10a4e5e9913128L, 1977}, + {0xc1d4ce1f63f57d72L, 1980}, {0xf24a01a73cf2dccfL, 1983}, + {0x976e41088617ca01L, 1987}, {0xbd49d14aa79dbc82L, 1990}, + {0xec9c459d51852ba2L, 1993}, {0x93e1ab8252f33b45L, 1997}, + {0xb8da1662e7b00a17L, 2000}, {0xe7109bfba19c0c9dL, 2003}, + {0x906a617d450187e2L, 2007}, {0xb484f9dc9641e9daL, 2010}, + {0xe1a63853bbd26451L, 2013}, {0x8d07e33455637eb2L, 2017}, + {0xb049dc016abc5e5fL, 2020}, {0xdc5c5301c56b75f7L, 2023}, + {0x89b9b3e11b6329baL, 2027}, {0xac2820d9623bf429L, 2030}, + {0xd732290fbacaf133L, 2033}, {0x867f59a9d4bed6c0L, 2037}, + {0xa81f301449ee8c70L, 2040}, {0xd226fc195c6a2f8cL, 2043}, + {0x83585d8fd9c25db7L, 2047}, {0xa42e74f3d032f525L, 2050}, + {0xcd3a1230c43fb26fL, 2053}, {0x80444b5e7aa7cf85L, 2057}, + {0xa0555e361951c366L, 2060}, {0xc86ab5c39fa63440L, 2063}, + {0xfa856334878fc150L, 2066}, {0x9c935e00d4b9d8d2L, 2070}, + {0xc3b8358109e84f07L, 2073}, {0xf4a642e14c6262c8L, 2076}, + {0x98e7e9cccfbd7dbdL, 2080}, {0xbf21e44003acdd2cL, 2083}, + {0xeeea5d5004981478L, 2086}, {0x95527a5202df0ccbL, 2090}, + {0xbaa718e68396cffdL, 2093}, {0xe950df20247c83fdL, 2096}, + {0x91d28b7416cdd27eL, 2100}, {0xb6472e511c81471dL, 2103}, + {0xe3d8f9e563a198e5L, 2106}, {0x8e679c2f5e44ff8fL, 2110}}; + +// A complement from power_of_ten_components +// complete to a 128-bit mantissa. +const uint64_t mantissa_128[] = {0x419ea3bd35385e2d, + 0x52064cac828675b9, + 0x7343efebd1940993, + 0x1014ebe6c5f90bf8, + 0xd41a26e077774ef6, + 0x8920b098955522b4, + 0x55b46e5f5d5535b0, + 0xeb2189f734aa831d, + 0xa5e9ec7501d523e4, + 0x47b233c92125366e, + 0x999ec0bb696e840a, + 0xc00670ea43ca250d, + 0x380406926a5e5728, + 0xc605083704f5ecf2, + 0xf7864a44c633682e, + 0x7ab3ee6afbe0211d, + 0x5960ea05bad82964, + 0x6fb92487298e33bd, + 0xa5d3b6d479f8e056, + 0x8f48a4899877186c, + 0x331acdabfe94de87, + 0x9ff0c08b7f1d0b14, + 0x7ecf0ae5ee44dd9, + 0xc9e82cd9f69d6150, + 0xbe311c083a225cd2, + 0x6dbd630a48aaf406, + 0x92cbbccdad5b108, + 0x25bbf56008c58ea5, + 0xaf2af2b80af6f24e, + 0x1af5af660db4aee1, + 0x50d98d9fc890ed4d, + 0xe50ff107bab528a0, + 0x1e53ed49a96272c8, + 0x25e8e89c13bb0f7a, + 0x77b191618c54e9ac, + 0xd59df5b9ef6a2417, + 0x4b0573286b44ad1d, + 0x4ee367f9430aec32, + 0x229c41f793cda73f, + 0x6b43527578c1110f, + 0x830a13896b78aaa9, + 0x23cc986bc656d553, + 0x2cbfbe86b7ec8aa8, + 0x7bf7d71432f3d6a9, + 0xdaf5ccd93fb0cc53, + 0xd1b3400f8f9cff68, + 0x23100809b9c21fa1, + 0xabd40a0c2832a78a, + 0x16c90c8f323f516c, + 0xae3da7d97f6792e3, + 0x99cd11cfdf41779c, + 0x40405643d711d583, + 0x482835ea666b2572, + 0xda3243650005eecf, + 0x90bed43e40076a82, + 0x5a7744a6e804a291, + 0x711515d0a205cb36, + 0xd5a5b44ca873e03, + 0xe858790afe9486c2, + 0x626e974dbe39a872, + 0xfb0a3d212dc8128f, + 0x7ce66634bc9d0b99, + 0x1c1fffc1ebc44e80, + 0xa327ffb266b56220, + 0x4bf1ff9f0062baa8, + 0x6f773fc3603db4a9, + 0xcb550fb4384d21d3, + 0x7e2a53a146606a48, + 0x2eda7444cbfc426d, + 0xfa911155fefb5308, + 0x793555ab7eba27ca, + 0x4bc1558b2f3458de, + 0x9eb1aaedfb016f16, + 0x465e15a979c1cadc, + 0xbfacd89ec191ec9, + 0xcef980ec671f667b, + 0x82b7e12780e7401a, + 0xd1b2ecb8b0908810, + 0x861fa7e6dcb4aa15, + 0x67a791e093e1d49a, + 0xe0c8bb2c5c6d24e0, + 0x58fae9f773886e18, + 0xaf39a475506a899e, + 0x6d8406c952429603, + 0xc8e5087ba6d33b83, + 0xfb1e4a9a90880a64, + 0x5cf2eea09a55067f, + 0xf42faa48c0ea481e, + 0xf13b94daf124da26, + 0x76c53d08d6b70858, + 0x54768c4b0c64ca6e, + 0xa9942f5dcf7dfd09, + 0xd3f93b35435d7c4c, + 0xc47bc5014a1a6daf, + 0x359ab6419ca1091b, + 0xc30163d203c94b62, + 0x79e0de63425dcf1d, + 0x985915fc12f542e4, + 0x3e6f5b7b17b2939d, + 0xa705992ceecf9c42, + 0x50c6ff782a838353, + 0xa4f8bf5635246428, + 0x871b7795e136be99, + 0x28e2557b59846e3f, + 0x331aeada2fe589cf, + 0x3ff0d2c85def7621, + 0xfed077a756b53a9, + 0xd3e8495912c62894, + 0x64712dd7abbbd95c, + 0xbd8d794d96aacfb3, + 0xecf0d7a0fc5583a0, + 0xf41686c49db57244, + 0x311c2875c522ced5, + 0x7d633293366b828b, + 0xae5dff9c02033197, + 0xd9f57f830283fdfc, + 0xd072df63c324fd7b, + 0x4247cb9e59f71e6d, + 0x52d9be85f074e608, + 0x67902e276c921f8b, + 0xba1cd8a3db53b6, + 0x80e8a40eccd228a4, + 0x6122cd128006b2cd, + 0x796b805720085f81, + 0xcbe3303674053bb0, + 0xbedbfc4411068a9c, + 0xee92fb5515482d44, + 0x751bdd152d4d1c4a, + 0xd262d45a78a0635d, + 0x86fb897116c87c34, + 0xd45d35e6ae3d4da0, + 0x8974836059cca109, + 0x2bd1a438703fc94b, + 0x7b6306a34627ddcf, + 0x1a3bc84c17b1d542, + 0x20caba5f1d9e4a93, + 0x547eb47b7282ee9c, + 0xe99e619a4f23aa43, + 0x6405fa00e2ec94d4, + 0xde83bc408dd3dd04, + 0x9624ab50b148d445, + 0x3badd624dd9b0957, + 0xe54ca5d70a80e5d6, + 0x5e9fcf4ccd211f4c, + 0x7647c3200069671f, + 0x29ecd9f40041e073, + 0xf468107100525890, + 0x7182148d4066eeb4, + 0xc6f14cd848405530, + 0xb8ada00e5a506a7c, + 0xa6d90811f0e4851c, + 0x908f4a166d1da663, + 0x9a598e4e043287fe, + 0x40eff1e1853f29fd, + 0xd12bee59e68ef47c, + 0x82bb74f8301958ce, + 0xe36a52363c1faf01, + 0xdc44e6c3cb279ac1, + 0x29ab103a5ef8c0b9, + 0x7415d448f6b6f0e7, + 0x111b495b3464ad21, + 0xcab10dd900beec34, + 0x3d5d514f40eea742, + 0xcb4a5a3112a5112, + 0x47f0e785eaba72ab, + 0x59ed216765690f56, + 0x306869c13ec3532c, + 0x1e414218c73a13fb, + 0xe5d1929ef90898fa, + 0xdf45f746b74abf39, + 0x6b8bba8c328eb783, + 0x66ea92f3f326564, + 0xc80a537b0efefebd, + 0xbd06742ce95f5f36, + 0x2c48113823b73704, + 0xf75a15862ca504c5, + 0x9a984d73dbe722fb, + 0xc13e60d0d2e0ebba, + 0x318df905079926a8, + 0xfdf17746497f7052, + 0xfeb6ea8bedefa633, + 0xfe64a52ee96b8fc0, + 0x3dfdce7aa3c673b0, + 0x6bea10ca65c084e, + 0x486e494fcff30a62, + 0x5a89dba3c3efccfa, + 0xf89629465a75e01c, + 0xf6bbb397f1135823, + 0x746aa07ded582e2c, + 0xa8c2a44eb4571cdc, + 0x92f34d62616ce413, + 0x77b020baf9c81d17, + 0xace1474dc1d122e, + 0xd819992132456ba, + 0x10e1fff697ed6c69, + 0xca8d3ffa1ef463c1, + 0xbd308ff8a6b17cb2, + 0xac7cb3f6d05ddbde, + 0x6bcdf07a423aa96b, + 0x86c16c98d2c953c6, + 0xe871c7bf077ba8b7, + 0x11471cd764ad4972, + 0xd598e40d3dd89bcf, + 0x4aff1d108d4ec2c3, + 0xcedf722a585139ba, + 0xc2974eb4ee658828, + 0x733d226229feea32, + 0x806357d5a3f525f, + 0xca07c2dcb0cf26f7, + 0xfc89b393dd02f0b5, + 0xbbac2078d443ace2, + 0xd54b944b84aa4c0d, + 0xa9e795e65d4df11, + 0x4d4617b5ff4a16d5, + 0x504bced1bf8e4e45, + 0xe45ec2862f71e1d6, + 0x5d767327bb4e5a4c, + 0x3a6a07f8d510f86f, + 0x890489f70a55368b, + 0x2b45ac74ccea842e, + 0x3b0b8bc90012929d, + 0x9ce6ebb40173744, + 0xcc420a6a101d0515, + 0x9fa946824a12232d, + 0x47939822dc96abf9, + 0x59787e2b93bc56f7, + 0x57eb4edb3c55b65a, + 0xede622920b6b23f1, + 0xe95fab368e45eced, + 0x11dbcb0218ebb414, + 0xd652bdc29f26a119, + 0x4be76d3346f0495f, + 0x6f70a4400c562ddb, + 0xcb4ccd500f6bb952, + 0x7e2000a41346a7a7, + 0x8ed400668c0c28c8, + 0x728900802f0f32fa, + 0x4f2b40a03ad2ffb9, + 0xe2f610c84987bfa8, + 0xdd9ca7d2df4d7c9, + 0x91503d1c79720dbb, + 0x75a44c6397ce912a, + 0xc986afbe3ee11aba, + 0xfbe85badce996168, + 0xfae27299423fb9c3, + 0xdccd879fc967d41a, + 0x5400e987bbc1c920, + 0x290123e9aab23b68, + 0xf9a0b6720aaf6521, + 0xf808e40e8d5b3e69, + 0xb60b1d1230b20e04, + 0xb1c6f22b5e6f48c2, + 0x1e38aeb6360b1af3, + 0x25c6da63c38de1b0, + 0x579c487e5a38ad0e, + 0x2d835a9df0c6d851, + 0xf8e431456cf88e65, + 0x1b8e9ecb641b58ff, + 0xe272467e3d222f3f, + 0x5b0ed81dcc6abb0f, + 0x98e947129fc2b4e9, + 0x3f2398d747b36224, + 0x8eec7f0d19a03aad, + 0x1953cf68300424ac, + 0x5fa8c3423c052dd7, + 0x3792f412cb06794d, + 0xe2bbd88bbee40bd0, + 0x5b6aceaeae9d0ec4, + 0xf245825a5a445275, + 0xeed6e2f0f0d56712, + 0x55464dd69685606b, + 0xaa97e14c3c26b886, + 0xd53dd99f4b3066a8, + 0xe546a8038efe4029, + 0xde98520472bdd033, + 0x963e66858f6d4440, + 0xdde7001379a44aa8, + 0x5560c018580d5d52, + 0xaab8f01e6e10b4a6, + 0xcab3961304ca70e8, + 0x3d607b97c5fd0d22, + 0x8cb89a7db77c506a, + 0x77f3608e92adb242, + 0x55f038b237591ed3, + 0x6b6c46dec52f6688, + 0x2323ac4b3b3da015, + 0xabec975e0a0d081a, + 0x96e7bd358c904a21, + 0x7e50d64177da2e54, + 0xdde50bd1d5d0b9e9, + 0x955e4ec64b44e864, + 0xbd5af13bef0b113e, + 0xecb1ad8aeacdd58e, + 0x67de18eda5814af2, + 0x80eacf948770ced7, + 0xa1258379a94d028d, + 0x96ee45813a04330, + 0x8bca9d6e188853fc, + 0x775ea264cf55347d, + 0x95364afe032a819d, + 0x3a83ddbd83f52204, + 0xc4926a9672793542, + 0x75b7053c0f178293, + 0x5324c68b12dd6338, + 0xd3f6fc16ebca5e03, + 0x88f4bb1ca6bcf584, + 0x2b31e9e3d06c32e5, + 0x3aff322e62439fcf, + 0x9befeb9fad487c2, + 0x4c2ebe687989a9b3, + 0xf9d37014bf60a10, + 0x538484c19ef38c94, + 0x2865a5f206b06fb9, + 0xf93f87b7442e45d3, + 0xf78f69a51539d748, + 0xb573440e5a884d1b, + 0x31680a88f8953030, + 0xfdc20d2b36ba7c3d, + 0x3d32907604691b4c, + 0xa63f9a49c2c1b10f, + 0xfcf80dc33721d53, + 0xd3c36113404ea4a8, + 0x645a1cac083126e9, + 0x3d70a3d70a3d70a3, + 0xcccccccccccccccc, + 0x0, + 0x0, + 0x0, + 0x0, + 0x0, + 0x0, + 0x0, + 0x0, + 0x0, + 0x0, + 0x0, + 0x0, + 0x0, + 0x0, + 0x0, + 0x0, + 0x0, + 0x0, + 0x0, + 0x0, + 0x0, + 0x0, + 0x0, + 0x0, + 0x0, + 0x0, + 0x0, + 0x0, + 0x4000000000000000, + 0x5000000000000000, + 0xa400000000000000, + 0x4d00000000000000, + 0xf020000000000000, + 0x6c28000000000000, + 0xc732000000000000, + 0x3c7f400000000000, + 0x4b9f100000000000, + 0x1e86d40000000000, + 0x1314448000000000, + 0x17d955a000000000, + 0x5dcfab0800000000, + 0x5aa1cae500000000, + 0xf14a3d9e40000000, + 0x6d9ccd05d0000000, + 0xe4820023a2000000, + 0xdda2802c8a800000, + 0xd50b2037ad200000, + 0x4526f422cc340000, + 0x9670b12b7f410000, + 0x3c0cdd765f114000, + 0xa5880a69fb6ac800, + 0x8eea0d047a457a00, + 0x72a4904598d6d880, + 0x47a6da2b7f864750, + 0x999090b65f67d924, + 0xfff4b4e3f741cf6d, + 0xbff8f10e7a8921a4, + 0xaff72d52192b6a0d, + 0x9bf4f8a69f764490, + 0x2f236d04753d5b4, + 0x1d762422c946590, + 0x424d3ad2b7b97ef5, + 0xd2e0898765a7deb2, + 0x63cc55f49f88eb2f, + 0x3cbf6b71c76b25fb, + 0x8bef464e3945ef7a, + 0x97758bf0e3cbb5ac, + 0x3d52eeed1cbea317, + 0x4ca7aaa863ee4bdd, + 0x8fe8caa93e74ef6a, + 0xb3e2fd538e122b44, + 0x60dbbca87196b616, + 0xbc8955e946fe31cd, + 0x6babab6398bdbe41, + 0xc696963c7eed2dd1, + 0xfc1e1de5cf543ca2, + 0x3b25a55f43294bcb, + 0x49ef0eb713f39ebe, + 0x6e3569326c784337, + 0x49c2c37f07965404, + 0xdc33745ec97be906, + 0x69a028bb3ded71a3, + 0xc40832ea0d68ce0c, + 0xf50a3fa490c30190, + 0x792667c6da79e0fa, + 0x577001b891185938, + 0xed4c0226b55e6f86, + 0x544f8158315b05b4, + 0x696361ae3db1c721, + 0x3bc3a19cd1e38e9, + 0x4ab48a04065c723, + 0x62eb0d64283f9c76, + 0x3ba5d0bd324f8394, + 0xca8f44ec7ee36479, + 0x7e998b13cf4e1ecb, + 0x9e3fedd8c321a67e, + 0xc5cfe94ef3ea101e, + 0xbba1f1d158724a12, + 0x2a8a6e45ae8edc97, + 0xf52d09d71a3293bd, + 0x593c2626705f9c56, + 0x6f8b2fb00c77836c, + 0xb6dfb9c0f956447, + 0x4724bd4189bd5eac, + 0x58edec91ec2cb657, + 0x2f2967b66737e3ed, + 0xbd79e0d20082ee74, + 0xecd8590680a3aa11, + 0xe80e6f4820cc9495, + 0x3109058d147fdcdd, + 0xbd4b46f0599fd415, + 0x6c9e18ac7007c91a, + 0x3e2cf6bc604ddb0, + 0x84db8346b786151c, + 0xe612641865679a63, + 0x4fcb7e8f3f60c07e, + 0xe3be5e330f38f09d, + 0x5cadf5bfd3072cc5, + 0x73d9732fc7c8f7f6, + 0x2867e7fddcdd9afa, + 0xb281e1fd541501b8, + 0x1f225a7ca91a4226, + 0x3375788de9b06958, + 0x52d6b1641c83ae, + 0xc0678c5dbd23a49a, + 0xf840b7ba963646e0, + 0xb650e5a93bc3d898, + 0xa3e51f138ab4cebe, + 0xc66f336c36b10137, + 0xb80b0047445d4184, + 0xa60dc059157491e5, + 0x87c89837ad68db2f, + 0x29babe4598c311fb, + 0xf4296dd6fef3d67a, + 0x1899e4a65f58660c, + 0x5ec05dcff72e7f8f, + 0x76707543f4fa1f73, + 0x6a06494a791c53a8, + 0x487db9d17636892, + 0x45a9d2845d3c42b6, + 0xb8a2392ba45a9b2, + 0x8e6cac7768d7141e, + 0x3207d795430cd926, + 0x7f44e6bd49e807b8, + 0x5f16206c9c6209a6, + 0x36dba887c37a8c0f, + 0xc2494954da2c9789, + 0xf2db9baa10b7bd6c, + 0x6f92829494e5acc7, + 0xcb772339ba1f17f9, + 0xff2a760414536efb, + 0xfef5138519684aba, + 0x7eb258665fc25d69, + 0xef2f773ffbd97a61, + 0xaafb550ffacfd8fa, + 0x95ba2a53f983cf38, + 0xdd945a747bf26183, + 0x94f971119aeef9e4, + 0x7a37cd5601aab85d, + 0xac62e055c10ab33a, + 0x577b986b314d6009, + 0xed5a7e85fda0b80b, + 0x14588f13be847307, + 0x596eb2d8ae258fc8, + 0x6fca5f8ed9aef3bb, + 0x25de7bb9480d5854, + 0xaf561aa79a10ae6a, + 0x1b2ba1518094da04, + 0x90fb44d2f05d0842, + 0x353a1607ac744a53, + 0x42889b8997915ce8, + 0x69956135febada11, + 0x43fab9837e699095, + 0x94f967e45e03f4bb, + 0x1d1be0eebac278f5, + 0x6462d92a69731732, + 0x7d7b8f7503cfdcfe, + 0x5cda735244c3d43e, + 0x3a0888136afa64a7, + 0x88aaa1845b8fdd0, + 0x8aad549e57273d45, + 0x36ac54e2f678864b, + 0x84576a1bb416a7dd, + 0x656d44a2a11c51d5, + 0x9f644ae5a4b1b325, + 0x873d5d9f0dde1fee, + 0xa90cb506d155a7ea, + 0x9a7f12442d588f2, + 0xc11ed6d538aeb2f, + 0x8f1668c8a86da5fa, + 0xf96e017d694487bc, + 0x37c981dcc395a9ac, + 0x85bbe253f47b1417, + 0x93956d7478ccec8e, + 0x387ac8d1970027b2, + 0x6997b05fcc0319e, + 0x441fece3bdf81f03, + 0xd527e81cad7626c3, + 0x8a71e223d8d3b074, + 0xf6872d5667844e49, + 0xb428f8ac016561db, + 0xe13336d701beba52, + 0xecc0024661173473, + 0x27f002d7f95d0190, + 0x31ec038df7b441f4, + 0x7e67047175a15271, + 0xf0062c6e984d386, + 0x52c07b78a3e60868, + 0xa7709a56ccdf8a82, + 0x88a66076400bb691, + 0x6acff893d00ea435, + 0x583f6b8c4124d43, + 0xc3727a337a8b704a, + 0x744f18c0592e4c5c, + 0x1162def06f79df73, + 0x8addcb5645ac2ba8, + 0x6d953e2bd7173692, + 0xc8fa8db6ccdd0437, + 0x1d9c9892400a22a2, + 0x2503beb6d00cab4b, + 0x2e44ae64840fd61d, + 0x5ceaecfed289e5d2, + 0x7425a83e872c5f47, + 0xd12f124e28f77719, + 0x82bd6b70d99aaa6f, + 0x636cc64d1001550b, + 0x3c47f7e05401aa4e, + 0x65acfaec34810a71, + 0x7f1839a741a14d0d, + 0x1ede48111209a050, + 0x934aed0aab460432, + 0xf81da84d5617853f, + 0x36251260ab9d668e, + 0xc1d72b7c6b426019, + 0xb24cf65b8612f81f, + 0xdee033f26797b627, + 0x169840ef017da3b1, + 0x8e1f289560ee864e, + 0xf1a6f2bab92a27e2, + 0xae10af696774b1db, + 0xacca6da1e0a8ef29, + 0x17fd090a58d32af3, + 0xddfc4b4cef07f5b0, + 0x4abdaf101564f98e, + 0x9d6d1ad41abe37f1, + 0x84c86189216dc5ed, + 0x32fd3cf5b4e49bb4, + 0x3fbc8c33221dc2a1, + 0xfabaf3feaa5334a, + 0x29cb4d87f2a7400e, + 0x743e20e9ef511012, + 0x914da9246b255416, + 0x1ad089b6c2f7548e, + 0xa184ac2473b529b1, + 0xc9e5d72d90a2741e, + 0x7e2fa67c7a658892, + 0xddbb901b98feeab7, + 0x552a74227f3ea565, + 0xd53a88958f87275f, + 0x8a892abaf368f137, + 0x2d2b7569b0432d85, + 0x9c3b29620e29fc73, + 0x8349f3ba91b47b8f, + 0x241c70a936219a73, + 0xed238cd383aa0110, + 0xf4363804324a40aa, + 0xb143c6053edcd0d5, + 0xdd94b7868e94050a, + 0xca7cf2b4191c8326, + 0xfd1c2f611f63a3f0, + 0xbc633b39673c8cec, + 0xd5be0503e085d813, + 0x4b2d8644d8a74e18, + 0xddf8e7d60ed1219e, + 0xcabb90e5c942b503, + 0x3d6a751f3b936243, + 0xcc512670a783ad4, + 0x27fb2b80668b24c5, + 0xb1f9f660802dedf6, + 0x5e7873f8a0396973, + 0xdb0b487b6423e1e8, + 0x91ce1a9a3d2cda62, + 0x7641a140cc7810fb, + 0xa9e904c87fcb0a9d, + 0x546345fa9fbdcd44, + 0xa97c177947ad4095, + 0x49ed8eabcccc485d, + 0x5c68f256bfff5a74, + 0x73832eec6fff3111, + 0xc831fd53c5ff7eab, + 0xba3e7ca8b77f5e55, + 0x28ce1bd2e55f35eb, + 0x7980d163cf5b81b3, + 0xd7e105bcc332621f, + 0x8dd9472bf3fefaa7, + 0xb14f98f6f0feb951, + 0x6ed1bf9a569f33d3, + 0xa862f80ec4700c8, + 0xcd27bb612758c0fa, + 0x8038d51cb897789c, + 0xe0470a63e6bd56c3, + 0x1858ccfce06cac74, + 0xf37801e0c43ebc8, + 0xd30560258f54e6ba, + 0x47c6b82ef32a2069, + 0x4cdc331d57fa5441, + 0xe0133fe4adf8e952, + 0x58180fddd97723a6, + 0x570f09eaa7ea7648}; + + } // namespace simdjson #endif @@ -5490,17 +6631,17 @@ really_inline error_code document::parser::on_success(error_code success_code) n } really_inline bool document::parser::on_start_document(uint32_t depth) noexcept { containing_scope_offset[depth] = current_loc; - write_tape(0, tape_type::ROOT); + write_tape(0, internal::tape_type::ROOT); return true; } really_inline bool document::parser::on_start_object(uint32_t depth) noexcept { containing_scope_offset[depth] = current_loc; - write_tape(0, tape_type::START_OBJECT); + write_tape(0, internal::tape_type::START_OBJECT); return true; } really_inline bool document::parser::on_start_array(uint32_t depth) noexcept { containing_scope_offset[depth] = current_loc; - write_tape(0, tape_type::START_ARRAY); + write_tape(0, internal::tape_type::START_ARRAY); return true; } // TODO we're not checking this bool @@ -5508,39 +6649,39 @@ really_inline bool document::parser::on_end_document(uint32_t depth) noexcept { // write our doc.tape location to the header scope // The root scope gets written *at* the previous location. annotate_previous_loc(containing_scope_offset[depth], current_loc); - write_tape(containing_scope_offset[depth], tape_type::ROOT); + write_tape(containing_scope_offset[depth], internal::tape_type::ROOT); return true; } really_inline bool document::parser::on_end_object(uint32_t depth) noexcept { // write our doc.tape location to the header scope - write_tape(containing_scope_offset[depth], tape_type::END_OBJECT); + write_tape(containing_scope_offset[depth], internal::tape_type::END_OBJECT); annotate_previous_loc(containing_scope_offset[depth], current_loc); return true; } really_inline bool document::parser::on_end_array(uint32_t depth) noexcept { // write our doc.tape location to the header scope - write_tape(containing_scope_offset[depth], tape_type::END_ARRAY); + write_tape(containing_scope_offset[depth], internal::tape_type::END_ARRAY); annotate_previous_loc(containing_scope_offset[depth], current_loc); return true; } really_inline bool document::parser::on_true_atom() noexcept { - write_tape(0, tape_type::TRUE_VALUE); + write_tape(0, internal::tape_type::TRUE_VALUE); return true; } really_inline bool document::parser::on_false_atom() noexcept { - write_tape(0, tape_type::FALSE_VALUE); + write_tape(0, internal::tape_type::FALSE_VALUE); return true; } really_inline bool document::parser::on_null_atom() noexcept { - write_tape(0, tape_type::NULL_VALUE); + write_tape(0, internal::tape_type::NULL_VALUE); return true; } really_inline uint8_t *document::parser::on_start_string() noexcept { /* we advance the point, accounting for the fact that we have a NULL * termination */ - write_tape(current_string_buf_loc - doc.string_buf.get(), tape_type::STRING); + write_tape(current_string_buf_loc - doc.string_buf.get(), internal::tape_type::STRING); return current_string_buf_loc + sizeof(uint32_t); } @@ -5558,25 +6699,25 @@ really_inline bool document::parser::on_end_string(uint8_t *dst) noexcept { } really_inline bool document::parser::on_number_s64(int64_t value) noexcept { - write_tape(0, tape_type::INT64); + write_tape(0, internal::tape_type::INT64); std::memcpy(&doc.tape[current_loc], &value, sizeof(value)); ++current_loc; return true; } really_inline bool document::parser::on_number_u64(uint64_t value) noexcept { - write_tape(0, tape_type::UINT64); + write_tape(0, internal::tape_type::UINT64); doc.tape[current_loc++] = value; return true; } really_inline bool document::parser::on_number_double(double value) noexcept { - write_tape(0, tape_type::DOUBLE); + write_tape(0, internal::tape_type::DOUBLE); static_assert(sizeof(value) == sizeof(doc.tape[current_loc]), "mismatch size"); memcpy(&doc.tape[current_loc++], &value, sizeof(double)); // doc.tape[doc.current_loc++] = *((uint64_t *)&d); return true; } -really_inline void document::parser::write_tape(uint64_t val, document::tape_type t) noexcept { +really_inline void document::parser::write_tape(uint64_t val, internal::tape_type t) noexcept { doc.tape[current_loc++] = val | ((static_cast(static_cast(t))) << 56); } @@ -5591,73 +6732,22 @@ really_inline void document::parser::annotate_previous_loc(uint32_t saved_loc, u using namespace simdjson; -WARN_UNUSED -really_inline bool is_valid_true_atom(const uint8_t *loc) { - uint32_t tv = *reinterpret_cast("true"); - uint32_t error = 0; - uint32_t - locval; // we want to avoid unaligned 64-bit loads (undefined in C/C++) - // this can read up to 3 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING); - std::memcpy(&locval, loc, sizeof(uint32_t)); - error = locval ^ tv; - error |= is_not_structural_or_whitespace(loc[4]); - return error == 0; -} - -WARN_UNUSED -really_inline bool is_valid_false_atom(const uint8_t *loc) { - // assume that loc starts with "f" - uint32_t fv = *reinterpret_cast("alse"); - uint32_t error = 0; - uint32_t - locval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) - // this can read up to 4 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING); - std::memcpy(&locval, loc + 1, sizeof(uint32_t)); - error = locval ^ fv; - error |= is_not_structural_or_whitespace(loc[5]); - return error == 0; -} - -WARN_UNUSED -really_inline bool is_valid_null_atom(const uint8_t *loc) { - uint32_t nv = *reinterpret_cast("null"); - uint32_t error = 0; - uint32_t - locval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) - // this can read up to 2 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(sizeof(uint32_t) - 1 <= SIMDJSON_PADDING); - std::memcpy(&locval, loc, sizeof(uint32_t)); - error = locval ^ nv; - error |= is_not_structural_or_whitespace(loc[4]); - return error == 0; -} - #ifdef JSON_TEST_STRINGS void found_string(const uint8_t *buf, const uint8_t *parsed_begin, const uint8_t *parsed_end); void found_bad_string(const uint8_t *buf); #endif +#if SIMDJSON_IMPLEMENTATION_ARM64 /* begin file src/arm64/stage2_build_tape.h */ #ifndef SIMDJSON_ARM64_STAGE2_BUILD_TAPE_H #define SIMDJSON_ARM64_STAGE2_BUILD_TAPE_H - -#ifdef IS_ARM64 - /* arm64/implementation.h already included: #include "arm64/implementation.h" */ /* begin file src/arm64/stringparsing.h */ #ifndef SIMDJSON_ARM64_STRINGPARSING_H #define SIMDJSON_ARM64_STRINGPARSING_H - -#ifdef IS_ARM64 - /* jsoncharutils.h already included: #include "jsoncharutils.h" */ /* arm64/simd.h already included: #include "arm64/simd.h" */ /* arm64/intrinsics.h already included: #include "arm64/intrinsics.h" */ @@ -5668,16 +6758,24 @@ namespace simdjson::arm64 { using namespace simd; // Holds backslashes and quotes locations. -struct parse_string_helper { +struct backslash_and_quote { +public: + static constexpr uint32_t BYTES_PROCESSED = 32; + really_inline static backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); + + really_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } + really_inline bool has_backslash() { return bs_bits != 0; } + really_inline int quote_index() { return trailing_zeroes(quote_bits); } + really_inline int backslash_index() { return trailing_zeroes(bs_bits); } + uint32_t bs_bits; uint32_t quote_bits; - static const uint32_t BYTES_PROCESSED = 32; -}; +}; // struct backslash_and_quote -really_inline parse_string_helper find_bs_bits_and_quote_bits(const uint8_t *src, uint8_t *dst) { +really_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { // this can read up to 31 bytes beyond the buffer size, but we require // SIMDJSON_PADDING of padding - static_assert(SIMDJSON_PADDING >= (parse_string_helper::BYTES_PROCESSED - 1)); + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1)); simd8 v0(src); simd8 v1(src + sizeof(v0)); v0.store(dst); @@ -5766,24 +6864,19 @@ really_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, return offset > 0; } -WARN_UNUSED really_inline uint8_t *parse_string(const uint8_t *buf, - uint32_t offset, - uint8_t *dst) { - const uint8_t *src = &buf[offset + 1]; /* we know that buf at offset is a " */ +WARN_UNUSED really_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst) { + src++; while (1) { - parse_string_helper helper = find_bs_bits_and_quote_bits(src, dst); - if (((helper.bs_bits - 1) & helper.quote_bits) != 0) { - /* we encountered quotes first. Move dst to point to quotes and exit - */ - - /* find out where the quote is... */ - auto quote_dist = trailing_zeroes(helper.quote_bits); - - return dst + quote_dist; + // Copy the next n bytes, and find the backslash and quote in them. + auto bs_quote = backslash_and_quote::copy_and_find(src, dst); + // If the next thing is the end quote, copy and return + if (bs_quote.has_quote_first()) { + // we encountered quotes first. Move dst to point to quotes and exit + return dst + bs_quote.quote_index(); } - if (((helper.quote_bits - 1) & helper.bs_bits) != 0) { + if (bs_quote.has_backslash()) { /* find out where the backspace is */ - auto bs_dist = trailing_zeroes(helper.bs_bits); + auto bs_dist = bs_quote.backslash_index(); uint8_t escape_char = src[bs_dist + 1]; /* we encountered backslash first. Handle backslash */ if (escape_char == 'u') { @@ -5810,8 +6903,8 @@ WARN_UNUSED really_inline uint8_t *parse_string(const uint8_t *buf, } else { /* they are the same. Since they can't co-occur, it means we * encountered neither. */ - src += parse_string_helper::BYTES_PROCESSED; - dst += parse_string_helper::BYTES_PROCESSED; + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; } } /* can't be reached */ @@ -5824,17 +6917,12 @@ WARN_UNUSED really_inline uint8_t *parse_string(const uint8_t *buf, } // namespace simdjson::amd64 -#endif // IS_ARM64 - #endif // SIMDJSON_ARM64_STRINGPARSING_H /* end file src/generic/stringparsing.h */ /* begin file src/arm64/numberparsing.h */ #ifndef SIMDJSON_ARM64_NUMBERPARSING_H #define SIMDJSON_ARM64_NUMBERPARSING_H - -#ifdef IS_ARM64 - /* jsoncharutils.h already included: #include "jsoncharutils.h" */ /* arm64/intrinsics.h already included: #include "arm64/intrinsics.h" */ /* arm64/bitmanipulation.h already included: #include "arm64/bitmanipulation.h" */ @@ -5866,81 +6954,214 @@ static inline uint32_t parse_eight_digits_unrolled(const char *chars) { /* begin file src/generic/numberparsing.h */ namespace numberparsing { -// Allowable floating-point values range -// std::numeric_limits::lowest() to std::numeric_limits::max(), -// so from -1.7976e308 all the way to 1.7975e308 in binary64. The lowest -// non-zero normal values is std::numeric_limits::min() or -// about 2.225074e-308. -static const double power_of_ten[] = { - 1e-308, 1e-307, 1e-306, 1e-305, 1e-304, 1e-303, 1e-302, 1e-301, 1e-300, - 1e-299, 1e-298, 1e-297, 1e-296, 1e-295, 1e-294, 1e-293, 1e-292, 1e-291, - 1e-290, 1e-289, 1e-288, 1e-287, 1e-286, 1e-285, 1e-284, 1e-283, 1e-282, - 1e-281, 1e-280, 1e-279, 1e-278, 1e-277, 1e-276, 1e-275, 1e-274, 1e-273, - 1e-272, 1e-271, 1e-270, 1e-269, 1e-268, 1e-267, 1e-266, 1e-265, 1e-264, - 1e-263, 1e-262, 1e-261, 1e-260, 1e-259, 1e-258, 1e-257, 1e-256, 1e-255, - 1e-254, 1e-253, 1e-252, 1e-251, 1e-250, 1e-249, 1e-248, 1e-247, 1e-246, - 1e-245, 1e-244, 1e-243, 1e-242, 1e-241, 1e-240, 1e-239, 1e-238, 1e-237, - 1e-236, 1e-235, 1e-234, 1e-233, 1e-232, 1e-231, 1e-230, 1e-229, 1e-228, - 1e-227, 1e-226, 1e-225, 1e-224, 1e-223, 1e-222, 1e-221, 1e-220, 1e-219, - 1e-218, 1e-217, 1e-216, 1e-215, 1e-214, 1e-213, 1e-212, 1e-211, 1e-210, - 1e-209, 1e-208, 1e-207, 1e-206, 1e-205, 1e-204, 1e-203, 1e-202, 1e-201, - 1e-200, 1e-199, 1e-198, 1e-197, 1e-196, 1e-195, 1e-194, 1e-193, 1e-192, - 1e-191, 1e-190, 1e-189, 1e-188, 1e-187, 1e-186, 1e-185, 1e-184, 1e-183, - 1e-182, 1e-181, 1e-180, 1e-179, 1e-178, 1e-177, 1e-176, 1e-175, 1e-174, - 1e-173, 1e-172, 1e-171, 1e-170, 1e-169, 1e-168, 1e-167, 1e-166, 1e-165, - 1e-164, 1e-163, 1e-162, 1e-161, 1e-160, 1e-159, 1e-158, 1e-157, 1e-156, - 1e-155, 1e-154, 1e-153, 1e-152, 1e-151, 1e-150, 1e-149, 1e-148, 1e-147, - 1e-146, 1e-145, 1e-144, 1e-143, 1e-142, 1e-141, 1e-140, 1e-139, 1e-138, - 1e-137, 1e-136, 1e-135, 1e-134, 1e-133, 1e-132, 1e-131, 1e-130, 1e-129, - 1e-128, 1e-127, 1e-126, 1e-125, 1e-124, 1e-123, 1e-122, 1e-121, 1e-120, - 1e-119, 1e-118, 1e-117, 1e-116, 1e-115, 1e-114, 1e-113, 1e-112, 1e-111, - 1e-110, 1e-109, 1e-108, 1e-107, 1e-106, 1e-105, 1e-104, 1e-103, 1e-102, - 1e-101, 1e-100, 1e-99, 1e-98, 1e-97, 1e-96, 1e-95, 1e-94, 1e-93, - 1e-92, 1e-91, 1e-90, 1e-89, 1e-88, 1e-87, 1e-86, 1e-85, 1e-84, - 1e-83, 1e-82, 1e-81, 1e-80, 1e-79, 1e-78, 1e-77, 1e-76, 1e-75, - 1e-74, 1e-73, 1e-72, 1e-71, 1e-70, 1e-69, 1e-68, 1e-67, 1e-66, - 1e-65, 1e-64, 1e-63, 1e-62, 1e-61, 1e-60, 1e-59, 1e-58, 1e-57, - 1e-56, 1e-55, 1e-54, 1e-53, 1e-52, 1e-51, 1e-50, 1e-49, 1e-48, - 1e-47, 1e-46, 1e-45, 1e-44, 1e-43, 1e-42, 1e-41, 1e-40, 1e-39, - 1e-38, 1e-37, 1e-36, 1e-35, 1e-34, 1e-33, 1e-32, 1e-31, 1e-30, - 1e-29, 1e-28, 1e-27, 1e-26, 1e-25, 1e-24, 1e-23, 1e-22, 1e-21, - 1e-20, 1e-19, 1e-18, 1e-17, 1e-16, 1e-15, 1e-14, 1e-13, 1e-12, - 1e-11, 1e-10, 1e-9, 1e-8, 1e-7, 1e-6, 1e-5, 1e-4, 1e-3, - 1e-2, 1e-1, 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, - 1e7, 1e8, 1e9, 1e10, 1e11, 1e12, 1e13, 1e14, 1e15, - 1e16, 1e17, 1e18, 1e19, 1e20, 1e21, 1e22, 1e23, 1e24, - 1e25, 1e26, 1e27, 1e28, 1e29, 1e30, 1e31, 1e32, 1e33, - 1e34, 1e35, 1e36, 1e37, 1e38, 1e39, 1e40, 1e41, 1e42, - 1e43, 1e44, 1e45, 1e46, 1e47, 1e48, 1e49, 1e50, 1e51, - 1e52, 1e53, 1e54, 1e55, 1e56, 1e57, 1e58, 1e59, 1e60, - 1e61, 1e62, 1e63, 1e64, 1e65, 1e66, 1e67, 1e68, 1e69, - 1e70, 1e71, 1e72, 1e73, 1e74, 1e75, 1e76, 1e77, 1e78, - 1e79, 1e80, 1e81, 1e82, 1e83, 1e84, 1e85, 1e86, 1e87, - 1e88, 1e89, 1e90, 1e91, 1e92, 1e93, 1e94, 1e95, 1e96, - 1e97, 1e98, 1e99, 1e100, 1e101, 1e102, 1e103, 1e104, 1e105, - 1e106, 1e107, 1e108, 1e109, 1e110, 1e111, 1e112, 1e113, 1e114, - 1e115, 1e116, 1e117, 1e118, 1e119, 1e120, 1e121, 1e122, 1e123, - 1e124, 1e125, 1e126, 1e127, 1e128, 1e129, 1e130, 1e131, 1e132, - 1e133, 1e134, 1e135, 1e136, 1e137, 1e138, 1e139, 1e140, 1e141, - 1e142, 1e143, 1e144, 1e145, 1e146, 1e147, 1e148, 1e149, 1e150, - 1e151, 1e152, 1e153, 1e154, 1e155, 1e156, 1e157, 1e158, 1e159, - 1e160, 1e161, 1e162, 1e163, 1e164, 1e165, 1e166, 1e167, 1e168, - 1e169, 1e170, 1e171, 1e172, 1e173, 1e174, 1e175, 1e176, 1e177, - 1e178, 1e179, 1e180, 1e181, 1e182, 1e183, 1e184, 1e185, 1e186, - 1e187, 1e188, 1e189, 1e190, 1e191, 1e192, 1e193, 1e194, 1e195, - 1e196, 1e197, 1e198, 1e199, 1e200, 1e201, 1e202, 1e203, 1e204, - 1e205, 1e206, 1e207, 1e208, 1e209, 1e210, 1e211, 1e212, 1e213, - 1e214, 1e215, 1e216, 1e217, 1e218, 1e219, 1e220, 1e221, 1e222, - 1e223, 1e224, 1e225, 1e226, 1e227, 1e228, 1e229, 1e230, 1e231, - 1e232, 1e233, 1e234, 1e235, 1e236, 1e237, 1e238, 1e239, 1e240, - 1e241, 1e242, 1e243, 1e244, 1e245, 1e246, 1e247, 1e248, 1e249, - 1e250, 1e251, 1e252, 1e253, 1e254, 1e255, 1e256, 1e257, 1e258, - 1e259, 1e260, 1e261, 1e262, 1e263, 1e264, 1e265, 1e266, 1e267, - 1e268, 1e269, 1e270, 1e271, 1e272, 1e273, 1e274, 1e275, 1e276, - 1e277, 1e278, 1e279, 1e280, 1e281, 1e282, 1e283, 1e284, 1e285, - 1e286, 1e287, 1e288, 1e289, 1e290, 1e291, 1e292, 1e293, 1e294, - 1e295, 1e296, 1e297, 1e298, 1e299, 1e300, 1e301, 1e302, 1e303, - 1e304, 1e305, 1e306, 1e307, 1e308}; + +// Attempts to compute i * 10^(power) exactly; and if "negative" is +// true, negate the result. +// This function will only work in some cases, when it does not work, success is +// set to false. This should work *most of the time* (like 99% of the time). +// We assume that power is in the [FASTFLOAT_SMALLEST_POWER, +// FASTFLOAT_LARGEST_POWER] interval: the caller is responsible for this check. +really_inline double compute_float_64(int64_t power, uint64_t i, bool negative, + bool *success) { + // we start with a fast path + // It was described in + // Clinger WD. How to read floating point numbers accurately. + // ACM SIGPLAN Notices. 1990 + if (-22 <= power && power <= 22 && i <= 9007199254740991) { + // convert the integer into a double. This is lossless since + // 0 <= i <= 2^53 - 1. + double d = i; + // + // The general idea is as follows. + // If 0 <= s < 2^53 and if 10^0 <= p <= 10^22 then + // 1) Both s and p can be represented exactly as 64-bit floating-point + // values + // (binary64). + // 2) Because s and p can be represented exactly as floating-point values, + // then s * p + // and s / p will produce correctly rounded values. + // + if (power < 0) { + d = d / power_of_ten[-power]; + } else { + d = d * power_of_ten[power]; + } + if (negative) { + d = -d; + } + *success = true; + return d; + } + // When 22 < power && power < 22 + 16, we could + // hope for another, secondary fast path. It wa + // described by David M. Gay in "Correctly rounded + // binary-decimal and decimal-binary conversions." (1990) + // If you need to compute i * 10^(22 + x) for x < 16, + // first compute i * 10^x, if you know that result is exact + // (e.g., when i * 10^x < 2^53), + // then you can still proceed and do (i * 10^x) * 10^22. + // Is this worth your time? + // You need 22 < power *and* power < 22 + 16 *and* (i * 10^(x-22) < 2^53) + // for this second fast path to work. + // If you you have 22 < power *and* power < 22 + 16, and then you + // optimistically compute "i * 10^(x-22)", there is still a chance that you + // have wasted your time if i * 10^(x-22) >= 2^53. It makes the use cases of + // this optimization maybe less common than we would like. Source: + // http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ + // also used in RapidJSON: https://rapidjson.org/strtod_8h_source.html + + // The fast path has now failed, so we are failing back on the slower path. + + // In the slow path, we need to adjust i so that it is > 1<<63 which is always + // possible, except if i == 0, so we handle i == 0 separately. + if(i == 0) { + return 0.0; + } + + // We are going to need to do some 64-bit arithmetic to get a more precise product. + // We use a table lookup approach. + components c = + power_of_ten_components[power - FASTFLOAT_SMALLEST_POWER]; + // safe because + // power >= FASTFLOAT_SMALLEST_POWER + // and power <= FASTFLOAT_LARGEST_POWER + // we recover the mantissa of the power, it has a leading 1. It is always + // rounded down. + uint64_t factor_mantissa = c.mantissa; + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(i); + i <<= lz; + // We want the most significant 64 bits of the product. We know + // this will be non-zero because the most significant bit of i is + // 1. + value128 product = full_multiplication(i, factor_mantissa); + uint64_t lower = product.low; + uint64_t upper = product.high; + + // We know that upper has at most one leading zero because + // both i and factor_mantissa have a leading one. This means + // that the result is at least as large as ((1<<63)*(1<<63))/(1<<64). + + // As long as the first 9 bits of "upper" are not "1", then we + // know that we have an exact computed value for the leading + // 55 bits because any imprecision would play out as a +1, in + // the worst case. + if (unlikely((upper & 0x1FF) == 0x1FF) && (lower + i < lower)) { + uint64_t factor_mantissa_low = + mantissa_128[power - FASTFLOAT_SMALLEST_POWER]; + // next, we compute the 64-bit x 128-bit multiplication, getting a 192-bit + // result (three 64-bit values) + product = full_multiplication(i, factor_mantissa_low); + uint64_t product_low = product.low; + uint64_t product_middle2 = product.high; + uint64_t product_middle1 = lower; + uint64_t product_high = upper; + uint64_t product_middle = product_middle1 + product_middle2; + if (product_middle < product_middle1) { + product_high++; // overflow carry + } + // We want to check whether mantissa *i + i would affect our result. + // This does happen, e.g. with 7.3177701707893310e+15. + if (((product_middle + 1 == 0) && ((product_high & 0x1FF) == 0x1FF) && + (product_low + i < product_low))) { // let us be prudent and bail out. + *success = false; + return 0; + } + upper = product_high; + lower = product_middle; + } + // The final mantissa should be 53 bits with a leading 1. + // We shift it so that it occupies 54 bits with a leading 1. + /////// + uint64_t upperbit = upper >> 63; + uint64_t mantissa = upper >> (upperbit + 9); + lz += 1 ^ upperbit; + + // Here we have mantissa < (1<<54). + + // We have to round to even. The "to even" part + // is only a problem when we are right in between two floats + // which we guard against. + // If we have lots of trailing zeros, we may fall right between two + // floating-point values. + if (unlikely((lower == 0) && ((upper & 0x1FF) == 0) && + ((mantissa & 3) == 1))) { + // if mantissa & 1 == 1 we might need to round up. + // + // Scenarios: + // 1. We are not in the middle. Then we should round up. + // + // 2. We are right in the middle. Whether we round up depends + // on the last significant bit: if it is "one" then we round + // up (round to even) otherwise, we do not. + // + // So if the last significant bit is 1, we can safely round up. + // Hence we only need to bail out if (mantissa & 3) == 1. + // Otherwise we may need more accuracy or analysis to determine whether + // we are exactly between two floating-point numbers. + // It can be triggered with 1e23. + // Note: because the factor_mantissa and factor_mantissa_low are + // almost always rounded down (except for small positive powers), + // almost always should round up. + *success = false; + return 0; + } + + mantissa += mantissa & 1; + mantissa >>= 1; + + // Here we have mantissa < (1<<53), unless there was an overflow + if (mantissa >= (1ULL << 53)) { + ////////// + // This will happen when parsing values such as 7.2057594037927933e+16 + //////// + mantissa = (1ULL << 52); + lz--; // undo previous addition + } + mantissa &= ~(1ULL << 52); + uint64_t real_exponent = c.exp - lz; + // we have to check that real_exponent is in range, otherwise we bail out + if (unlikely((real_exponent < 1) || (real_exponent > 2046))) { + *success = false; + return 0; + } + mantissa |= real_exponent << 52; + mantissa |= (((uint64_t)negative) << 63); + double d; + memcpy(&d, &mantissa, sizeof(d)); + *success = true; + return d; +} + +static bool parse_float_strtod(const char *ptr, double *outDouble) { + char *endptr; + *outDouble = strtod(ptr, &endptr); + // Some libraries will set errno = ERANGE when the value is subnormal, + // yet we may want to be able to parse subnormal values. + // However, we do not want to tolerate NAN or infinite values. + // + // Values like infinity or NaN are not allowed in the JSON specification. + // If you consume a large value and you map it to "infinity", you will no + // longer be able to serialize back a standard-compliant JSON. And there is + // no realistic application where you might need values so large than they + // can't fit in binary64. The maximal value is about 1.7976931348623157 × + // 10^308 It is an unimaginable large number. There will never be any piece of + // engineering involving as many as 10^308 parts. It is estimated that there + // are about 10^80 atoms in the universe.  The estimate for the total number + // of electrons is similar. Using a double-precision floating-point value, we + // can represent easily the number of atoms in the universe. We could also + // represent the number of ways you can pick any three individual atoms at + // random in the universe. If you ever encounter a number much larger than + // 10^308, you know that you have a bug. RapidJSON will reject a document with + // a float that does not fit in binary64. JSON for Modern C++ (nlohmann/json) + // will flat out throw an exception. + // + if ((endptr == ptr) || (!std::isfinite(*outDouble))) { + return false; + } + return true; +} really_inline bool is_integer(char c) { return (c >= '0' && c <= '9'); @@ -5948,7 +7169,7 @@ really_inline bool is_integer(char c) { } // We need to check that the character following a zero is valid. This is -// probably frequent and it is hard than it looks. We are building all of this +// probably frequent and it is harder than it looks. We are building all of this // just to differentiate between 0x1 (invalid), 0,1 (valid) 0e1 (valid)... const bool structural_or_whitespace_or_exponent_or_decimal_negated[256] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -5986,162 +7207,6 @@ really_inline bool is_made_of_eight_digits_fast(const char *chars) { 0x3333333333333333); } - -// -// This function computes base * 10 ^ (- negative_exponent ). -// It is only even going to be used when negative_exponent is tiny. -really_inline double subnormal_power10(double base, int64_t negative_exponent) { - // avoid integer overflows in the pow expression, those values would - // become zero anyway. - if(negative_exponent < -1000) { - return 0; - } - - // this is probably not going to be fast - return base * 1e-308 * pow(10, negative_exponent + 308); -} - -// called by parse_number when we know that the output is a float, -// but where there might be some integer overflow. The trick here is to -// parse using floats from the start. -// Do not call this function directly as it skips some of the checks from -// parse_number -// -// This function will almost never be called!!! -// -// Note: a redesign could avoid this function entirely. -// -never_inline bool parse_float(const uint8_t *const buf, document::parser &parser, - const uint32_t offset, bool found_minus) { - const char *p = reinterpret_cast(buf + offset); - bool negative = false; - if (found_minus) { - ++p; - negative = true; - } - long double i; - if (*p == '0') { // 0 cannot be followed by an integer - ++p; - i = 0; - } else { - unsigned char digit = *p - '0'; - i = digit; - p++; - while (is_integer(*p)) { - digit = *p - '0'; - i = 10 * i + digit; - ++p; - } - } - if ('.' == *p) { - ++p; - int fractional_weight = 308; - if (is_integer(*p)) { - unsigned char digit = *p - '0'; - ++p; - - fractional_weight--; - i = i + digit * (fractional_weight >= 0 ? power_of_ten[fractional_weight] - : 0); - } else { -#ifdef JSON_TEST_NUMBERS // for unit testing - found_invalid_number(buf + offset); -#endif - return false; - } - while (is_integer(*p)) { - unsigned char digit = *p - '0'; - ++p; - fractional_weight--; - i = i + digit * (fractional_weight >= 0 ? power_of_ten[fractional_weight] - : 0); - } - } - if (('e' == *p) || ('E' == *p)) { - ++p; - bool neg_exp = false; - if ('-' == *p) { - neg_exp = true; - ++p; - } else if ('+' == *p) { - ++p; - } - if (!is_integer(*p)) { -#ifdef JSON_TEST_NUMBERS // for unit testing - found_invalid_number(buf + offset); -#endif - return false; - } - unsigned char digit = *p - '0'; - int64_t exp_number = digit; // exponential part - p++; - if (is_integer(*p)) { - digit = *p - '0'; - exp_number = 10 * exp_number + digit; - ++p; - } - if (is_integer(*p)) { - digit = *p - '0'; - exp_number = 10 * exp_number + digit; - ++p; - } - if (is_integer(*p)) { - digit = *p - '0'; - exp_number = 10 * exp_number + digit; - ++p; - } - while (is_integer(*p)) { - if (exp_number > 0x100000000) { // we need to check for overflows -// we refuse to parse this -#ifdef JSON_TEST_NUMBERS // for unit testing - found_invalid_number(buf + offset); -#endif - return false; - } - digit = *p - '0'; - exp_number = 10 * exp_number + digit; - ++p; - } - if (unlikely(exp_number > 308)) { - // this path is unlikely - if (neg_exp) { - // We either have zero or a subnormal. - // We expect this to be uncommon so we go through a slow path. - i = subnormal_power10(i, -exp_number); - } else { -// We know for sure that we have a number that is too large, -// we refuse to parse this -#ifdef JSON_TEST_NUMBERS // for unit testing - found_invalid_number(buf + offset); -#endif - return false; - } - } else { - int exponent = (neg_exp ? -exp_number : exp_number); - // we have that exp_number is [0,308] so that - // exponent is [-308,308] so that - // 308 + exponent is in [0, 2 * 308] - i *= power_of_ten[308 + exponent]; - } - } - if (is_not_structural_or_whitespace(*p)) { - return false; - } - // check that we can go from long double to double safely. - if(i > std::numeric_limits::max()) { -#ifdef JSON_TEST_NUMBERS // for unit testing - found_invalid_number(buf + offset); -#endif - return false; - } - double d = negative ? -i : i; - parser.on_number_double(d); -#ifdef JSON_TEST_NUMBERS // for unit testing - found_float(d, buf + offset); -#endif - return is_structural_or_whitespace(*p); -} - // called by parse_number when we know that the output is an integer, // but where there might be some integer overflow. // we want to catch overflows! @@ -6150,11 +7215,10 @@ never_inline bool parse_float(const uint8_t *const buf, document::parser &parser // // This function will almost never be called!!! // -never_inline bool parse_large_integer(const uint8_t *const buf, - document::parser &parser, - const uint32_t offset, - bool found_minus) { - const char *p = reinterpret_cast(buf + offset); +never_inline bool parse_large_integer(const uint8_t *const src, + document::parser &parser, + bool found_minus) { + const char *p = reinterpret_cast(src); bool negative = false; if (found_minus) { @@ -6175,13 +7239,13 @@ never_inline bool parse_large_integer(const uint8_t *const buf, digit = *p - '0'; if (mul_overflow(i, 10, &i)) { #ifdef JSON_TEST_NUMBERS // for unit testing - found_invalid_number(buf + offset); + found_invalid_number(src); #endif return false; // overflow } if (add_overflow(i, digit, &i)) { #ifdef JSON_TEST_NUMBERS // for unit testing - found_invalid_number(buf + offset); + found_invalid_number(src); #endif return false; // overflow } @@ -6190,40 +7254,40 @@ never_inline bool parse_large_integer(const uint8_t *const buf, } if (negative) { if (i > 0x8000000000000000) { - // overflows! + // overflows! #ifdef JSON_TEST_NUMBERS // for unit testing - found_invalid_number(buf + offset); + found_invalid_number(src); #endif return false; // overflow } else if (i == 0x8000000000000000) { // In two's complement, we cannot represent 0x8000000000000000 - // as a positive signed integer, but the negative version is + // as a positive signed integer, but the negative version is // possible. constexpr int64_t signed_answer = INT64_MIN; parser.on_number_s64(signed_answer); #ifdef JSON_TEST_NUMBERS // for unit testing - found_integer(signed_answer, buf + offset); + found_integer(signed_answer, src); #endif } else { // we can negate safely int64_t signed_answer = -static_cast(i); parser.on_number_s64(signed_answer); #ifdef JSON_TEST_NUMBERS // for unit testing - found_integer(signed_answer, buf + offset); + found_integer(signed_answer, src); #endif } } else { // we have a positive integer, the contract is that - // we try to represent it as a signed integer and only + // we try to represent it as a signed integer and only // fallback on unsigned integers if absolutely necessary. - if(i < 0x8000000000000000) { + if (i < 0x8000000000000000) { #ifdef JSON_TEST_NUMBERS // for unit testing - found_integer(i, buf + offset); + found_integer(i, src); #endif parser.on_number_s64(i); } else { #ifdef JSON_TEST_NUMBERS // for unit testing - found_unsigned_integer(i, buf + offset); + found_unsigned_integer(i, src); #endif parser.on_number_u64(i); } @@ -6231,7 +7295,22 @@ never_inline bool parse_large_integer(const uint8_t *const buf, return is_structural_or_whitespace(*p); } -// parse the number at buf + offset +bool slow_float_parsing(UNUSED const char * src, document::parser &parser) { + double d; + if (parse_float_strtod(src, &d)) { + parser.on_number_double(d); +#ifdef JSON_TEST_NUMBERS // for unit testing + found_float(d, (const uint8_t *)src); +#endif + return true; + } +#ifdef JSON_TEST_NUMBERS // for unit testing + found_invalid_number((const uint8_t *)src); +#endif + return false; +} + +// parse the number at src // define JSON_TEST_NUMBERS for unit testing // // It is assumed that the number is followed by a structural ({,},],[) character @@ -6239,24 +7318,23 @@ never_inline bool parse_large_integer(const uint8_t *const buf, // document is made of a single number), then it is necessary to copy the // content and append a space before calling this function. // -// Our objective is accurate parsing (ULP of 0 or 1) at high speed. -really_inline bool parse_number(UNUSED const uint8_t *const buf, - UNUSED const uint32_t offset, +// Our objective is accurate parsing (ULP of 0) at high speed. +really_inline bool parse_number(UNUSED const uint8_t *const src, UNUSED bool found_minus, document::parser &parser) { #ifdef SIMDJSON_SKIPNUMBERPARSING // for performance analysis, it is sometimes // useful to skip parsing - parser.on_number_s64(0); // always write zero + parser.on_number_s64(0); // always write zero return true; // always succeeds #else - const char *p = reinterpret_cast(buf + offset); + const char *p = reinterpret_cast(src); bool negative = false; if (found_minus) { ++p; negative = true; if (!is_integer(*p)) { // a negative sign must be followed by an integer #ifdef JSON_TEST_NUMBERS // for unit testing - found_invalid_number(buf + offset); + found_invalid_number(src); #endif return false; } @@ -6268,7 +7346,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const buf, ++p; if (is_not_structural_or_whitespace_or_exponent_or_decimal(*p)) { #ifdef JSON_TEST_NUMBERS // for unit testing - found_invalid_number(buf + offset); + found_invalid_number(src); #endif return false; } @@ -6276,7 +7354,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const buf, } else { if (!(is_integer(*p))) { // must start with an integer #ifdef JSON_TEST_NUMBERS // for unit testing - found_invalid_number(buf + offset); + found_invalid_number(src); #endif return false; } @@ -6311,7 +7389,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const buf, // we will handle the overflow later } else { #ifdef JSON_TEST_NUMBERS // for unit testing - found_invalid_number(buf + offset); + found_invalid_number(src); #endif return false; } @@ -6346,7 +7424,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const buf, } if (!is_integer(*p)) { #ifdef JSON_TEST_NUMBERS // for unit testing - found_invalid_number(buf + offset); + found_invalid_number(src); #endif return false; } @@ -6367,7 +7445,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const buf, if (exp_number > 0x100000000) { // we need to check for overflows // we refuse to parse this #ifdef JSON_TEST_NUMBERS // for unit testing - found_invalid_number(buf + offset); + found_invalid_number(src); #endif return false; } @@ -6378,7 +7456,9 @@ really_inline bool parse_number(UNUSED const uint8_t *const buf, exponent += (neg_exp ? -exp_number : exp_number); } if (is_float) { - uint64_t power_index = 308 + exponent; + // If we frequently had to deal with long strings of digits, + // we could extend our code by using a 128-bit integer instead + // of a 64-bit integer. However, this is uncommon in practice. if (unlikely((digit_count >= 19))) { // this is uncommon // It is possible that the integer had an overflow. // We have to handle the case where we have 0.0000somenumber. @@ -6392,31 +7472,47 @@ really_inline bool parse_number(UNUSED const uint8_t *const buf, // Ok, chances are good that we had an overflow! // this is almost never going to get called!!! // we start anew, going slowly!!! - return parse_float(buf, parser, offset, found_minus); + // This will happen in the following examples: + // 10000000000000000000000000000000000000000000e+308 + // 3.1415926535897932384626433832795028841971693993751 + // + return slow_float_parsing((const char *) src, parser); } } - if (unlikely((power_index > 2 * 308))) { // this is uncommon!!! + if (unlikely(exponent < FASTFLOAT_SMALLEST_POWER) || + (exponent > FASTFLOAT_LARGEST_POWER)) { // this is uncommon!!! // this is almost never going to get called!!! // we start anew, going slowly!!! - return parse_float(buf, parser, offset, found_minus); + return slow_float_parsing((const char *) src, parser); } - double factor = power_of_ten[power_index]; - factor = negative ? -factor : factor; - double d = i * factor; - parser.on_number_double(d); + bool success = true; + double d = compute_float_64(exponent, i, negative, &success); + if (!success) { + // we are almost never going to get here. + success = parse_float_strtod((const char *)src, &d); + } + if (success) { + parser.on_number_double(d); #ifdef JSON_TEST_NUMBERS // for unit testing - found_float(d, buf + offset); + found_float(d, src); #endif + return true; + } else { +#ifdef JSON_TEST_NUMBERS // for unit testing + found_invalid_number(src); +#endif + return false; + } } else { if (unlikely(digit_count >= 18)) { // this is uncommon!!! // there is a good chance that we had an overflow, so we need // need to recover: we parse the whole thing again. - return parse_large_integer(buf, parser, offset, found_minus); + return parse_large_integer(src, parser, found_minus); } i = negative ? 0 - i : i; parser.on_number_s64(i); #ifdef JSON_TEST_NUMBERS // for unit testing - found_integer(i, buf + offset); + found_integer(i, src); #endif } return is_structural_or_whitespace(*p); @@ -6426,16 +7522,64 @@ really_inline bool parse_number(UNUSED const uint8_t *const buf, } // namespace numberparsing /* end file src/generic/numberparsing.h */ +} // namespace simdjson::arm64 -}// namespace simdjson::arm64 - - -#endif // IS_ARM64 -#endif // SIMDJSON_ARM64_NUMBERPARSING_H +#endif // SIMDJSON_ARM64_NUMBERPARSING_H /* end file src/generic/numberparsing.h */ namespace simdjson::arm64 { +/* begin file src/generic/atomparsing.h */ +namespace atomparsing { + +really_inline uint32_t string_to_uint32(const char* str) { return *reinterpret_cast(str); } + +WARN_UNUSED +really_inline bool str4ncmp(const uint8_t *src, const char* atom) { + uint32_t srcval; // we want to avoid unaligned 64-bit loads (undefined in C/C++) + static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING); + std::memcpy(&srcval, src, sizeof(uint32_t)); + return srcval ^ string_to_uint32(atom); +} + +WARN_UNUSED +really_inline bool is_valid_true_atom(const uint8_t *src) { + return (str4ncmp(src, "true") | is_not_structural_or_whitespace(src[4])) == 0; +} + +WARN_UNUSED +really_inline bool is_valid_true_atom(const uint8_t *src, size_t len) { + if (len > 4) { return is_valid_true_atom(src); } + else if (len == 4) { return !str4ncmp(src, "true"); } + else { return false; } +} + +WARN_UNUSED +really_inline bool is_valid_false_atom(const uint8_t *src) { + return (str4ncmp(src+1, "alse") | is_not_structural_or_whitespace(src[5])) == 0; +} + +WARN_UNUSED +really_inline bool is_valid_false_atom(const uint8_t *src, size_t len) { + if (len > 5) { return is_valid_false_atom(src); } + else if (len == 5) { return !str4ncmp(src+1, "alse"); } + else { return false; } +} + +WARN_UNUSED +really_inline bool is_valid_null_atom(const uint8_t *src) { + return (str4ncmp(src, "null") | is_not_structural_or_whitespace(src[4])) == 0; +} + +WARN_UNUSED +really_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { + if (len > 4) { return is_valid_null_atom(src); } + else if (len == 4) { return !str4ncmp(src, "null"); } + else { return false; } +} + +} // namespace atomparsing +/* end file src/generic/atomparsing.h */ /* begin file src/generic/stage2_build_tape.h */ // This file contains the common code every implementation uses for stage2 // It is intended to be included multiple times and compiled multiple times @@ -6486,28 +7630,25 @@ struct unified_machine_addresses { #undef FAIL_IF #define FAIL_IF(EXPR) { if (EXPR) { return addresses.error; } } -struct structural_parser { - const uint8_t* const buf; - const size_t len; - document::parser &doc_parser; - size_t i; // next structural index - size_t idx; // location of the structural character in the input (buf) - uint8_t c; // used to track the (structural) character we are looking at - uint32_t depth = 0; // could have an arbitrary starting depth - - really_inline structural_parser( - const uint8_t *_buf, - size_t _len, - document::parser &_doc_parser, - uint32_t _i = 0 - ) : buf{_buf}, len{_len}, doc_parser{_doc_parser}, i{_i} {} - +class structural_iterator { +public: + really_inline structural_iterator(const uint8_t* _buf, size_t _len, const uint32_t *_structural_indexes, size_t next_structural_index) + : buf{_buf}, len{_len}, structural_indexes{_structural_indexes}, next_structural{next_structural_index} {} really_inline char advance_char() { - idx = doc_parser.structural_indexes[i++]; - c = buf[idx]; + idx = structural_indexes[next_structural]; + next_structural++; + c = *current(); return c; } - + really_inline char current_char() { + return c; + } + really_inline const uint8_t* current() { + return &buf[idx]; + } + really_inline size_t remaining_len() { + return len - idx; + } template really_inline bool with_space_terminated_copy(const F& f) { /** @@ -6533,6 +7674,35 @@ struct structural_parser { free(copy); return result; } + really_inline bool past_end(uint32_t n_structural_indexes) { + return next_structural+1 > n_structural_indexes; + } + really_inline bool at_end(uint32_t n_structural_indexes) { + return next_structural+1 == n_structural_indexes; + } + really_inline size_t next_structural_index() { + return next_structural; + } + + const uint8_t* const buf; + const size_t len; + const uint32_t* const structural_indexes; + size_t next_structural; // next structural index + size_t idx; // location of the structural character in the input (buf) + uint8_t c; // used to track the (structural) character we are looking at +}; + +struct structural_parser { + structural_iterator structurals; + document::parser &doc_parser; + uint32_t depth; + + really_inline structural_parser( + const uint8_t *buf, + size_t len, + document::parser &_doc_parser, + uint32_t next_structural = 0 + ) : structurals(buf, len, _doc_parser.structural_indexes.get(), next_structural), doc_parser{_doc_parser}, depth{0} {} WARN_UNUSED really_inline bool start_document(ret_address continue_state) { doc_parser.on_start_document(depth); @@ -6573,32 +7743,32 @@ struct structural_parser { WARN_UNUSED really_inline bool parse_string() { uint8_t *dst = doc_parser.on_start_string(); - dst = stringparsing::parse_string(buf, idx, dst); + dst = stringparsing::parse_string(structurals.current(), dst); if (dst == nullptr) { return true; } return !doc_parser.on_end_string(dst); } - WARN_UNUSED really_inline bool parse_number(const uint8_t *copy, uint32_t offset, bool found_minus) { - return !numberparsing::parse_number(copy, offset, found_minus, doc_parser); + WARN_UNUSED really_inline bool parse_number(const uint8_t *src, bool found_minus) { + return !numberparsing::parse_number(src, found_minus, doc_parser); } WARN_UNUSED really_inline bool parse_number(bool found_minus) { - return parse_number(buf, idx, found_minus); + return parse_number(structurals.current(), found_minus); } - WARN_UNUSED really_inline bool parse_atom(const uint8_t *copy, uint32_t offset) { - switch (c) { + WARN_UNUSED really_inline bool parse_atom() { + switch (structurals.current_char()) { case 't': - if (!is_valid_true_atom(copy + offset)) { return true; } + if (!atomparsing::is_valid_true_atom(structurals.current())) { return true; } doc_parser.on_true_atom(); break; case 'f': - if (!is_valid_false_atom(copy + offset)) { return true; } + if (!atomparsing::is_valid_false_atom(structurals.current())) { return true; } doc_parser.on_false_atom(); break; case 'n': - if (!is_valid_null_atom(copy + offset)) { return true; } + if (!atomparsing::is_valid_null_atom(structurals.current())) { return true; } doc_parser.on_null_atom(); break; default: @@ -6607,12 +7777,28 @@ struct structural_parser { return false; } - WARN_UNUSED really_inline bool parse_atom() { - return parse_atom(buf, idx); + WARN_UNUSED really_inline bool parse_single_atom() { + switch (structurals.current_char()) { + case 't': + if (!atomparsing::is_valid_true_atom(structurals.current(), structurals.remaining_len())) { return true; } + doc_parser.on_true_atom(); + break; + case 'f': + if (!atomparsing::is_valid_false_atom(structurals.current(), structurals.remaining_len())) { return true; } + doc_parser.on_false_atom(); + break; + case 'n': + if (!atomparsing::is_valid_null_atom(structurals.current(), structurals.remaining_len())) { return true; } + doc_parser.on_null_atom(); + break; + default: + return true; + } + return false; } WARN_UNUSED really_inline ret_address parse_value(const unified_machine_addresses &addresses, ret_address continue_state) { - switch (c) { + switch (structurals.current_char()) { case '"': FAIL_IF( parse_string() ); return continue_state; @@ -6639,7 +7825,7 @@ struct structural_parser { WARN_UNUSED really_inline error_code finish() { // the string might not be NULL terminated. - if ( i + 1 != doc_parser.n_structural_indexes ) { + if ( !structurals.at_end(doc_parser.n_structural_indexes) ) { return doc_parser.on_error(TAPE_ERROR); } end_document(); @@ -6667,7 +7853,7 @@ struct structural_parser { if (depth >= doc_parser.max_depth()) { return doc_parser.on_error(DEPTH_ERROR); } - switch (c) { + switch (structurals.current_char()) { case '"': return doc_parser.on_error(STRING_ERROR); case '0': @@ -6693,19 +7879,23 @@ struct structural_parser { } } - WARN_UNUSED really_inline error_code start(ret_address finish_state) { + WARN_UNUSED really_inline error_code start(size_t len, ret_address finish_state) { doc_parser.init_stage2(); // sets is_valid to false if (len > doc_parser.capacity()) { return CAPACITY; } // Advance to the first character as soon as possible - advance_char(); + structurals.advance_char(); // Push the root scope (there is always at least one scope) if (start_document(finish_state)) { return doc_parser.on_error(DEPTH_ERROR); } return SUCCESS; } + + really_inline char advance_char() { + return structurals.advance_char(); + } }; // Redefine FAIL_IF to use goto since it'll be used inside the function now @@ -6721,13 +7911,13 @@ struct structural_parser { WARN_UNUSED error_code implementation::stage2(const uint8_t *buf, size_t len, document::parser &doc_parser) const noexcept { static constexpr stage2::unified_machine_addresses addresses = INIT_ADDRESSES(); stage2::structural_parser parser(buf, len, doc_parser); - error_code result = parser.start(addresses.finish); + error_code result = parser.start(len, addresses.finish); if (result) { return result; } // // Read first value // - switch (parser.c) { + switch (parser.structurals.current_char()) { case '{': FAIL_IF( parser.start_object(addresses.finish) ); goto object_begin; @@ -6738,24 +7928,20 @@ WARN_UNUSED error_code implementation::stage2(const uint8_t *buf, size_t len, do FAIL_IF( parser.parse_string() ); goto finish; case 't': case 'f': case 'n': - FAIL_IF( - parser.with_space_terminated_copy([&](auto copy, auto idx) { - return parser.parse_atom(copy, idx); - }) - ); + FAIL_IF( parser.parse_single_atom() ); goto finish; case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': FAIL_IF( - parser.with_space_terminated_copy([&](auto copy, auto idx) { - return parser.parse_number(copy, idx, false); + parser.structurals.with_space_terminated_copy([&](auto copy, auto idx) { + return parser.parse_number(©[idx], false); }) ); goto finish; case '-': FAIL_IF( - parser.with_space_terminated_copy([&](auto copy, auto idx) { - return parser.parse_number(copy, idx, true); + parser.structurals.with_space_terminated_copy([&](auto copy, auto idx) { + return parser.parse_number(©[idx], true); }) ); goto finish; @@ -6767,8 +7953,7 @@ WARN_UNUSED error_code implementation::stage2(const uint8_t *buf, size_t len, do // Object parser states // object_begin: - parser.advance_char(); - switch (parser.c) { + switch (parser.advance_char()) { case '"': { FAIL_IF( parser.parse_string() ); goto object_key_state; @@ -6849,7 +8034,7 @@ struct streaming_structural_parser: structural_parser { really_inline streaming_structural_parser(const uint8_t *_buf, size_t _len, document::parser &_doc_parser, size_t _i) : structural_parser(_buf, _len, _doc_parser, _i) {} // override to add streaming - WARN_UNUSED really_inline error_code start(ret_address finish_parser) { + WARN_UNUSED really_inline error_code start(UNUSED size_t len, ret_address finish_parser) { doc_parser.init_stage2(); // sets is_valid to false // Capacity ain't no thang for streaming, so we don't check it. // Advance to the first character as soon as possible @@ -6863,7 +8048,7 @@ struct streaming_structural_parser: structural_parser { // override to add streaming WARN_UNUSED really_inline error_code finish() { - if ( i + 1 > doc_parser.n_structural_indexes ) { + if ( structurals.past_end(doc_parser.n_structural_indexes) ) { return doc_parser.on_error(TAPE_ERROR); } end_document(); @@ -6873,7 +8058,7 @@ struct streaming_structural_parser: structural_parser { if (doc_parser.containing_scope_offset[depth] != 0) { return doc_parser.on_error(TAPE_ERROR); } - bool finished = i + 1 == doc_parser.n_structural_indexes; + bool finished = structurals.at_end(doc_parser.n_structural_indexes); return doc_parser.on_success(finished ? SUCCESS : SUCCESS_AND_HAS_MORE); } }; @@ -6887,12 +8072,12 @@ struct streaming_structural_parser: structural_parser { WARN_UNUSED error_code implementation::stage2(const uint8_t *buf, size_t len, document::parser &doc_parser, size_t &next_json) const noexcept { static constexpr stage2::unified_machine_addresses addresses = INIT_ADDRESSES(); stage2::streaming_structural_parser parser(buf, len, doc_parser, next_json); - error_code result = parser.start(addresses.finish); + error_code result = parser.start(len, addresses.finish); if (result) { return result; } // // Read first value // - switch (parser.c) { + switch (parser.structurals.current_char()) { case '{': FAIL_IF( parser.start_object(addresses.finish) ); goto object_begin; @@ -6903,24 +8088,20 @@ WARN_UNUSED error_code implementation::stage2(const uint8_t *buf, size_t len, do FAIL_IF( parser.parse_string() ); goto finish; case 't': case 'f': case 'n': - FAIL_IF( - parser.with_space_terminated_copy([&](auto copy, auto idx) { - return parser.parse_atom(copy, idx); - }) - ); + FAIL_IF( parser.parse_single_atom() ); goto finish; case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': FAIL_IF( - parser.with_space_terminated_copy([&](auto copy, auto idx) { - return parser.parse_number(copy, idx, false); + parser.structurals.with_space_terminated_copy([&](auto copy, auto idx) { + return parser.parse_number(©[idx], false); }) ); goto finish; case '-': FAIL_IF( - parser.with_space_terminated_copy([&](auto copy, auto idx) { - return parser.parse_number(copy, idx, true); + parser.structurals.with_space_terminated_copy([&](auto copy, auto idx) { + return parser.parse_number(©[idx], true); }) ); goto finish; @@ -6932,8 +8113,7 @@ WARN_UNUSED error_code implementation::stage2(const uint8_t *buf, size_t len, do // Object parser parsers // object_begin: - parser.advance_char(); - switch (parser.c) { + switch (parser.advance_char()) { case '"': { FAIL_IF( parser.parse_string() ); goto object_key_parser; @@ -6993,7 +8173,7 @@ array_continue: } finish: - next_json = parser.i; + next_json = parser.structurals.next_structural_index(); return parser.finish(); error: @@ -7003,25 +8183,1507 @@ error: } // namespace simdjson::arm64 -#endif // IS_ARM64 - #endif // SIMDJSON_ARM64_STAGE2_BUILD_TAPE_H /* end file src/generic/stage2_streaming_build_tape.h */ +#endif +#if SIMDJSON_IMPLEMENTATION_FALLBACK +/* begin file src/fallback/stage2_build_tape.h */ +#ifndef SIMDJSON_FALLBACK_STAGE2_BUILD_TAPE_H +#define SIMDJSON_FALLBACK_STAGE2_BUILD_TAPE_H + + +/* fallback/implementation.h already included: #include "fallback/implementation.h" */ +/* begin file src/fallback/stringparsing.h */ +#ifndef SIMDJSON_FALLBACK_STRINGPARSING_H +#define SIMDJSON_FALLBACK_STRINGPARSING_H + +/* jsoncharutils.h already included: #include "jsoncharutils.h" */ + +namespace simdjson::fallback { + +// Holds backslashes and quotes locations. +struct backslash_and_quote { +public: + static constexpr uint32_t BYTES_PROCESSED = 1; + really_inline static backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); + + really_inline bool has_quote_first() { return c == '"'; } + really_inline bool has_backslash() { return c == '\\'; } + really_inline int quote_index() { return c == '"' ? 0 : 1; } + really_inline int backslash_index() { return c == '\\' ? 0 : 1; } + + uint8_t c; +}; // struct backslash_and_quote + +really_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { + // store to dest unconditionally - we can overwrite the bits we don't like later + dst[0] = src[0]; + return { src[0] }; +} + +/* begin file src/generic/stringparsing.h */ +// This file contains the common code every implementation uses +// It is intended to be included multiple times and compiled multiple times +// We assume the file in which it is include already includes +// "stringparsing.h" (this simplifies amalgation) + +namespace stringparsing { + +// begin copypasta +// These chars yield themselves: " \ / +// b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab +// u not handled in this table as it's complex +static const uint8_t escape_map[256] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0. + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0x22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2f, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4. + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5c, 0, 0, 0, // 0x5. + 0, 0, 0x08, 0, 0, 0, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0x0a, 0, // 0x6. + 0, 0, 0x0d, 0, 0x09, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x7. + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; + +// handle a unicode codepoint +// write appropriate values into dest +// src will advance 6 bytes or 12 bytes +// dest will advance a variable amount (return via pointer) +// return true if the unicode codepoint was valid +// We work in little-endian then swap at write time +WARN_UNUSED +really_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, + uint8_t **dst_ptr) { + // hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the + // conversion isn't valid; we defer the check for this to inside the + // multilingual plane check + uint32_t code_point = hex_to_u32_nocheck(*src_ptr + 2); + *src_ptr += 6; + // check for low surrogate for characters outside the Basic + // Multilingual Plane. + if (code_point >= 0xd800 && code_point < 0xdc00) { + if (((*src_ptr)[0] != '\\') || (*src_ptr)[1] != 'u') { + return false; + } + uint32_t code_point_2 = hex_to_u32_nocheck(*src_ptr + 2); + + // if the first code point is invalid we will get here, as we will go past + // the check for being outside the Basic Multilingual plane. If we don't + // find a \u immediately afterwards we fail out anyhow, but if we do, + // this check catches both the case of the first code point being invalid + // or the second code point being invalid. + if ((code_point | code_point_2) >> 16) { + return false; + } + + code_point = + (((code_point - 0xd800) << 10) | (code_point_2 - 0xdc00)) + 0x10000; + *src_ptr += 6; + } + size_t offset = codepoint_to_utf8(code_point, *dst_ptr); + *dst_ptr += offset; + return offset > 0; +} + +WARN_UNUSED really_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst) { + src++; + while (1) { + // Copy the next n bytes, and find the backslash and quote in them. + auto bs_quote = backslash_and_quote::copy_and_find(src, dst); + // If the next thing is the end quote, copy and return + if (bs_quote.has_quote_first()) { + // we encountered quotes first. Move dst to point to quotes and exit + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + /* find out where the backspace is */ + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + /* we encountered backslash first. Handle backslash */ + if (escape_char == 'u') { + /* move src/dst up to the start; they will be further adjusted + within the unicode codepoint handling code. */ + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint(&src, &dst)) { + return nullptr; + } + } else { + /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and + * write bs_dist+1 characters to output + * note this may reach beyond the part of the buffer we've actually + * seen. I think this is ok */ + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; /* bogus escape value is an error */ + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } else { + /* they are the same. Since they can't co-occur, it means we + * encountered neither. */ + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } + /* can't be reached */ + return nullptr; +} + +} // namespace stringparsing +/* end file src/generic/stringparsing.h */ + +} // namespace simdjson::fallback + +#endif // SIMDJSON_FALLBACK_STRINGPARSING_H +/* end file src/generic/stringparsing.h */ +/* begin file src/fallback/numberparsing.h */ +#ifndef SIMDJSON_FALLBACK_NUMBERPARSING_H +#define SIMDJSON_FALLBACK_NUMBERPARSING_H + +/* jsoncharutils.h already included: #include "jsoncharutils.h" */ +/* begin file src/fallback/bitmanipulation.h */ +#ifndef SIMDJSON_FALLBACK_BITMANIPULATION_H +#define SIMDJSON_FALLBACK_BITMANIPULATION_H + +#include + +namespace simdjson::fallback { + +#ifndef _MSC_VER +// We sometimes call trailing_zero on inputs that are zero, +// but the algorithms do not end up using the returned value. +// Sadly, sanitizers are not smart enough to figure it out. +__attribute__((no_sanitize("undefined"))) // this is deliberate +#endif // _MSC_VER +/* result might be undefined when input_num is zero */ +really_inline int trailing_zeroes(uint64_t input_num) { + +#ifdef _MSC_VER + unsigned long ret; + // Search the mask data from least significant bit (LSB) + // to the most significant bit (MSB) for a set bit (1). + _BitScanForward64(&ret, input_num); + return (int)ret; +#else + return __builtin_ctzll(input_num); +#endif // _MSC_VER + +} // namespace simdjson::arm64 + +/* result might be undefined when input_num is zero */ +really_inline uint64_t clear_lowest_bit(uint64_t input_num) { + return input_num & (input_num-1); +} + +/* result might be undefined when input_num is zero */ +really_inline int leading_zeroes(uint64_t input_num) { +#ifdef _MSC_VER + unsigned long leading_zero = 0; + // Search the mask data from most significant bit (MSB) + // to least significant bit (LSB) for a set bit (1). + if (_BitScanReverse64(&leading_zero, input_num)) + return (int)(63 - leading_zero); + else + return 64; +#else + return __builtin_clzll(input_num); +#endif// _MSC_VER +} + +really_inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t *result) { + *result = value1 + value2; + return *result < value1; +} + +really_inline bool mul_overflow(uint64_t value1, uint64_t value2, uint64_t *result) { + *result = value1 * value2; + // TODO there must be a faster way + return value2 > 0 && value1 > std::numeric_limits::max() / value2; +} + +} // namespace simdjson::fallback + +#endif // SIMDJSON_FALLBACK_BITMANIPULATION_H +/* end file src/fallback/bitmanipulation.h */ +#include +#include + +#ifdef JSON_TEST_NUMBERS // for unit testing +void found_invalid_number(const uint8_t *buf); +void found_integer(int64_t result, const uint8_t *buf); +void found_unsigned_integer(uint64_t result, const uint8_t *buf); +void found_float(double result, const uint8_t *buf); +#endif + +namespace simdjson::fallback { +static inline uint32_t parse_eight_digits_unrolled(const char *chars) { + uint32_t result = 0; + for (int i=0;i<8;i++) { + result = result*10 + (chars[i] - '0'); + } + return result; +} + +#define SWAR_NUMBER_PARSING + +/* begin file src/generic/numberparsing.h */ +namespace numberparsing { + + +// Attempts to compute i * 10^(power) exactly; and if "negative" is +// true, negate the result. +// This function will only work in some cases, when it does not work, success is +// set to false. This should work *most of the time* (like 99% of the time). +// We assume that power is in the [FASTFLOAT_SMALLEST_POWER, +// FASTFLOAT_LARGEST_POWER] interval: the caller is responsible for this check. +really_inline double compute_float_64(int64_t power, uint64_t i, bool negative, + bool *success) { + // we start with a fast path + // It was described in + // Clinger WD. How to read floating point numbers accurately. + // ACM SIGPLAN Notices. 1990 + if (-22 <= power && power <= 22 && i <= 9007199254740991) { + // convert the integer into a double. This is lossless since + // 0 <= i <= 2^53 - 1. + double d = i; + // + // The general idea is as follows. + // If 0 <= s < 2^53 and if 10^0 <= p <= 10^22 then + // 1) Both s and p can be represented exactly as 64-bit floating-point + // values + // (binary64). + // 2) Because s and p can be represented exactly as floating-point values, + // then s * p + // and s / p will produce correctly rounded values. + // + if (power < 0) { + d = d / power_of_ten[-power]; + } else { + d = d * power_of_ten[power]; + } + if (negative) { + d = -d; + } + *success = true; + return d; + } + // When 22 < power && power < 22 + 16, we could + // hope for another, secondary fast path. It wa + // described by David M. Gay in "Correctly rounded + // binary-decimal and decimal-binary conversions." (1990) + // If you need to compute i * 10^(22 + x) for x < 16, + // first compute i * 10^x, if you know that result is exact + // (e.g., when i * 10^x < 2^53), + // then you can still proceed and do (i * 10^x) * 10^22. + // Is this worth your time? + // You need 22 < power *and* power < 22 + 16 *and* (i * 10^(x-22) < 2^53) + // for this second fast path to work. + // If you you have 22 < power *and* power < 22 + 16, and then you + // optimistically compute "i * 10^(x-22)", there is still a chance that you + // have wasted your time if i * 10^(x-22) >= 2^53. It makes the use cases of + // this optimization maybe less common than we would like. Source: + // http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ + // also used in RapidJSON: https://rapidjson.org/strtod_8h_source.html + + // The fast path has now failed, so we are failing back on the slower path. + + // In the slow path, we need to adjust i so that it is > 1<<63 which is always + // possible, except if i == 0, so we handle i == 0 separately. + if(i == 0) { + return 0.0; + } + + // We are going to need to do some 64-bit arithmetic to get a more precise product. + // We use a table lookup approach. + components c = + power_of_ten_components[power - FASTFLOAT_SMALLEST_POWER]; + // safe because + // power >= FASTFLOAT_SMALLEST_POWER + // and power <= FASTFLOAT_LARGEST_POWER + // we recover the mantissa of the power, it has a leading 1. It is always + // rounded down. + uint64_t factor_mantissa = c.mantissa; + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(i); + i <<= lz; + // We want the most significant 64 bits of the product. We know + // this will be non-zero because the most significant bit of i is + // 1. + value128 product = full_multiplication(i, factor_mantissa); + uint64_t lower = product.low; + uint64_t upper = product.high; + + // We know that upper has at most one leading zero because + // both i and factor_mantissa have a leading one. This means + // that the result is at least as large as ((1<<63)*(1<<63))/(1<<64). + + // As long as the first 9 bits of "upper" are not "1", then we + // know that we have an exact computed value for the leading + // 55 bits because any imprecision would play out as a +1, in + // the worst case. + if (unlikely((upper & 0x1FF) == 0x1FF) && (lower + i < lower)) { + uint64_t factor_mantissa_low = + mantissa_128[power - FASTFLOAT_SMALLEST_POWER]; + // next, we compute the 64-bit x 128-bit multiplication, getting a 192-bit + // result (three 64-bit values) + product = full_multiplication(i, factor_mantissa_low); + uint64_t product_low = product.low; + uint64_t product_middle2 = product.high; + uint64_t product_middle1 = lower; + uint64_t product_high = upper; + uint64_t product_middle = product_middle1 + product_middle2; + if (product_middle < product_middle1) { + product_high++; // overflow carry + } + // We want to check whether mantissa *i + i would affect our result. + // This does happen, e.g. with 7.3177701707893310e+15. + if (((product_middle + 1 == 0) && ((product_high & 0x1FF) == 0x1FF) && + (product_low + i < product_low))) { // let us be prudent and bail out. + *success = false; + return 0; + } + upper = product_high; + lower = product_middle; + } + // The final mantissa should be 53 bits with a leading 1. + // We shift it so that it occupies 54 bits with a leading 1. + /////// + uint64_t upperbit = upper >> 63; + uint64_t mantissa = upper >> (upperbit + 9); + lz += 1 ^ upperbit; + + // Here we have mantissa < (1<<54). + + // We have to round to even. The "to even" part + // is only a problem when we are right in between two floats + // which we guard against. + // If we have lots of trailing zeros, we may fall right between two + // floating-point values. + if (unlikely((lower == 0) && ((upper & 0x1FF) == 0) && + ((mantissa & 3) == 1))) { + // if mantissa & 1 == 1 we might need to round up. + // + // Scenarios: + // 1. We are not in the middle. Then we should round up. + // + // 2. We are right in the middle. Whether we round up depends + // on the last significant bit: if it is "one" then we round + // up (round to even) otherwise, we do not. + // + // So if the last significant bit is 1, we can safely round up. + // Hence we only need to bail out if (mantissa & 3) == 1. + // Otherwise we may need more accuracy or analysis to determine whether + // we are exactly between two floating-point numbers. + // It can be triggered with 1e23. + // Note: because the factor_mantissa and factor_mantissa_low are + // almost always rounded down (except for small positive powers), + // almost always should round up. + *success = false; + return 0; + } + + mantissa += mantissa & 1; + mantissa >>= 1; + + // Here we have mantissa < (1<<53), unless there was an overflow + if (mantissa >= (1ULL << 53)) { + ////////// + // This will happen when parsing values such as 7.2057594037927933e+16 + //////// + mantissa = (1ULL << 52); + lz--; // undo previous addition + } + mantissa &= ~(1ULL << 52); + uint64_t real_exponent = c.exp - lz; + // we have to check that real_exponent is in range, otherwise we bail out + if (unlikely((real_exponent < 1) || (real_exponent > 2046))) { + *success = false; + return 0; + } + mantissa |= real_exponent << 52; + mantissa |= (((uint64_t)negative) << 63); + double d; + memcpy(&d, &mantissa, sizeof(d)); + *success = true; + return d; +} + +static bool parse_float_strtod(const char *ptr, double *outDouble) { + char *endptr; + *outDouble = strtod(ptr, &endptr); + // Some libraries will set errno = ERANGE when the value is subnormal, + // yet we may want to be able to parse subnormal values. + // However, we do not want to tolerate NAN or infinite values. + // + // Values like infinity or NaN are not allowed in the JSON specification. + // If you consume a large value and you map it to "infinity", you will no + // longer be able to serialize back a standard-compliant JSON. And there is + // no realistic application where you might need values so large than they + // can't fit in binary64. The maximal value is about 1.7976931348623157 × + // 10^308 It is an unimaginable large number. There will never be any piece of + // engineering involving as many as 10^308 parts. It is estimated that there + // are about 10^80 atoms in the universe.  The estimate for the total number + // of electrons is similar. Using a double-precision floating-point value, we + // can represent easily the number of atoms in the universe. We could also + // represent the number of ways you can pick any three individual atoms at + // random in the universe. If you ever encounter a number much larger than + // 10^308, you know that you have a bug. RapidJSON will reject a document with + // a float that does not fit in binary64. JSON for Modern C++ (nlohmann/json) + // will flat out throw an exception. + // + if ((endptr == ptr) || (!std::isfinite(*outDouble))) { + return false; + } + return true; +} + +really_inline bool is_integer(char c) { + return (c >= '0' && c <= '9'); + // this gets compiled to (uint8_t)(c - '0') <= 9 on all decent compilers +} + +// We need to check that the character following a zero is valid. This is +// probably frequent and it is harder than it looks. We are building all of this +// just to differentiate between 0x1 (invalid), 0,1 (valid) 0e1 (valid)... +const bool structural_or_whitespace_or_exponent_or_decimal_negated[256] = { + 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, + 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; + +really_inline bool +is_not_structural_or_whitespace_or_exponent_or_decimal(unsigned char c) { + return structural_or_whitespace_or_exponent_or_decimal_negated[c]; +} + +// check quickly whether the next 8 chars are made of digits +// at a glance, it looks better than Mula's +// http://0x80.pl/articles/swar-digits-validate.html +really_inline bool is_made_of_eight_digits_fast(const char *chars) { + uint64_t val; + // this can read up to 7 bytes beyond the buffer size, but we require + // SIMDJSON_PADDING of padding + static_assert(7 <= SIMDJSON_PADDING); + memcpy(&val, chars, 8); + // a branchy method might be faster: + // return (( val & 0xF0F0F0F0F0F0F0F0 ) == 0x3030303030303030) + // && (( (val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0 ) == + // 0x3030303030303030); + return (((val & 0xF0F0F0F0F0F0F0F0) | + (((val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0) >> 4)) == + 0x3333333333333333); +} + +// called by parse_number when we know that the output is an integer, +// but where there might be some integer overflow. +// we want to catch overflows! +// Do not call this function directly as it skips some of the checks from +// parse_number +// +// This function will almost never be called!!! +// +never_inline bool parse_large_integer(const uint8_t *const src, + document::parser &parser, + bool found_minus) { + const char *p = reinterpret_cast(src); + + bool negative = false; + if (found_minus) { + ++p; + negative = true; + } + uint64_t i; + if (*p == '0') { // 0 cannot be followed by an integer + ++p; + i = 0; + } else { + unsigned char digit = *p - '0'; + i = digit; + p++; + // the is_made_of_eight_digits_fast routine is unlikely to help here because + // we rarely see large integer parts like 123456789 + while (is_integer(*p)) { + digit = *p - '0'; + if (mul_overflow(i, 10, &i)) { +#ifdef JSON_TEST_NUMBERS // for unit testing + found_invalid_number(src); +#endif + return false; // overflow + } + if (add_overflow(i, digit, &i)) { +#ifdef JSON_TEST_NUMBERS // for unit testing + found_invalid_number(src); +#endif + return false; // overflow + } + ++p; + } + } + if (negative) { + if (i > 0x8000000000000000) { + // overflows! +#ifdef JSON_TEST_NUMBERS // for unit testing + found_invalid_number(src); +#endif + return false; // overflow + } else if (i == 0x8000000000000000) { + // In two's complement, we cannot represent 0x8000000000000000 + // as a positive signed integer, but the negative version is + // possible. + constexpr int64_t signed_answer = INT64_MIN; + parser.on_number_s64(signed_answer); +#ifdef JSON_TEST_NUMBERS // for unit testing + found_integer(signed_answer, src); +#endif + } else { + // we can negate safely + int64_t signed_answer = -static_cast(i); + parser.on_number_s64(signed_answer); +#ifdef JSON_TEST_NUMBERS // for unit testing + found_integer(signed_answer, src); +#endif + } + } else { + // we have a positive integer, the contract is that + // we try to represent it as a signed integer and only + // fallback on unsigned integers if absolutely necessary. + if (i < 0x8000000000000000) { +#ifdef JSON_TEST_NUMBERS // for unit testing + found_integer(i, src); +#endif + parser.on_number_s64(i); + } else { +#ifdef JSON_TEST_NUMBERS // for unit testing + found_unsigned_integer(i, src); +#endif + parser.on_number_u64(i); + } + } + return is_structural_or_whitespace(*p); +} + +bool slow_float_parsing(UNUSED const char * src, document::parser &parser) { + double d; + if (parse_float_strtod(src, &d)) { + parser.on_number_double(d); +#ifdef JSON_TEST_NUMBERS // for unit testing + found_float(d, (const uint8_t *)src); +#endif + return true; + } +#ifdef JSON_TEST_NUMBERS // for unit testing + found_invalid_number((const uint8_t *)src); +#endif + return false; +} + +// parse the number at src +// define JSON_TEST_NUMBERS for unit testing +// +// It is assumed that the number is followed by a structural ({,},],[) character +// or a white space character. If that is not the case (e.g., when the JSON +// document is made of a single number), then it is necessary to copy the +// content and append a space before calling this function. +// +// Our objective is accurate parsing (ULP of 0) at high speed. +really_inline bool parse_number(UNUSED const uint8_t *const src, + UNUSED bool found_minus, + document::parser &parser) { +#ifdef SIMDJSON_SKIPNUMBERPARSING // for performance analysis, it is sometimes + // useful to skip parsing + parser.on_number_s64(0); // always write zero + return true; // always succeeds +#else + const char *p = reinterpret_cast(src); + bool negative = false; + if (found_minus) { + ++p; + negative = true; + if (!is_integer(*p)) { // a negative sign must be followed by an integer +#ifdef JSON_TEST_NUMBERS // for unit testing + found_invalid_number(src); +#endif + return false; + } + } + const char *const start_digits = p; + + uint64_t i; // an unsigned int avoids signed overflows (which are bad) + if (*p == '0') { // 0 cannot be followed by an integer + ++p; + if (is_not_structural_or_whitespace_or_exponent_or_decimal(*p)) { +#ifdef JSON_TEST_NUMBERS // for unit testing + found_invalid_number(src); +#endif + return false; + } + i = 0; + } else { + if (!(is_integer(*p))) { // must start with an integer +#ifdef JSON_TEST_NUMBERS // for unit testing + found_invalid_number(src); +#endif + return false; + } + unsigned char digit = *p - '0'; + i = digit; + p++; + // the is_made_of_eight_digits_fast routine is unlikely to help here because + // we rarely see large integer parts like 123456789 + while (is_integer(*p)) { + digit = *p - '0'; + // a multiplication by 10 is cheaper than an arbitrary integer + // multiplication + i = 10 * i + digit; // might overflow, we will handle the overflow later + ++p; + } + } + int64_t exponent = 0; + bool is_float = false; + if ('.' == *p) { + is_float = true; // At this point we know that we have a float + // we continue with the fiction that we have an integer. If the + // floating point number is representable as x * 10^z for some integer + // z that fits in 53 bits, then we will be able to convert back the + // the integer into a float in a lossless manner. + ++p; + const char *const first_after_period = p; + if (is_integer(*p)) { + unsigned char digit = *p - '0'; + ++p; + i = i * 10 + digit; // might overflow + multiplication by 10 is likely + // cheaper than arbitrary mult. + // we will handle the overflow later + } else { +#ifdef JSON_TEST_NUMBERS // for unit testing + found_invalid_number(src); +#endif + return false; + } +#ifdef SWAR_NUMBER_PARSING + // this helps if we have lots of decimals! + // this turns out to be frequent enough. + if (is_made_of_eight_digits_fast(p)) { + i = i * 100000000 + parse_eight_digits_unrolled(p); + p += 8; + } +#endif + while (is_integer(*p)) { + unsigned char digit = *p - '0'; + ++p; + i = i * 10 + digit; // in rare cases, this will overflow, but that's ok + // because we have parse_highprecision_float later. + } + exponent = first_after_period - p; + } + int digit_count = + p - start_digits - 1; // used later to guard against overflows + int64_t exp_number = 0; // exponential part + if (('e' == *p) || ('E' == *p)) { + is_float = true; + ++p; + bool neg_exp = false; + if ('-' == *p) { + neg_exp = true; + ++p; + } else if ('+' == *p) { + ++p; + } + if (!is_integer(*p)) { +#ifdef JSON_TEST_NUMBERS // for unit testing + found_invalid_number(src); +#endif + return false; + } + unsigned char digit = *p - '0'; + exp_number = digit; + p++; + if (is_integer(*p)) { + digit = *p - '0'; + exp_number = 10 * exp_number + digit; + ++p; + } + if (is_integer(*p)) { + digit = *p - '0'; + exp_number = 10 * exp_number + digit; + ++p; + } + while (is_integer(*p)) { + if (exp_number > 0x100000000) { // we need to check for overflows + // we refuse to parse this +#ifdef JSON_TEST_NUMBERS // for unit testing + found_invalid_number(src); +#endif + return false; + } + digit = *p - '0'; + exp_number = 10 * exp_number + digit; + ++p; + } + exponent += (neg_exp ? -exp_number : exp_number); + } + if (is_float) { + // If we frequently had to deal with long strings of digits, + // we could extend our code by using a 128-bit integer instead + // of a 64-bit integer. However, this is uncommon in practice. + if (unlikely((digit_count >= 19))) { // this is uncommon + // It is possible that the integer had an overflow. + // We have to handle the case where we have 0.0000somenumber. + const char *start = start_digits; + while ((*start == '0') || (*start == '.')) { + start++; + } + // we over-decrement by one when there is a '.' + digit_count -= (start - start_digits); + if (digit_count >= 19) { + // Ok, chances are good that we had an overflow! + // this is almost never going to get called!!! + // we start anew, going slowly!!! + // This will happen in the following examples: + // 10000000000000000000000000000000000000000000e+308 + // 3.1415926535897932384626433832795028841971693993751 + // + return slow_float_parsing((const char *) src, parser); + } + } + if (unlikely(exponent < FASTFLOAT_SMALLEST_POWER) || + (exponent > FASTFLOAT_LARGEST_POWER)) { // this is uncommon!!! + // this is almost never going to get called!!! + // we start anew, going slowly!!! + return slow_float_parsing((const char *) src, parser); + } + bool success = true; + double d = compute_float_64(exponent, i, negative, &success); + if (!success) { + // we are almost never going to get here. + success = parse_float_strtod((const char *)src, &d); + } + if (success) { + parser.on_number_double(d); +#ifdef JSON_TEST_NUMBERS // for unit testing + found_float(d, src); +#endif + return true; + } else { +#ifdef JSON_TEST_NUMBERS // for unit testing + found_invalid_number(src); +#endif + return false; + } + } else { + if (unlikely(digit_count >= 18)) { // this is uncommon!!! + // there is a good chance that we had an overflow, so we need + // need to recover: we parse the whole thing again. + return parse_large_integer(src, parser, found_minus); + } + i = negative ? 0 - i : i; + parser.on_number_s64(i); +#ifdef JSON_TEST_NUMBERS // for unit testing + found_integer(i, src); +#endif + } + return is_structural_or_whitespace(*p); +#endif // SIMDJSON_SKIPNUMBERPARSING +} + +} // namespace numberparsing +/* end file src/generic/numberparsing.h */ + +} // namespace simdjson::fallback + +#endif // SIMDJSON_FALLBACK_NUMBERPARSING_H +/* end file src/generic/numberparsing.h */ + +namespace simdjson::fallback { + +/* begin file src/generic/atomparsing.h */ +namespace atomparsing { + +really_inline uint32_t string_to_uint32(const char* str) { return *reinterpret_cast(str); } + +WARN_UNUSED +really_inline bool str4ncmp(const uint8_t *src, const char* atom) { + uint32_t srcval; // we want to avoid unaligned 64-bit loads (undefined in C/C++) + static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING); + std::memcpy(&srcval, src, sizeof(uint32_t)); + return srcval ^ string_to_uint32(atom); +} + +WARN_UNUSED +really_inline bool is_valid_true_atom(const uint8_t *src) { + return (str4ncmp(src, "true") | is_not_structural_or_whitespace(src[4])) == 0; +} + +WARN_UNUSED +really_inline bool is_valid_true_atom(const uint8_t *src, size_t len) { + if (len > 4) { return is_valid_true_atom(src); } + else if (len == 4) { return !str4ncmp(src, "true"); } + else { return false; } +} + +WARN_UNUSED +really_inline bool is_valid_false_atom(const uint8_t *src) { + return (str4ncmp(src+1, "alse") | is_not_structural_or_whitespace(src[5])) == 0; +} + +WARN_UNUSED +really_inline bool is_valid_false_atom(const uint8_t *src, size_t len) { + if (len > 5) { return is_valid_false_atom(src); } + else if (len == 5) { return !str4ncmp(src+1, "alse"); } + else { return false; } +} + +WARN_UNUSED +really_inline bool is_valid_null_atom(const uint8_t *src) { + return (str4ncmp(src, "null") | is_not_structural_or_whitespace(src[4])) == 0; +} + +WARN_UNUSED +really_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { + if (len > 4) { return is_valid_null_atom(src); } + else if (len == 4) { return !str4ncmp(src, "null"); } + else { return false; } +} + +} // namespace atomparsing +/* end file src/generic/atomparsing.h */ +/* begin file src/generic/stage2_build_tape.h */ +// This file contains the common code every implementation uses for stage2 +// It is intended to be included multiple times and compiled multiple times +// We assume the file in which it is include already includes +// "simdjson/stage2_build_tape.h" (this simplifies amalgation) + +namespace stage2 { + +#ifdef SIMDJSON_USE_COMPUTED_GOTO +typedef void* ret_address; +#define INIT_ADDRESSES() { &&array_begin, &&array_continue, &&error, &&finish, &&object_begin, &&object_continue } +#define GOTO(address) { goto *(address); } +#define CONTINUE(address) { goto *(address); } +#else +typedef char ret_address; +#define INIT_ADDRESSES() { '[', 'a', 'e', 'f', '{', 'o' }; +#define GOTO(address) \ + { \ + switch(address) { \ + case '[': goto array_begin; \ + case 'a': goto array_continue; \ + case 'e': goto error; \ + case 'f': goto finish; \ + case '{': goto object_begin; \ + case 'o': goto object_continue; \ + } \ + } +// For the more constrained end_xxx() situation +#define CONTINUE(address) \ + { \ + switch(address) { \ + case 'a': goto array_continue; \ + case 'o': goto object_continue; \ + case 'f': goto finish; \ + } \ + } +#endif + +struct unified_machine_addresses { + ret_address array_begin; + ret_address array_continue; + ret_address error; + ret_address finish; + ret_address object_begin; + ret_address object_continue; +}; + +#undef FAIL_IF +#define FAIL_IF(EXPR) { if (EXPR) { return addresses.error; } } + +class structural_iterator { +public: + really_inline structural_iterator(const uint8_t* _buf, size_t _len, const uint32_t *_structural_indexes, size_t next_structural_index) + : buf{_buf}, len{_len}, structural_indexes{_structural_indexes}, next_structural{next_structural_index} {} + really_inline char advance_char() { + idx = structural_indexes[next_structural]; + next_structural++; + c = *current(); + return c; + } + really_inline char current_char() { + return c; + } + really_inline const uint8_t* current() { + return &buf[idx]; + } + really_inline size_t remaining_len() { + return len - idx; + } + template + really_inline bool with_space_terminated_copy(const F& f) { + /** + * We need to make a copy to make sure that the string is space terminated. + * This is not about padding the input, which should already padded up + * to len + SIMDJSON_PADDING. However, we have no control at this stage + * on how the padding was done. What if the input string was padded with nulls? + * It is quite common for an input string to have an extra null character (C string). + * We do not want to allow 9\0 (where \0 is the null character) inside a JSON + * document, but the string "9\0" by itself is fine. So we make a copy and + * pad the input with spaces when we know that there is just one input element. + * This copy is relatively expensive, but it will almost never be called in + * practice unless you are in the strange scenario where you have many JSON + * documents made of single atoms. + */ + char *copy = static_cast(malloc(len + SIMDJSON_PADDING)); + if (copy == nullptr) { + return true; + } + memcpy(copy, buf, len); + memset(copy + len, ' ', SIMDJSON_PADDING); + bool result = f(reinterpret_cast(copy), idx); + free(copy); + return result; + } + really_inline bool past_end(uint32_t n_structural_indexes) { + return next_structural+1 > n_structural_indexes; + } + really_inline bool at_end(uint32_t n_structural_indexes) { + return next_structural+1 == n_structural_indexes; + } + really_inline size_t next_structural_index() { + return next_structural; + } + + const uint8_t* const buf; + const size_t len; + const uint32_t* const structural_indexes; + size_t next_structural; // next structural index + size_t idx; // location of the structural character in the input (buf) + uint8_t c; // used to track the (structural) character we are looking at +}; + +struct structural_parser { + structural_iterator structurals; + document::parser &doc_parser; + uint32_t depth; + + really_inline structural_parser( + const uint8_t *buf, + size_t len, + document::parser &_doc_parser, + uint32_t next_structural = 0 + ) : structurals(buf, len, _doc_parser.structural_indexes.get(), next_structural), doc_parser{_doc_parser}, depth{0} {} + + WARN_UNUSED really_inline bool start_document(ret_address continue_state) { + doc_parser.on_start_document(depth); + doc_parser.ret_address[depth] = continue_state; + depth++; + return depth >= doc_parser.max_depth(); + } + + WARN_UNUSED really_inline bool start_object(ret_address continue_state) { + doc_parser.on_start_object(depth); + doc_parser.ret_address[depth] = continue_state; + depth++; + return depth >= doc_parser.max_depth(); + } + + WARN_UNUSED really_inline bool start_array(ret_address continue_state) { + doc_parser.on_start_array(depth); + doc_parser.ret_address[depth] = continue_state; + depth++; + return depth >= doc_parser.max_depth(); + } + + really_inline bool end_object() { + depth--; + doc_parser.on_end_object(depth); + return false; + } + really_inline bool end_array() { + depth--; + doc_parser.on_end_array(depth); + return false; + } + really_inline bool end_document() { + depth--; + doc_parser.on_end_document(depth); + return false; + } + + WARN_UNUSED really_inline bool parse_string() { + uint8_t *dst = doc_parser.on_start_string(); + dst = stringparsing::parse_string(structurals.current(), dst); + if (dst == nullptr) { + return true; + } + return !doc_parser.on_end_string(dst); + } + + WARN_UNUSED really_inline bool parse_number(const uint8_t *src, bool found_minus) { + return !numberparsing::parse_number(src, found_minus, doc_parser); + } + WARN_UNUSED really_inline bool parse_number(bool found_minus) { + return parse_number(structurals.current(), found_minus); + } + + WARN_UNUSED really_inline bool parse_atom() { + switch (structurals.current_char()) { + case 't': + if (!atomparsing::is_valid_true_atom(structurals.current())) { return true; } + doc_parser.on_true_atom(); + break; + case 'f': + if (!atomparsing::is_valid_false_atom(structurals.current())) { return true; } + doc_parser.on_false_atom(); + break; + case 'n': + if (!atomparsing::is_valid_null_atom(structurals.current())) { return true; } + doc_parser.on_null_atom(); + break; + default: + return true; + } + return false; + } + + WARN_UNUSED really_inline bool parse_single_atom() { + switch (structurals.current_char()) { + case 't': + if (!atomparsing::is_valid_true_atom(structurals.current(), structurals.remaining_len())) { return true; } + doc_parser.on_true_atom(); + break; + case 'f': + if (!atomparsing::is_valid_false_atom(structurals.current(), structurals.remaining_len())) { return true; } + doc_parser.on_false_atom(); + break; + case 'n': + if (!atomparsing::is_valid_null_atom(structurals.current(), structurals.remaining_len())) { return true; } + doc_parser.on_null_atom(); + break; + default: + return true; + } + return false; + } + + WARN_UNUSED really_inline ret_address parse_value(const unified_machine_addresses &addresses, ret_address continue_state) { + switch (structurals.current_char()) { + case '"': + FAIL_IF( parse_string() ); + return continue_state; + case 't': case 'f': case 'n': + FAIL_IF( parse_atom() ); + return continue_state; + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + FAIL_IF( parse_number(false) ); + return continue_state; + case '-': + FAIL_IF( parse_number(true) ); + return continue_state; + case '{': + FAIL_IF( start_object(continue_state) ); + return addresses.object_begin; + case '[': + FAIL_IF( start_array(continue_state) ); + return addresses.array_begin; + default: + return addresses.error; + } + } + + WARN_UNUSED really_inline error_code finish() { + // the string might not be NULL terminated. + if ( !structurals.at_end(doc_parser.n_structural_indexes) ) { + return doc_parser.on_error(TAPE_ERROR); + } + end_document(); + if (depth != 0) { + return doc_parser.on_error(TAPE_ERROR); + } + if (doc_parser.containing_scope_offset[depth] != 0) { + return doc_parser.on_error(TAPE_ERROR); + } + + return doc_parser.on_success(SUCCESS); + } + + WARN_UNUSED really_inline error_code error() { + /* We do not need the next line because this is done by doc_parser.init_stage2(), + * pessimistically. + * doc_parser.is_valid = false; + * At this point in the code, we have all the time in the world. + * Note that we know exactly where we are in the document so we could, + * without any overhead on the processing code, report a specific + * location. + * We could even trigger special code paths to assess what happened + * carefully, + * all without any added cost. */ + if (depth >= doc_parser.max_depth()) { + return doc_parser.on_error(DEPTH_ERROR); + } + switch (structurals.current_char()) { + case '"': + return doc_parser.on_error(STRING_ERROR); + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + case '-': + return doc_parser.on_error(NUMBER_ERROR); + case 't': + return doc_parser.on_error(T_ATOM_ERROR); + case 'n': + return doc_parser.on_error(N_ATOM_ERROR); + case 'f': + return doc_parser.on_error(F_ATOM_ERROR); + default: + return doc_parser.on_error(TAPE_ERROR); + } + } + + WARN_UNUSED really_inline error_code start(size_t len, ret_address finish_state) { + doc_parser.init_stage2(); // sets is_valid to false + if (len > doc_parser.capacity()) { + return CAPACITY; + } + // Advance to the first character as soon as possible + structurals.advance_char(); + // Push the root scope (there is always at least one scope) + if (start_document(finish_state)) { + return doc_parser.on_error(DEPTH_ERROR); + } + return SUCCESS; + } + + really_inline char advance_char() { + return structurals.advance_char(); + } +}; + +// Redefine FAIL_IF to use goto since it'll be used inside the function now +#undef FAIL_IF +#define FAIL_IF(EXPR) { if (EXPR) { goto error; } } + +} // namespace stage2 + +/************ + * The JSON is parsed to a tape, see the accompanying tape.md file + * for documentation. + ***********/ +WARN_UNUSED error_code implementation::stage2(const uint8_t *buf, size_t len, document::parser &doc_parser) const noexcept { + static constexpr stage2::unified_machine_addresses addresses = INIT_ADDRESSES(); + stage2::structural_parser parser(buf, len, doc_parser); + error_code result = parser.start(len, addresses.finish); + if (result) { return result; } + + // + // Read first value + // + switch (parser.structurals.current_char()) { + case '{': + FAIL_IF( parser.start_object(addresses.finish) ); + goto object_begin; + case '[': + FAIL_IF( parser.start_array(addresses.finish) ); + goto array_begin; + case '"': + FAIL_IF( parser.parse_string() ); + goto finish; + case 't': case 'f': case 'n': + FAIL_IF( parser.parse_single_atom() ); + goto finish; + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + FAIL_IF( + parser.structurals.with_space_terminated_copy([&](auto copy, auto idx) { + return parser.parse_number(©[idx], false); + }) + ); + goto finish; + case '-': + FAIL_IF( + parser.structurals.with_space_terminated_copy([&](auto copy, auto idx) { + return parser.parse_number(©[idx], true); + }) + ); + goto finish; + default: + goto error; + } + +// +// Object parser states +// +object_begin: + switch (parser.advance_char()) { + case '"': { + FAIL_IF( parser.parse_string() ); + goto object_key_state; + } + case '}': + parser.end_object(); + goto scope_end; + default: + goto error; + } + +object_key_state: + FAIL_IF( parser.advance_char() != ':' ); + parser.advance_char(); + GOTO( parser.parse_value(addresses, addresses.object_continue) ); + +object_continue: + switch (parser.advance_char()) { + case ',': + FAIL_IF( parser.advance_char() != '"' ); + FAIL_IF( parser.parse_string() ); + goto object_key_state; + case '}': + parser.end_object(); + goto scope_end; + default: + goto error; + } + +scope_end: + CONTINUE( parser.doc_parser.ret_address[parser.depth] ); + +// +// Array parser states +// +array_begin: + if (parser.advance_char() == ']') { + parser.end_array(); + goto scope_end; + } + +main_array_switch: + /* we call update char on all paths in, so we can peek at parser.c on the + * on paths that can accept a close square brace (post-, and at start) */ + GOTO( parser.parse_value(addresses, addresses.array_continue) ); + +array_continue: + switch (parser.advance_char()) { + case ',': + parser.advance_char(); + goto main_array_switch; + case ']': + parser.end_array(); + goto scope_end; + default: + goto error; + } + +finish: + return parser.finish(); + +error: + return parser.error(); +} + +WARN_UNUSED error_code implementation::parse(const uint8_t *buf, size_t len, document::parser &doc_parser) const noexcept { + error_code code = stage1(buf, len, doc_parser, false); + if (!code) { + code = stage2(buf, len, doc_parser); + } + return code; +} +/* end file src/generic/stage2_build_tape.h */ +/* begin file src/generic/stage2_streaming_build_tape.h */ +namespace stage2 { + +struct streaming_structural_parser: structural_parser { + really_inline streaming_structural_parser(const uint8_t *_buf, size_t _len, document::parser &_doc_parser, size_t _i) : structural_parser(_buf, _len, _doc_parser, _i) {} + + // override to add streaming + WARN_UNUSED really_inline error_code start(UNUSED size_t len, ret_address finish_parser) { + doc_parser.init_stage2(); // sets is_valid to false + // Capacity ain't no thang for streaming, so we don't check it. + // Advance to the first character as soon as possible + advance_char(); + // Push the root scope (there is always at least one scope) + if (start_document(finish_parser)) { + return doc_parser.on_error(DEPTH_ERROR); + } + return SUCCESS; + } + + // override to add streaming + WARN_UNUSED really_inline error_code finish() { + if ( structurals.past_end(doc_parser.n_structural_indexes) ) { + return doc_parser.on_error(TAPE_ERROR); + } + end_document(); + if (depth != 0) { + return doc_parser.on_error(TAPE_ERROR); + } + if (doc_parser.containing_scope_offset[depth] != 0) { + return doc_parser.on_error(TAPE_ERROR); + } + bool finished = structurals.at_end(doc_parser.n_structural_indexes); + return doc_parser.on_success(finished ? SUCCESS : SUCCESS_AND_HAS_MORE); + } +}; + +} // namespace stage2 + +/************ + * The JSON is parsed to a tape, see the accompanying tape.md file + * for documentation. + ***********/ +WARN_UNUSED error_code implementation::stage2(const uint8_t *buf, size_t len, document::parser &doc_parser, size_t &next_json) const noexcept { + static constexpr stage2::unified_machine_addresses addresses = INIT_ADDRESSES(); + stage2::streaming_structural_parser parser(buf, len, doc_parser, next_json); + error_code result = parser.start(len, addresses.finish); + if (result) { return result; } + // + // Read first value + // + switch (parser.structurals.current_char()) { + case '{': + FAIL_IF( parser.start_object(addresses.finish) ); + goto object_begin; + case '[': + FAIL_IF( parser.start_array(addresses.finish) ); + goto array_begin; + case '"': + FAIL_IF( parser.parse_string() ); + goto finish; + case 't': case 'f': case 'n': + FAIL_IF( parser.parse_single_atom() ); + goto finish; + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + FAIL_IF( + parser.structurals.with_space_terminated_copy([&](auto copy, auto idx) { + return parser.parse_number(©[idx], false); + }) + ); + goto finish; + case '-': + FAIL_IF( + parser.structurals.with_space_terminated_copy([&](auto copy, auto idx) { + return parser.parse_number(©[idx], true); + }) + ); + goto finish; + default: + goto error; + } + +// +// Object parser parsers +// +object_begin: + switch (parser.advance_char()) { + case '"': { + FAIL_IF( parser.parse_string() ); + goto object_key_parser; + } + case '}': + parser.end_object(); + goto scope_end; + default: + goto error; + } + +object_key_parser: + FAIL_IF( parser.advance_char() != ':' ); + parser.advance_char(); + GOTO( parser.parse_value(addresses, addresses.object_continue) ); + +object_continue: + switch (parser.advance_char()) { + case ',': + FAIL_IF( parser.advance_char() != '"' ); + FAIL_IF( parser.parse_string() ); + goto object_key_parser; + case '}': + parser.end_object(); + goto scope_end; + default: + goto error; + } + +scope_end: + CONTINUE( parser.doc_parser.ret_address[parser.depth] ); + +// +// Array parser parsers +// +array_begin: + if (parser.advance_char() == ']') { + parser.end_array(); + goto scope_end; + } + +main_array_switch: + /* we call update char on all paths in, so we can peek at parser.c on the + * on paths that can accept a close square brace (post-, and at start) */ + GOTO( parser.parse_value(addresses, addresses.array_continue) ); + +array_continue: + switch (parser.advance_char()) { + case ',': + parser.advance_char(); + goto main_array_switch; + case ']': + parser.end_array(); + goto scope_end; + default: + goto error; + } + +finish: + next_json = parser.structurals.next_structural_index(); + return parser.finish(); + +error: + return parser.error(); +} +/* end file src/generic/stage2_streaming_build_tape.h */ + +} // namespace simdjson + +#endif // SIMDJSON_FALLBACK_STAGE2_BUILD_TAPE_H +/* end file src/generic/stage2_streaming_build_tape.h */ +#endif +#if SIMDJSON_IMPLEMENTATION_HASWELL /* begin file src/haswell/stage2_build_tape.h */ #ifndef SIMDJSON_HASWELL_STAGE2_BUILD_TAPE_H #define SIMDJSON_HASWELL_STAGE2_BUILD_TAPE_H - -#ifdef IS_X86_64 - /* haswell/implementation.h already included: #include "haswell/implementation.h" */ /* begin file src/haswell/stringparsing.h */ #ifndef SIMDJSON_HASWELL_STRINGPARSING_H #define SIMDJSON_HASWELL_STRINGPARSING_H - -#ifdef IS_X86_64 - /* jsoncharutils.h already included: #include "jsoncharutils.h" */ /* haswell/simd.h already included: #include "haswell/simd.h" */ /* haswell/intrinsics.h already included: #include "haswell/intrinsics.h" */ @@ -7033,16 +9695,24 @@ namespace simdjson::haswell { using namespace simd; // Holds backslashes and quotes locations. -struct parse_string_helper { +struct backslash_and_quote { +public: + static constexpr uint32_t BYTES_PROCESSED = 32; + really_inline static backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); + + really_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } + really_inline bool has_backslash() { return ((quote_bits - 1) & bs_bits) != 0; } + really_inline int quote_index() { return trailing_zeroes(quote_bits); } + really_inline int backslash_index() { return trailing_zeroes(bs_bits); } + uint32_t bs_bits; uint32_t quote_bits; - static const uint32_t BYTES_PROCESSED = 32; -}; +}; // struct backslash_and_quote -really_inline parse_string_helper find_bs_bits_and_quote_bits(const uint8_t *src, uint8_t *dst) { +really_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { // this can read up to 15 bytes beyond the buffer size, but we require // SIMDJSON_PADDING of padding - static_assert(SIMDJSON_PADDING >= (parse_string_helper::BYTES_PROCESSED - 1)); + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1)); simd8 v(src); // store to dest unconditionally - we can overwrite the bits we don't like later v.store(dst); @@ -7126,24 +9796,19 @@ really_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, return offset > 0; } -WARN_UNUSED really_inline uint8_t *parse_string(const uint8_t *buf, - uint32_t offset, - uint8_t *dst) { - const uint8_t *src = &buf[offset + 1]; /* we know that buf at offset is a " */ +WARN_UNUSED really_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst) { + src++; while (1) { - parse_string_helper helper = find_bs_bits_and_quote_bits(src, dst); - if (((helper.bs_bits - 1) & helper.quote_bits) != 0) { - /* we encountered quotes first. Move dst to point to quotes and exit - */ - - /* find out where the quote is... */ - auto quote_dist = trailing_zeroes(helper.quote_bits); - - return dst + quote_dist; + // Copy the next n bytes, and find the backslash and quote in them. + auto bs_quote = backslash_and_quote::copy_and_find(src, dst); + // If the next thing is the end quote, copy and return + if (bs_quote.has_quote_first()) { + // we encountered quotes first. Move dst to point to quotes and exit + return dst + bs_quote.quote_index(); } - if (((helper.quote_bits - 1) & helper.bs_bits) != 0) { + if (bs_quote.has_backslash()) { /* find out where the backspace is */ - auto bs_dist = trailing_zeroes(helper.bs_bits); + auto bs_dist = bs_quote.backslash_index(); uint8_t escape_char = src[bs_dist + 1]; /* we encountered backslash first. Handle backslash */ if (escape_char == 'u') { @@ -7170,8 +9835,8 @@ WARN_UNUSED really_inline uint8_t *parse_string(const uint8_t *buf, } else { /* they are the same. Since they can't co-occur, it means we * encountered neither. */ - src += parse_string_helper::BYTES_PROCESSED; - dst += parse_string_helper::BYTES_PROCESSED; + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; } } /* can't be reached */ @@ -7184,8 +9849,6 @@ WARN_UNUSED really_inline uint8_t *parse_string(const uint8_t *buf, } // namespace simdjson::haswell UNTARGET_REGION -#endif // IS_X86_64 - #endif // SIMDJSON_HASWELL_STRINGPARSING_H /* end file src/generic/stringparsing.h */ /* begin file src/haswell/numberparsing.h */ @@ -7193,8 +9856,6 @@ UNTARGET_REGION #define SIMDJSON_HASWELL_NUMBERPARSING_H -#ifdef IS_X86_64 - /* jsoncharutils.h already included: #include "jsoncharutils.h" */ /* haswell/intrinsics.h already included: #include "haswell/intrinsics.h" */ /* haswell/bitmanipulation.h already included: #include "haswell/bitmanipulation.h" */ @@ -7233,81 +9894,214 @@ static inline uint32_t parse_eight_digits_unrolled(const char *chars) { /* begin file src/generic/numberparsing.h */ namespace numberparsing { -// Allowable floating-point values range -// std::numeric_limits::lowest() to std::numeric_limits::max(), -// so from -1.7976e308 all the way to 1.7975e308 in binary64. The lowest -// non-zero normal values is std::numeric_limits::min() or -// about 2.225074e-308. -static const double power_of_ten[] = { - 1e-308, 1e-307, 1e-306, 1e-305, 1e-304, 1e-303, 1e-302, 1e-301, 1e-300, - 1e-299, 1e-298, 1e-297, 1e-296, 1e-295, 1e-294, 1e-293, 1e-292, 1e-291, - 1e-290, 1e-289, 1e-288, 1e-287, 1e-286, 1e-285, 1e-284, 1e-283, 1e-282, - 1e-281, 1e-280, 1e-279, 1e-278, 1e-277, 1e-276, 1e-275, 1e-274, 1e-273, - 1e-272, 1e-271, 1e-270, 1e-269, 1e-268, 1e-267, 1e-266, 1e-265, 1e-264, - 1e-263, 1e-262, 1e-261, 1e-260, 1e-259, 1e-258, 1e-257, 1e-256, 1e-255, - 1e-254, 1e-253, 1e-252, 1e-251, 1e-250, 1e-249, 1e-248, 1e-247, 1e-246, - 1e-245, 1e-244, 1e-243, 1e-242, 1e-241, 1e-240, 1e-239, 1e-238, 1e-237, - 1e-236, 1e-235, 1e-234, 1e-233, 1e-232, 1e-231, 1e-230, 1e-229, 1e-228, - 1e-227, 1e-226, 1e-225, 1e-224, 1e-223, 1e-222, 1e-221, 1e-220, 1e-219, - 1e-218, 1e-217, 1e-216, 1e-215, 1e-214, 1e-213, 1e-212, 1e-211, 1e-210, - 1e-209, 1e-208, 1e-207, 1e-206, 1e-205, 1e-204, 1e-203, 1e-202, 1e-201, - 1e-200, 1e-199, 1e-198, 1e-197, 1e-196, 1e-195, 1e-194, 1e-193, 1e-192, - 1e-191, 1e-190, 1e-189, 1e-188, 1e-187, 1e-186, 1e-185, 1e-184, 1e-183, - 1e-182, 1e-181, 1e-180, 1e-179, 1e-178, 1e-177, 1e-176, 1e-175, 1e-174, - 1e-173, 1e-172, 1e-171, 1e-170, 1e-169, 1e-168, 1e-167, 1e-166, 1e-165, - 1e-164, 1e-163, 1e-162, 1e-161, 1e-160, 1e-159, 1e-158, 1e-157, 1e-156, - 1e-155, 1e-154, 1e-153, 1e-152, 1e-151, 1e-150, 1e-149, 1e-148, 1e-147, - 1e-146, 1e-145, 1e-144, 1e-143, 1e-142, 1e-141, 1e-140, 1e-139, 1e-138, - 1e-137, 1e-136, 1e-135, 1e-134, 1e-133, 1e-132, 1e-131, 1e-130, 1e-129, - 1e-128, 1e-127, 1e-126, 1e-125, 1e-124, 1e-123, 1e-122, 1e-121, 1e-120, - 1e-119, 1e-118, 1e-117, 1e-116, 1e-115, 1e-114, 1e-113, 1e-112, 1e-111, - 1e-110, 1e-109, 1e-108, 1e-107, 1e-106, 1e-105, 1e-104, 1e-103, 1e-102, - 1e-101, 1e-100, 1e-99, 1e-98, 1e-97, 1e-96, 1e-95, 1e-94, 1e-93, - 1e-92, 1e-91, 1e-90, 1e-89, 1e-88, 1e-87, 1e-86, 1e-85, 1e-84, - 1e-83, 1e-82, 1e-81, 1e-80, 1e-79, 1e-78, 1e-77, 1e-76, 1e-75, - 1e-74, 1e-73, 1e-72, 1e-71, 1e-70, 1e-69, 1e-68, 1e-67, 1e-66, - 1e-65, 1e-64, 1e-63, 1e-62, 1e-61, 1e-60, 1e-59, 1e-58, 1e-57, - 1e-56, 1e-55, 1e-54, 1e-53, 1e-52, 1e-51, 1e-50, 1e-49, 1e-48, - 1e-47, 1e-46, 1e-45, 1e-44, 1e-43, 1e-42, 1e-41, 1e-40, 1e-39, - 1e-38, 1e-37, 1e-36, 1e-35, 1e-34, 1e-33, 1e-32, 1e-31, 1e-30, - 1e-29, 1e-28, 1e-27, 1e-26, 1e-25, 1e-24, 1e-23, 1e-22, 1e-21, - 1e-20, 1e-19, 1e-18, 1e-17, 1e-16, 1e-15, 1e-14, 1e-13, 1e-12, - 1e-11, 1e-10, 1e-9, 1e-8, 1e-7, 1e-6, 1e-5, 1e-4, 1e-3, - 1e-2, 1e-1, 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, - 1e7, 1e8, 1e9, 1e10, 1e11, 1e12, 1e13, 1e14, 1e15, - 1e16, 1e17, 1e18, 1e19, 1e20, 1e21, 1e22, 1e23, 1e24, - 1e25, 1e26, 1e27, 1e28, 1e29, 1e30, 1e31, 1e32, 1e33, - 1e34, 1e35, 1e36, 1e37, 1e38, 1e39, 1e40, 1e41, 1e42, - 1e43, 1e44, 1e45, 1e46, 1e47, 1e48, 1e49, 1e50, 1e51, - 1e52, 1e53, 1e54, 1e55, 1e56, 1e57, 1e58, 1e59, 1e60, - 1e61, 1e62, 1e63, 1e64, 1e65, 1e66, 1e67, 1e68, 1e69, - 1e70, 1e71, 1e72, 1e73, 1e74, 1e75, 1e76, 1e77, 1e78, - 1e79, 1e80, 1e81, 1e82, 1e83, 1e84, 1e85, 1e86, 1e87, - 1e88, 1e89, 1e90, 1e91, 1e92, 1e93, 1e94, 1e95, 1e96, - 1e97, 1e98, 1e99, 1e100, 1e101, 1e102, 1e103, 1e104, 1e105, - 1e106, 1e107, 1e108, 1e109, 1e110, 1e111, 1e112, 1e113, 1e114, - 1e115, 1e116, 1e117, 1e118, 1e119, 1e120, 1e121, 1e122, 1e123, - 1e124, 1e125, 1e126, 1e127, 1e128, 1e129, 1e130, 1e131, 1e132, - 1e133, 1e134, 1e135, 1e136, 1e137, 1e138, 1e139, 1e140, 1e141, - 1e142, 1e143, 1e144, 1e145, 1e146, 1e147, 1e148, 1e149, 1e150, - 1e151, 1e152, 1e153, 1e154, 1e155, 1e156, 1e157, 1e158, 1e159, - 1e160, 1e161, 1e162, 1e163, 1e164, 1e165, 1e166, 1e167, 1e168, - 1e169, 1e170, 1e171, 1e172, 1e173, 1e174, 1e175, 1e176, 1e177, - 1e178, 1e179, 1e180, 1e181, 1e182, 1e183, 1e184, 1e185, 1e186, - 1e187, 1e188, 1e189, 1e190, 1e191, 1e192, 1e193, 1e194, 1e195, - 1e196, 1e197, 1e198, 1e199, 1e200, 1e201, 1e202, 1e203, 1e204, - 1e205, 1e206, 1e207, 1e208, 1e209, 1e210, 1e211, 1e212, 1e213, - 1e214, 1e215, 1e216, 1e217, 1e218, 1e219, 1e220, 1e221, 1e222, - 1e223, 1e224, 1e225, 1e226, 1e227, 1e228, 1e229, 1e230, 1e231, - 1e232, 1e233, 1e234, 1e235, 1e236, 1e237, 1e238, 1e239, 1e240, - 1e241, 1e242, 1e243, 1e244, 1e245, 1e246, 1e247, 1e248, 1e249, - 1e250, 1e251, 1e252, 1e253, 1e254, 1e255, 1e256, 1e257, 1e258, - 1e259, 1e260, 1e261, 1e262, 1e263, 1e264, 1e265, 1e266, 1e267, - 1e268, 1e269, 1e270, 1e271, 1e272, 1e273, 1e274, 1e275, 1e276, - 1e277, 1e278, 1e279, 1e280, 1e281, 1e282, 1e283, 1e284, 1e285, - 1e286, 1e287, 1e288, 1e289, 1e290, 1e291, 1e292, 1e293, 1e294, - 1e295, 1e296, 1e297, 1e298, 1e299, 1e300, 1e301, 1e302, 1e303, - 1e304, 1e305, 1e306, 1e307, 1e308}; + +// Attempts to compute i * 10^(power) exactly; and if "negative" is +// true, negate the result. +// This function will only work in some cases, when it does not work, success is +// set to false. This should work *most of the time* (like 99% of the time). +// We assume that power is in the [FASTFLOAT_SMALLEST_POWER, +// FASTFLOAT_LARGEST_POWER] interval: the caller is responsible for this check. +really_inline double compute_float_64(int64_t power, uint64_t i, bool negative, + bool *success) { + // we start with a fast path + // It was described in + // Clinger WD. How to read floating point numbers accurately. + // ACM SIGPLAN Notices. 1990 + if (-22 <= power && power <= 22 && i <= 9007199254740991) { + // convert the integer into a double. This is lossless since + // 0 <= i <= 2^53 - 1. + double d = i; + // + // The general idea is as follows. + // If 0 <= s < 2^53 and if 10^0 <= p <= 10^22 then + // 1) Both s and p can be represented exactly as 64-bit floating-point + // values + // (binary64). + // 2) Because s and p can be represented exactly as floating-point values, + // then s * p + // and s / p will produce correctly rounded values. + // + if (power < 0) { + d = d / power_of_ten[-power]; + } else { + d = d * power_of_ten[power]; + } + if (negative) { + d = -d; + } + *success = true; + return d; + } + // When 22 < power && power < 22 + 16, we could + // hope for another, secondary fast path. It wa + // described by David M. Gay in "Correctly rounded + // binary-decimal and decimal-binary conversions." (1990) + // If you need to compute i * 10^(22 + x) for x < 16, + // first compute i * 10^x, if you know that result is exact + // (e.g., when i * 10^x < 2^53), + // then you can still proceed and do (i * 10^x) * 10^22. + // Is this worth your time? + // You need 22 < power *and* power < 22 + 16 *and* (i * 10^(x-22) < 2^53) + // for this second fast path to work. + // If you you have 22 < power *and* power < 22 + 16, and then you + // optimistically compute "i * 10^(x-22)", there is still a chance that you + // have wasted your time if i * 10^(x-22) >= 2^53. It makes the use cases of + // this optimization maybe less common than we would like. Source: + // http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ + // also used in RapidJSON: https://rapidjson.org/strtod_8h_source.html + + // The fast path has now failed, so we are failing back on the slower path. + + // In the slow path, we need to adjust i so that it is > 1<<63 which is always + // possible, except if i == 0, so we handle i == 0 separately. + if(i == 0) { + return 0.0; + } + + // We are going to need to do some 64-bit arithmetic to get a more precise product. + // We use a table lookup approach. + components c = + power_of_ten_components[power - FASTFLOAT_SMALLEST_POWER]; + // safe because + // power >= FASTFLOAT_SMALLEST_POWER + // and power <= FASTFLOAT_LARGEST_POWER + // we recover the mantissa of the power, it has a leading 1. It is always + // rounded down. + uint64_t factor_mantissa = c.mantissa; + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(i); + i <<= lz; + // We want the most significant 64 bits of the product. We know + // this will be non-zero because the most significant bit of i is + // 1. + value128 product = full_multiplication(i, factor_mantissa); + uint64_t lower = product.low; + uint64_t upper = product.high; + + // We know that upper has at most one leading zero because + // both i and factor_mantissa have a leading one. This means + // that the result is at least as large as ((1<<63)*(1<<63))/(1<<64). + + // As long as the first 9 bits of "upper" are not "1", then we + // know that we have an exact computed value for the leading + // 55 bits because any imprecision would play out as a +1, in + // the worst case. + if (unlikely((upper & 0x1FF) == 0x1FF) && (lower + i < lower)) { + uint64_t factor_mantissa_low = + mantissa_128[power - FASTFLOAT_SMALLEST_POWER]; + // next, we compute the 64-bit x 128-bit multiplication, getting a 192-bit + // result (three 64-bit values) + product = full_multiplication(i, factor_mantissa_low); + uint64_t product_low = product.low; + uint64_t product_middle2 = product.high; + uint64_t product_middle1 = lower; + uint64_t product_high = upper; + uint64_t product_middle = product_middle1 + product_middle2; + if (product_middle < product_middle1) { + product_high++; // overflow carry + } + // We want to check whether mantissa *i + i would affect our result. + // This does happen, e.g. with 7.3177701707893310e+15. + if (((product_middle + 1 == 0) && ((product_high & 0x1FF) == 0x1FF) && + (product_low + i < product_low))) { // let us be prudent and bail out. + *success = false; + return 0; + } + upper = product_high; + lower = product_middle; + } + // The final mantissa should be 53 bits with a leading 1. + // We shift it so that it occupies 54 bits with a leading 1. + /////// + uint64_t upperbit = upper >> 63; + uint64_t mantissa = upper >> (upperbit + 9); + lz += 1 ^ upperbit; + + // Here we have mantissa < (1<<54). + + // We have to round to even. The "to even" part + // is only a problem when we are right in between two floats + // which we guard against. + // If we have lots of trailing zeros, we may fall right between two + // floating-point values. + if (unlikely((lower == 0) && ((upper & 0x1FF) == 0) && + ((mantissa & 3) == 1))) { + // if mantissa & 1 == 1 we might need to round up. + // + // Scenarios: + // 1. We are not in the middle. Then we should round up. + // + // 2. We are right in the middle. Whether we round up depends + // on the last significant bit: if it is "one" then we round + // up (round to even) otherwise, we do not. + // + // So if the last significant bit is 1, we can safely round up. + // Hence we only need to bail out if (mantissa & 3) == 1. + // Otherwise we may need more accuracy or analysis to determine whether + // we are exactly between two floating-point numbers. + // It can be triggered with 1e23. + // Note: because the factor_mantissa and factor_mantissa_low are + // almost always rounded down (except for small positive powers), + // almost always should round up. + *success = false; + return 0; + } + + mantissa += mantissa & 1; + mantissa >>= 1; + + // Here we have mantissa < (1<<53), unless there was an overflow + if (mantissa >= (1ULL << 53)) { + ////////// + // This will happen when parsing values such as 7.2057594037927933e+16 + //////// + mantissa = (1ULL << 52); + lz--; // undo previous addition + } + mantissa &= ~(1ULL << 52); + uint64_t real_exponent = c.exp - lz; + // we have to check that real_exponent is in range, otherwise we bail out + if (unlikely((real_exponent < 1) || (real_exponent > 2046))) { + *success = false; + return 0; + } + mantissa |= real_exponent << 52; + mantissa |= (((uint64_t)negative) << 63); + double d; + memcpy(&d, &mantissa, sizeof(d)); + *success = true; + return d; +} + +static bool parse_float_strtod(const char *ptr, double *outDouble) { + char *endptr; + *outDouble = strtod(ptr, &endptr); + // Some libraries will set errno = ERANGE when the value is subnormal, + // yet we may want to be able to parse subnormal values. + // However, we do not want to tolerate NAN or infinite values. + // + // Values like infinity or NaN are not allowed in the JSON specification. + // If you consume a large value and you map it to "infinity", you will no + // longer be able to serialize back a standard-compliant JSON. And there is + // no realistic application where you might need values so large than they + // can't fit in binary64. The maximal value is about 1.7976931348623157 × + // 10^308 It is an unimaginable large number. There will never be any piece of + // engineering involving as many as 10^308 parts. It is estimated that there + // are about 10^80 atoms in the universe.  The estimate for the total number + // of electrons is similar. Using a double-precision floating-point value, we + // can represent easily the number of atoms in the universe. We could also + // represent the number of ways you can pick any three individual atoms at + // random in the universe. If you ever encounter a number much larger than + // 10^308, you know that you have a bug. RapidJSON will reject a document with + // a float that does not fit in binary64. JSON for Modern C++ (nlohmann/json) + // will flat out throw an exception. + // + if ((endptr == ptr) || (!std::isfinite(*outDouble))) { + return false; + } + return true; +} really_inline bool is_integer(char c) { return (c >= '0' && c <= '9'); @@ -7315,7 +10109,7 @@ really_inline bool is_integer(char c) { } // We need to check that the character following a zero is valid. This is -// probably frequent and it is hard than it looks. We are building all of this +// probably frequent and it is harder than it looks. We are building all of this // just to differentiate between 0x1 (invalid), 0,1 (valid) 0e1 (valid)... const bool structural_or_whitespace_or_exponent_or_decimal_negated[256] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -7353,162 +10147,6 @@ really_inline bool is_made_of_eight_digits_fast(const char *chars) { 0x3333333333333333); } - -// -// This function computes base * 10 ^ (- negative_exponent ). -// It is only even going to be used when negative_exponent is tiny. -really_inline double subnormal_power10(double base, int64_t negative_exponent) { - // avoid integer overflows in the pow expression, those values would - // become zero anyway. - if(negative_exponent < -1000) { - return 0; - } - - // this is probably not going to be fast - return base * 1e-308 * pow(10, negative_exponent + 308); -} - -// called by parse_number when we know that the output is a float, -// but where there might be some integer overflow. The trick here is to -// parse using floats from the start. -// Do not call this function directly as it skips some of the checks from -// parse_number -// -// This function will almost never be called!!! -// -// Note: a redesign could avoid this function entirely. -// -never_inline bool parse_float(const uint8_t *const buf, document::parser &parser, - const uint32_t offset, bool found_minus) { - const char *p = reinterpret_cast(buf + offset); - bool negative = false; - if (found_minus) { - ++p; - negative = true; - } - long double i; - if (*p == '0') { // 0 cannot be followed by an integer - ++p; - i = 0; - } else { - unsigned char digit = *p - '0'; - i = digit; - p++; - while (is_integer(*p)) { - digit = *p - '0'; - i = 10 * i + digit; - ++p; - } - } - if ('.' == *p) { - ++p; - int fractional_weight = 308; - if (is_integer(*p)) { - unsigned char digit = *p - '0'; - ++p; - - fractional_weight--; - i = i + digit * (fractional_weight >= 0 ? power_of_ten[fractional_weight] - : 0); - } else { -#ifdef JSON_TEST_NUMBERS // for unit testing - found_invalid_number(buf + offset); -#endif - return false; - } - while (is_integer(*p)) { - unsigned char digit = *p - '0'; - ++p; - fractional_weight--; - i = i + digit * (fractional_weight >= 0 ? power_of_ten[fractional_weight] - : 0); - } - } - if (('e' == *p) || ('E' == *p)) { - ++p; - bool neg_exp = false; - if ('-' == *p) { - neg_exp = true; - ++p; - } else if ('+' == *p) { - ++p; - } - if (!is_integer(*p)) { -#ifdef JSON_TEST_NUMBERS // for unit testing - found_invalid_number(buf + offset); -#endif - return false; - } - unsigned char digit = *p - '0'; - int64_t exp_number = digit; // exponential part - p++; - if (is_integer(*p)) { - digit = *p - '0'; - exp_number = 10 * exp_number + digit; - ++p; - } - if (is_integer(*p)) { - digit = *p - '0'; - exp_number = 10 * exp_number + digit; - ++p; - } - if (is_integer(*p)) { - digit = *p - '0'; - exp_number = 10 * exp_number + digit; - ++p; - } - while (is_integer(*p)) { - if (exp_number > 0x100000000) { // we need to check for overflows -// we refuse to parse this -#ifdef JSON_TEST_NUMBERS // for unit testing - found_invalid_number(buf + offset); -#endif - return false; - } - digit = *p - '0'; - exp_number = 10 * exp_number + digit; - ++p; - } - if (unlikely(exp_number > 308)) { - // this path is unlikely - if (neg_exp) { - // We either have zero or a subnormal. - // We expect this to be uncommon so we go through a slow path. - i = subnormal_power10(i, -exp_number); - } else { -// We know for sure that we have a number that is too large, -// we refuse to parse this -#ifdef JSON_TEST_NUMBERS // for unit testing - found_invalid_number(buf + offset); -#endif - return false; - } - } else { - int exponent = (neg_exp ? -exp_number : exp_number); - // we have that exp_number is [0,308] so that - // exponent is [-308,308] so that - // 308 + exponent is in [0, 2 * 308] - i *= power_of_ten[308 + exponent]; - } - } - if (is_not_structural_or_whitespace(*p)) { - return false; - } - // check that we can go from long double to double safely. - if(i > std::numeric_limits::max()) { -#ifdef JSON_TEST_NUMBERS // for unit testing - found_invalid_number(buf + offset); -#endif - return false; - } - double d = negative ? -i : i; - parser.on_number_double(d); -#ifdef JSON_TEST_NUMBERS // for unit testing - found_float(d, buf + offset); -#endif - return is_structural_or_whitespace(*p); -} - // called by parse_number when we know that the output is an integer, // but where there might be some integer overflow. // we want to catch overflows! @@ -7517,11 +10155,10 @@ never_inline bool parse_float(const uint8_t *const buf, document::parser &parser // // This function will almost never be called!!! // -never_inline bool parse_large_integer(const uint8_t *const buf, - document::parser &parser, - const uint32_t offset, - bool found_minus) { - const char *p = reinterpret_cast(buf + offset); +never_inline bool parse_large_integer(const uint8_t *const src, + document::parser &parser, + bool found_minus) { + const char *p = reinterpret_cast(src); bool negative = false; if (found_minus) { @@ -7542,13 +10179,13 @@ never_inline bool parse_large_integer(const uint8_t *const buf, digit = *p - '0'; if (mul_overflow(i, 10, &i)) { #ifdef JSON_TEST_NUMBERS // for unit testing - found_invalid_number(buf + offset); + found_invalid_number(src); #endif return false; // overflow } if (add_overflow(i, digit, &i)) { #ifdef JSON_TEST_NUMBERS // for unit testing - found_invalid_number(buf + offset); + found_invalid_number(src); #endif return false; // overflow } @@ -7557,40 +10194,40 @@ never_inline bool parse_large_integer(const uint8_t *const buf, } if (negative) { if (i > 0x8000000000000000) { - // overflows! + // overflows! #ifdef JSON_TEST_NUMBERS // for unit testing - found_invalid_number(buf + offset); + found_invalid_number(src); #endif return false; // overflow } else if (i == 0x8000000000000000) { // In two's complement, we cannot represent 0x8000000000000000 - // as a positive signed integer, but the negative version is + // as a positive signed integer, but the negative version is // possible. constexpr int64_t signed_answer = INT64_MIN; parser.on_number_s64(signed_answer); #ifdef JSON_TEST_NUMBERS // for unit testing - found_integer(signed_answer, buf + offset); + found_integer(signed_answer, src); #endif } else { // we can negate safely int64_t signed_answer = -static_cast(i); parser.on_number_s64(signed_answer); #ifdef JSON_TEST_NUMBERS // for unit testing - found_integer(signed_answer, buf + offset); + found_integer(signed_answer, src); #endif } } else { // we have a positive integer, the contract is that - // we try to represent it as a signed integer and only + // we try to represent it as a signed integer and only // fallback on unsigned integers if absolutely necessary. - if(i < 0x8000000000000000) { + if (i < 0x8000000000000000) { #ifdef JSON_TEST_NUMBERS // for unit testing - found_integer(i, buf + offset); + found_integer(i, src); #endif parser.on_number_s64(i); } else { #ifdef JSON_TEST_NUMBERS // for unit testing - found_unsigned_integer(i, buf + offset); + found_unsigned_integer(i, src); #endif parser.on_number_u64(i); } @@ -7598,7 +10235,22 @@ never_inline bool parse_large_integer(const uint8_t *const buf, return is_structural_or_whitespace(*p); } -// parse the number at buf + offset +bool slow_float_parsing(UNUSED const char * src, document::parser &parser) { + double d; + if (parse_float_strtod(src, &d)) { + parser.on_number_double(d); +#ifdef JSON_TEST_NUMBERS // for unit testing + found_float(d, (const uint8_t *)src); +#endif + return true; + } +#ifdef JSON_TEST_NUMBERS // for unit testing + found_invalid_number((const uint8_t *)src); +#endif + return false; +} + +// parse the number at src // define JSON_TEST_NUMBERS for unit testing // // It is assumed that the number is followed by a structural ({,},],[) character @@ -7606,24 +10258,23 @@ never_inline bool parse_large_integer(const uint8_t *const buf, // document is made of a single number), then it is necessary to copy the // content and append a space before calling this function. // -// Our objective is accurate parsing (ULP of 0 or 1) at high speed. -really_inline bool parse_number(UNUSED const uint8_t *const buf, - UNUSED const uint32_t offset, +// Our objective is accurate parsing (ULP of 0) at high speed. +really_inline bool parse_number(UNUSED const uint8_t *const src, UNUSED bool found_minus, document::parser &parser) { #ifdef SIMDJSON_SKIPNUMBERPARSING // for performance analysis, it is sometimes // useful to skip parsing - parser.on_number_s64(0); // always write zero + parser.on_number_s64(0); // always write zero return true; // always succeeds #else - const char *p = reinterpret_cast(buf + offset); + const char *p = reinterpret_cast(src); bool negative = false; if (found_minus) { ++p; negative = true; if (!is_integer(*p)) { // a negative sign must be followed by an integer #ifdef JSON_TEST_NUMBERS // for unit testing - found_invalid_number(buf + offset); + found_invalid_number(src); #endif return false; } @@ -7635,7 +10286,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const buf, ++p; if (is_not_structural_or_whitespace_or_exponent_or_decimal(*p)) { #ifdef JSON_TEST_NUMBERS // for unit testing - found_invalid_number(buf + offset); + found_invalid_number(src); #endif return false; } @@ -7643,7 +10294,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const buf, } else { if (!(is_integer(*p))) { // must start with an integer #ifdef JSON_TEST_NUMBERS // for unit testing - found_invalid_number(buf + offset); + found_invalid_number(src); #endif return false; } @@ -7678,7 +10329,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const buf, // we will handle the overflow later } else { #ifdef JSON_TEST_NUMBERS // for unit testing - found_invalid_number(buf + offset); + found_invalid_number(src); #endif return false; } @@ -7713,7 +10364,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const buf, } if (!is_integer(*p)) { #ifdef JSON_TEST_NUMBERS // for unit testing - found_invalid_number(buf + offset); + found_invalid_number(src); #endif return false; } @@ -7734,7 +10385,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const buf, if (exp_number > 0x100000000) { // we need to check for overflows // we refuse to parse this #ifdef JSON_TEST_NUMBERS // for unit testing - found_invalid_number(buf + offset); + found_invalid_number(src); #endif return false; } @@ -7745,7 +10396,9 @@ really_inline bool parse_number(UNUSED const uint8_t *const buf, exponent += (neg_exp ? -exp_number : exp_number); } if (is_float) { - uint64_t power_index = 308 + exponent; + // If we frequently had to deal with long strings of digits, + // we could extend our code by using a 128-bit integer instead + // of a 64-bit integer. However, this is uncommon in practice. if (unlikely((digit_count >= 19))) { // this is uncommon // It is possible that the integer had an overflow. // We have to handle the case where we have 0.0000somenumber. @@ -7759,31 +10412,47 @@ really_inline bool parse_number(UNUSED const uint8_t *const buf, // Ok, chances are good that we had an overflow! // this is almost never going to get called!!! // we start anew, going slowly!!! - return parse_float(buf, parser, offset, found_minus); + // This will happen in the following examples: + // 10000000000000000000000000000000000000000000e+308 + // 3.1415926535897932384626433832795028841971693993751 + // + return slow_float_parsing((const char *) src, parser); } } - if (unlikely((power_index > 2 * 308))) { // this is uncommon!!! + if (unlikely(exponent < FASTFLOAT_SMALLEST_POWER) || + (exponent > FASTFLOAT_LARGEST_POWER)) { // this is uncommon!!! // this is almost never going to get called!!! // we start anew, going slowly!!! - return parse_float(buf, parser, offset, found_minus); + return slow_float_parsing((const char *) src, parser); } - double factor = power_of_ten[power_index]; - factor = negative ? -factor : factor; - double d = i * factor; - parser.on_number_double(d); + bool success = true; + double d = compute_float_64(exponent, i, negative, &success); + if (!success) { + // we are almost never going to get here. + success = parse_float_strtod((const char *)src, &d); + } + if (success) { + parser.on_number_double(d); #ifdef JSON_TEST_NUMBERS // for unit testing - found_float(d, buf + offset); + found_float(d, src); #endif + return true; + } else { +#ifdef JSON_TEST_NUMBERS // for unit testing + found_invalid_number(src); +#endif + return false; + } } else { if (unlikely(digit_count >= 18)) { // this is uncommon!!! // there is a good chance that we had an overflow, so we need // need to recover: we parse the whole thing again. - return parse_large_integer(buf, parser, offset, found_minus); + return parse_large_integer(src, parser, found_minus); } i = negative ? 0 - i : i; parser.on_number_s64(i); #ifdef JSON_TEST_NUMBERS // for unit testing - found_integer(i, buf + offset); + found_integer(i, src); #endif } return is_structural_or_whitespace(*p); @@ -7796,14 +10465,63 @@ really_inline bool parse_number(UNUSED const uint8_t *const buf, } // namespace simdjson::haswell UNTARGET_REGION -#endif // IS_X86_64 - #endif // SIMDJSON_HASWELL_NUMBERPARSING_H /* end file src/generic/numberparsing.h */ TARGET_HASWELL namespace simdjson::haswell { +/* begin file src/generic/atomparsing.h */ +namespace atomparsing { + +really_inline uint32_t string_to_uint32(const char* str) { return *reinterpret_cast(str); } + +WARN_UNUSED +really_inline bool str4ncmp(const uint8_t *src, const char* atom) { + uint32_t srcval; // we want to avoid unaligned 64-bit loads (undefined in C/C++) + static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING); + std::memcpy(&srcval, src, sizeof(uint32_t)); + return srcval ^ string_to_uint32(atom); +} + +WARN_UNUSED +really_inline bool is_valid_true_atom(const uint8_t *src) { + return (str4ncmp(src, "true") | is_not_structural_or_whitespace(src[4])) == 0; +} + +WARN_UNUSED +really_inline bool is_valid_true_atom(const uint8_t *src, size_t len) { + if (len > 4) { return is_valid_true_atom(src); } + else if (len == 4) { return !str4ncmp(src, "true"); } + else { return false; } +} + +WARN_UNUSED +really_inline bool is_valid_false_atom(const uint8_t *src) { + return (str4ncmp(src+1, "alse") | is_not_structural_or_whitespace(src[5])) == 0; +} + +WARN_UNUSED +really_inline bool is_valid_false_atom(const uint8_t *src, size_t len) { + if (len > 5) { return is_valid_false_atom(src); } + else if (len == 5) { return !str4ncmp(src+1, "alse"); } + else { return false; } +} + +WARN_UNUSED +really_inline bool is_valid_null_atom(const uint8_t *src) { + return (str4ncmp(src, "null") | is_not_structural_or_whitespace(src[4])) == 0; +} + +WARN_UNUSED +really_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { + if (len > 4) { return is_valid_null_atom(src); } + else if (len == 4) { return !str4ncmp(src, "null"); } + else { return false; } +} + +} // namespace atomparsing +/* end file src/generic/atomparsing.h */ /* begin file src/generic/stage2_build_tape.h */ // This file contains the common code every implementation uses for stage2 // It is intended to be included multiple times and compiled multiple times @@ -7854,28 +10572,25 @@ struct unified_machine_addresses { #undef FAIL_IF #define FAIL_IF(EXPR) { if (EXPR) { return addresses.error; } } -struct structural_parser { - const uint8_t* const buf; - const size_t len; - document::parser &doc_parser; - size_t i; // next structural index - size_t idx; // location of the structural character in the input (buf) - uint8_t c; // used to track the (structural) character we are looking at - uint32_t depth = 0; // could have an arbitrary starting depth - - really_inline structural_parser( - const uint8_t *_buf, - size_t _len, - document::parser &_doc_parser, - uint32_t _i = 0 - ) : buf{_buf}, len{_len}, doc_parser{_doc_parser}, i{_i} {} - +class structural_iterator { +public: + really_inline structural_iterator(const uint8_t* _buf, size_t _len, const uint32_t *_structural_indexes, size_t next_structural_index) + : buf{_buf}, len{_len}, structural_indexes{_structural_indexes}, next_structural{next_structural_index} {} really_inline char advance_char() { - idx = doc_parser.structural_indexes[i++]; - c = buf[idx]; + idx = structural_indexes[next_structural]; + next_structural++; + c = *current(); return c; } - + really_inline char current_char() { + return c; + } + really_inline const uint8_t* current() { + return &buf[idx]; + } + really_inline size_t remaining_len() { + return len - idx; + } template really_inline bool with_space_terminated_copy(const F& f) { /** @@ -7901,6 +10616,35 @@ struct structural_parser { free(copy); return result; } + really_inline bool past_end(uint32_t n_structural_indexes) { + return next_structural+1 > n_structural_indexes; + } + really_inline bool at_end(uint32_t n_structural_indexes) { + return next_structural+1 == n_structural_indexes; + } + really_inline size_t next_structural_index() { + return next_structural; + } + + const uint8_t* const buf; + const size_t len; + const uint32_t* const structural_indexes; + size_t next_structural; // next structural index + size_t idx; // location of the structural character in the input (buf) + uint8_t c; // used to track the (structural) character we are looking at +}; + +struct structural_parser { + structural_iterator structurals; + document::parser &doc_parser; + uint32_t depth; + + really_inline structural_parser( + const uint8_t *buf, + size_t len, + document::parser &_doc_parser, + uint32_t next_structural = 0 + ) : structurals(buf, len, _doc_parser.structural_indexes.get(), next_structural), doc_parser{_doc_parser}, depth{0} {} WARN_UNUSED really_inline bool start_document(ret_address continue_state) { doc_parser.on_start_document(depth); @@ -7941,32 +10685,32 @@ struct structural_parser { WARN_UNUSED really_inline bool parse_string() { uint8_t *dst = doc_parser.on_start_string(); - dst = stringparsing::parse_string(buf, idx, dst); + dst = stringparsing::parse_string(structurals.current(), dst); if (dst == nullptr) { return true; } return !doc_parser.on_end_string(dst); } - WARN_UNUSED really_inline bool parse_number(const uint8_t *copy, uint32_t offset, bool found_minus) { - return !numberparsing::parse_number(copy, offset, found_minus, doc_parser); + WARN_UNUSED really_inline bool parse_number(const uint8_t *src, bool found_minus) { + return !numberparsing::parse_number(src, found_minus, doc_parser); } WARN_UNUSED really_inline bool parse_number(bool found_minus) { - return parse_number(buf, idx, found_minus); + return parse_number(structurals.current(), found_minus); } - WARN_UNUSED really_inline bool parse_atom(const uint8_t *copy, uint32_t offset) { - switch (c) { + WARN_UNUSED really_inline bool parse_atom() { + switch (structurals.current_char()) { case 't': - if (!is_valid_true_atom(copy + offset)) { return true; } + if (!atomparsing::is_valid_true_atom(structurals.current())) { return true; } doc_parser.on_true_atom(); break; case 'f': - if (!is_valid_false_atom(copy + offset)) { return true; } + if (!atomparsing::is_valid_false_atom(structurals.current())) { return true; } doc_parser.on_false_atom(); break; case 'n': - if (!is_valid_null_atom(copy + offset)) { return true; } + if (!atomparsing::is_valid_null_atom(structurals.current())) { return true; } doc_parser.on_null_atom(); break; default: @@ -7975,12 +10719,28 @@ struct structural_parser { return false; } - WARN_UNUSED really_inline bool parse_atom() { - return parse_atom(buf, idx); + WARN_UNUSED really_inline bool parse_single_atom() { + switch (structurals.current_char()) { + case 't': + if (!atomparsing::is_valid_true_atom(structurals.current(), structurals.remaining_len())) { return true; } + doc_parser.on_true_atom(); + break; + case 'f': + if (!atomparsing::is_valid_false_atom(structurals.current(), structurals.remaining_len())) { return true; } + doc_parser.on_false_atom(); + break; + case 'n': + if (!atomparsing::is_valid_null_atom(structurals.current(), structurals.remaining_len())) { return true; } + doc_parser.on_null_atom(); + break; + default: + return true; + } + return false; } WARN_UNUSED really_inline ret_address parse_value(const unified_machine_addresses &addresses, ret_address continue_state) { - switch (c) { + switch (structurals.current_char()) { case '"': FAIL_IF( parse_string() ); return continue_state; @@ -8007,7 +10767,7 @@ struct structural_parser { WARN_UNUSED really_inline error_code finish() { // the string might not be NULL terminated. - if ( i + 1 != doc_parser.n_structural_indexes ) { + if ( !structurals.at_end(doc_parser.n_structural_indexes) ) { return doc_parser.on_error(TAPE_ERROR); } end_document(); @@ -8035,7 +10795,7 @@ struct structural_parser { if (depth >= doc_parser.max_depth()) { return doc_parser.on_error(DEPTH_ERROR); } - switch (c) { + switch (structurals.current_char()) { case '"': return doc_parser.on_error(STRING_ERROR); case '0': @@ -8061,19 +10821,23 @@ struct structural_parser { } } - WARN_UNUSED really_inline error_code start(ret_address finish_state) { + WARN_UNUSED really_inline error_code start(size_t len, ret_address finish_state) { doc_parser.init_stage2(); // sets is_valid to false if (len > doc_parser.capacity()) { return CAPACITY; } // Advance to the first character as soon as possible - advance_char(); + structurals.advance_char(); // Push the root scope (there is always at least one scope) if (start_document(finish_state)) { return doc_parser.on_error(DEPTH_ERROR); } return SUCCESS; } + + really_inline char advance_char() { + return structurals.advance_char(); + } }; // Redefine FAIL_IF to use goto since it'll be used inside the function now @@ -8089,13 +10853,13 @@ struct structural_parser { WARN_UNUSED error_code implementation::stage2(const uint8_t *buf, size_t len, document::parser &doc_parser) const noexcept { static constexpr stage2::unified_machine_addresses addresses = INIT_ADDRESSES(); stage2::structural_parser parser(buf, len, doc_parser); - error_code result = parser.start(addresses.finish); + error_code result = parser.start(len, addresses.finish); if (result) { return result; } // // Read first value // - switch (parser.c) { + switch (parser.structurals.current_char()) { case '{': FAIL_IF( parser.start_object(addresses.finish) ); goto object_begin; @@ -8106,24 +10870,20 @@ WARN_UNUSED error_code implementation::stage2(const uint8_t *buf, size_t len, do FAIL_IF( parser.parse_string() ); goto finish; case 't': case 'f': case 'n': - FAIL_IF( - parser.with_space_terminated_copy([&](auto copy, auto idx) { - return parser.parse_atom(copy, idx); - }) - ); + FAIL_IF( parser.parse_single_atom() ); goto finish; case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': FAIL_IF( - parser.with_space_terminated_copy([&](auto copy, auto idx) { - return parser.parse_number(copy, idx, false); + parser.structurals.with_space_terminated_copy([&](auto copy, auto idx) { + return parser.parse_number(©[idx], false); }) ); goto finish; case '-': FAIL_IF( - parser.with_space_terminated_copy([&](auto copy, auto idx) { - return parser.parse_number(copy, idx, true); + parser.structurals.with_space_terminated_copy([&](auto copy, auto idx) { + return parser.parse_number(©[idx], true); }) ); goto finish; @@ -8135,8 +10895,7 @@ WARN_UNUSED error_code implementation::stage2(const uint8_t *buf, size_t len, do // Object parser states // object_begin: - parser.advance_char(); - switch (parser.c) { + switch (parser.advance_char()) { case '"': { FAIL_IF( parser.parse_string() ); goto object_key_state; @@ -8217,7 +10976,7 @@ struct streaming_structural_parser: structural_parser { really_inline streaming_structural_parser(const uint8_t *_buf, size_t _len, document::parser &_doc_parser, size_t _i) : structural_parser(_buf, _len, _doc_parser, _i) {} // override to add streaming - WARN_UNUSED really_inline error_code start(ret_address finish_parser) { + WARN_UNUSED really_inline error_code start(UNUSED size_t len, ret_address finish_parser) { doc_parser.init_stage2(); // sets is_valid to false // Capacity ain't no thang for streaming, so we don't check it. // Advance to the first character as soon as possible @@ -8231,7 +10990,7 @@ struct streaming_structural_parser: structural_parser { // override to add streaming WARN_UNUSED really_inline error_code finish() { - if ( i + 1 > doc_parser.n_structural_indexes ) { + if ( structurals.past_end(doc_parser.n_structural_indexes) ) { return doc_parser.on_error(TAPE_ERROR); } end_document(); @@ -8241,7 +11000,7 @@ struct streaming_structural_parser: structural_parser { if (doc_parser.containing_scope_offset[depth] != 0) { return doc_parser.on_error(TAPE_ERROR); } - bool finished = i + 1 == doc_parser.n_structural_indexes; + bool finished = structurals.at_end(doc_parser.n_structural_indexes); return doc_parser.on_success(finished ? SUCCESS : SUCCESS_AND_HAS_MORE); } }; @@ -8255,12 +11014,12 @@ struct streaming_structural_parser: structural_parser { WARN_UNUSED error_code implementation::stage2(const uint8_t *buf, size_t len, document::parser &doc_parser, size_t &next_json) const noexcept { static constexpr stage2::unified_machine_addresses addresses = INIT_ADDRESSES(); stage2::streaming_structural_parser parser(buf, len, doc_parser, next_json); - error_code result = parser.start(addresses.finish); + error_code result = parser.start(len, addresses.finish); if (result) { return result; } // // Read first value // - switch (parser.c) { + switch (parser.structurals.current_char()) { case '{': FAIL_IF( parser.start_object(addresses.finish) ); goto object_begin; @@ -8271,24 +11030,20 @@ WARN_UNUSED error_code implementation::stage2(const uint8_t *buf, size_t len, do FAIL_IF( parser.parse_string() ); goto finish; case 't': case 'f': case 'n': - FAIL_IF( - parser.with_space_terminated_copy([&](auto copy, auto idx) { - return parser.parse_atom(copy, idx); - }) - ); + FAIL_IF( parser.parse_single_atom() ); goto finish; case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': FAIL_IF( - parser.with_space_terminated_copy([&](auto copy, auto idx) { - return parser.parse_number(copy, idx, false); + parser.structurals.with_space_terminated_copy([&](auto copy, auto idx) { + return parser.parse_number(©[idx], false); }) ); goto finish; case '-': FAIL_IF( - parser.with_space_terminated_copy([&](auto copy, auto idx) { - return parser.parse_number(copy, idx, true); + parser.structurals.with_space_terminated_copy([&](auto copy, auto idx) { + return parser.parse_number(©[idx], true); }) ); goto finish; @@ -8300,8 +11055,7 @@ WARN_UNUSED error_code implementation::stage2(const uint8_t *buf, size_t len, do // Object parser parsers // object_begin: - parser.advance_char(); - switch (parser.c) { + switch (parser.advance_char()) { case '"': { FAIL_IF( parser.parse_string() ); goto object_key_parser; @@ -8361,7 +11115,7 @@ array_continue: } finish: - next_json = parser.i; + next_json = parser.structurals.next_structural_index(); return parser.finish(); error: @@ -8372,25 +11126,19 @@ error: } // namespace simdjson UNTARGET_REGION -#endif // IS_X86_64 - #endif // SIMDJSON_HASWELL_STAGE2_BUILD_TAPE_H /* end file src/generic/stage2_streaming_build_tape.h */ +#endif +#if SIMDJSON_IMPLEMENTATION_WESTMERE /* begin file src/westmere/stage2_build_tape.h */ #ifndef SIMDJSON_WESTMERE_STAGE2_BUILD_TAPE_H #define SIMDJSON_WESTMERE_STAGE2_BUILD_TAPE_H - -#ifdef IS_X86_64 - /* westmere/implementation.h already included: #include "westmere/implementation.h" */ /* begin file src/westmere/stringparsing.h */ #ifndef SIMDJSON_WESTMERE_STRINGPARSING_H #define SIMDJSON_WESTMERE_STRINGPARSING_H - -#ifdef IS_X86_64 - /* jsoncharutils.h already included: #include "jsoncharutils.h" */ /* westmere/simd.h already included: #include "westmere/simd.h" */ /* westmere/intrinsics.h already included: #include "westmere/intrinsics.h" */ @@ -8402,16 +11150,24 @@ namespace simdjson::westmere { using namespace simd; // Holds backslashes and quotes locations. -struct parse_string_helper { +struct backslash_and_quote { +public: + static constexpr uint32_t BYTES_PROCESSED = 32; + really_inline static backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); + + really_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } + really_inline bool has_backslash() { return bs_bits != 0; } + really_inline int quote_index() { return trailing_zeroes(quote_bits); } + really_inline int backslash_index() { return trailing_zeroes(bs_bits); } + uint32_t bs_bits; uint32_t quote_bits; - static const uint32_t BYTES_PROCESSED = 32; -}; +}; // struct backslash_and_quote -really_inline parse_string_helper find_bs_bits_and_quote_bits(const uint8_t *src, uint8_t *dst) { +really_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { // this can read up to 31 bytes beyond the buffer size, but we require // SIMDJSON_PADDING of padding - static_assert(SIMDJSON_PADDING >= (parse_string_helper::BYTES_PROCESSED - 1)); + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1)); simd8 v0(src); simd8 v1(src + 16); v0.store(dst); @@ -8497,24 +11253,19 @@ really_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, return offset > 0; } -WARN_UNUSED really_inline uint8_t *parse_string(const uint8_t *buf, - uint32_t offset, - uint8_t *dst) { - const uint8_t *src = &buf[offset + 1]; /* we know that buf at offset is a " */ +WARN_UNUSED really_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst) { + src++; while (1) { - parse_string_helper helper = find_bs_bits_and_quote_bits(src, dst); - if (((helper.bs_bits - 1) & helper.quote_bits) != 0) { - /* we encountered quotes first. Move dst to point to quotes and exit - */ - - /* find out where the quote is... */ - auto quote_dist = trailing_zeroes(helper.quote_bits); - - return dst + quote_dist; + // Copy the next n bytes, and find the backslash and quote in them. + auto bs_quote = backslash_and_quote::copy_and_find(src, dst); + // If the next thing is the end quote, copy and return + if (bs_quote.has_quote_first()) { + // we encountered quotes first. Move dst to point to quotes and exit + return dst + bs_quote.quote_index(); } - if (((helper.quote_bits - 1) & helper.bs_bits) != 0) { + if (bs_quote.has_backslash()) { /* find out where the backspace is */ - auto bs_dist = trailing_zeroes(helper.bs_bits); + auto bs_dist = bs_quote.backslash_index(); uint8_t escape_char = src[bs_dist + 1]; /* we encountered backslash first. Handle backslash */ if (escape_char == 'u') { @@ -8541,8 +11292,8 @@ WARN_UNUSED really_inline uint8_t *parse_string(const uint8_t *buf, } else { /* they are the same. Since they can't co-occur, it means we * encountered neither. */ - src += parse_string_helper::BYTES_PROCESSED; - dst += parse_string_helper::BYTES_PROCESSED; + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; } } /* can't be reached */ @@ -8555,17 +11306,12 @@ WARN_UNUSED really_inline uint8_t *parse_string(const uint8_t *buf, } // namespace simdjson::westmere UNTARGET_REGION -#endif // IS_X86_64 - #endif // SIMDJSON_WESTMERE_STRINGPARSING_H /* end file src/generic/stringparsing.h */ /* begin file src/westmere/numberparsing.h */ #ifndef SIMDJSON_WESTMERE_NUMBERPARSING_H #define SIMDJSON_WESTMERE_NUMBERPARSING_H - -#ifdef IS_X86_64 - /* jsoncharutils.h already included: #include "jsoncharutils.h" */ /* westmere/intrinsics.h already included: #include "westmere/intrinsics.h" */ /* westmere/bitmanipulation.h already included: #include "westmere/bitmanipulation.h" */ @@ -8606,81 +11352,214 @@ static inline uint32_t parse_eight_digits_unrolled(const char *chars) { /* begin file src/generic/numberparsing.h */ namespace numberparsing { -// Allowable floating-point values range -// std::numeric_limits::lowest() to std::numeric_limits::max(), -// so from -1.7976e308 all the way to 1.7975e308 in binary64. The lowest -// non-zero normal values is std::numeric_limits::min() or -// about 2.225074e-308. -static const double power_of_ten[] = { - 1e-308, 1e-307, 1e-306, 1e-305, 1e-304, 1e-303, 1e-302, 1e-301, 1e-300, - 1e-299, 1e-298, 1e-297, 1e-296, 1e-295, 1e-294, 1e-293, 1e-292, 1e-291, - 1e-290, 1e-289, 1e-288, 1e-287, 1e-286, 1e-285, 1e-284, 1e-283, 1e-282, - 1e-281, 1e-280, 1e-279, 1e-278, 1e-277, 1e-276, 1e-275, 1e-274, 1e-273, - 1e-272, 1e-271, 1e-270, 1e-269, 1e-268, 1e-267, 1e-266, 1e-265, 1e-264, - 1e-263, 1e-262, 1e-261, 1e-260, 1e-259, 1e-258, 1e-257, 1e-256, 1e-255, - 1e-254, 1e-253, 1e-252, 1e-251, 1e-250, 1e-249, 1e-248, 1e-247, 1e-246, - 1e-245, 1e-244, 1e-243, 1e-242, 1e-241, 1e-240, 1e-239, 1e-238, 1e-237, - 1e-236, 1e-235, 1e-234, 1e-233, 1e-232, 1e-231, 1e-230, 1e-229, 1e-228, - 1e-227, 1e-226, 1e-225, 1e-224, 1e-223, 1e-222, 1e-221, 1e-220, 1e-219, - 1e-218, 1e-217, 1e-216, 1e-215, 1e-214, 1e-213, 1e-212, 1e-211, 1e-210, - 1e-209, 1e-208, 1e-207, 1e-206, 1e-205, 1e-204, 1e-203, 1e-202, 1e-201, - 1e-200, 1e-199, 1e-198, 1e-197, 1e-196, 1e-195, 1e-194, 1e-193, 1e-192, - 1e-191, 1e-190, 1e-189, 1e-188, 1e-187, 1e-186, 1e-185, 1e-184, 1e-183, - 1e-182, 1e-181, 1e-180, 1e-179, 1e-178, 1e-177, 1e-176, 1e-175, 1e-174, - 1e-173, 1e-172, 1e-171, 1e-170, 1e-169, 1e-168, 1e-167, 1e-166, 1e-165, - 1e-164, 1e-163, 1e-162, 1e-161, 1e-160, 1e-159, 1e-158, 1e-157, 1e-156, - 1e-155, 1e-154, 1e-153, 1e-152, 1e-151, 1e-150, 1e-149, 1e-148, 1e-147, - 1e-146, 1e-145, 1e-144, 1e-143, 1e-142, 1e-141, 1e-140, 1e-139, 1e-138, - 1e-137, 1e-136, 1e-135, 1e-134, 1e-133, 1e-132, 1e-131, 1e-130, 1e-129, - 1e-128, 1e-127, 1e-126, 1e-125, 1e-124, 1e-123, 1e-122, 1e-121, 1e-120, - 1e-119, 1e-118, 1e-117, 1e-116, 1e-115, 1e-114, 1e-113, 1e-112, 1e-111, - 1e-110, 1e-109, 1e-108, 1e-107, 1e-106, 1e-105, 1e-104, 1e-103, 1e-102, - 1e-101, 1e-100, 1e-99, 1e-98, 1e-97, 1e-96, 1e-95, 1e-94, 1e-93, - 1e-92, 1e-91, 1e-90, 1e-89, 1e-88, 1e-87, 1e-86, 1e-85, 1e-84, - 1e-83, 1e-82, 1e-81, 1e-80, 1e-79, 1e-78, 1e-77, 1e-76, 1e-75, - 1e-74, 1e-73, 1e-72, 1e-71, 1e-70, 1e-69, 1e-68, 1e-67, 1e-66, - 1e-65, 1e-64, 1e-63, 1e-62, 1e-61, 1e-60, 1e-59, 1e-58, 1e-57, - 1e-56, 1e-55, 1e-54, 1e-53, 1e-52, 1e-51, 1e-50, 1e-49, 1e-48, - 1e-47, 1e-46, 1e-45, 1e-44, 1e-43, 1e-42, 1e-41, 1e-40, 1e-39, - 1e-38, 1e-37, 1e-36, 1e-35, 1e-34, 1e-33, 1e-32, 1e-31, 1e-30, - 1e-29, 1e-28, 1e-27, 1e-26, 1e-25, 1e-24, 1e-23, 1e-22, 1e-21, - 1e-20, 1e-19, 1e-18, 1e-17, 1e-16, 1e-15, 1e-14, 1e-13, 1e-12, - 1e-11, 1e-10, 1e-9, 1e-8, 1e-7, 1e-6, 1e-5, 1e-4, 1e-3, - 1e-2, 1e-1, 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, - 1e7, 1e8, 1e9, 1e10, 1e11, 1e12, 1e13, 1e14, 1e15, - 1e16, 1e17, 1e18, 1e19, 1e20, 1e21, 1e22, 1e23, 1e24, - 1e25, 1e26, 1e27, 1e28, 1e29, 1e30, 1e31, 1e32, 1e33, - 1e34, 1e35, 1e36, 1e37, 1e38, 1e39, 1e40, 1e41, 1e42, - 1e43, 1e44, 1e45, 1e46, 1e47, 1e48, 1e49, 1e50, 1e51, - 1e52, 1e53, 1e54, 1e55, 1e56, 1e57, 1e58, 1e59, 1e60, - 1e61, 1e62, 1e63, 1e64, 1e65, 1e66, 1e67, 1e68, 1e69, - 1e70, 1e71, 1e72, 1e73, 1e74, 1e75, 1e76, 1e77, 1e78, - 1e79, 1e80, 1e81, 1e82, 1e83, 1e84, 1e85, 1e86, 1e87, - 1e88, 1e89, 1e90, 1e91, 1e92, 1e93, 1e94, 1e95, 1e96, - 1e97, 1e98, 1e99, 1e100, 1e101, 1e102, 1e103, 1e104, 1e105, - 1e106, 1e107, 1e108, 1e109, 1e110, 1e111, 1e112, 1e113, 1e114, - 1e115, 1e116, 1e117, 1e118, 1e119, 1e120, 1e121, 1e122, 1e123, - 1e124, 1e125, 1e126, 1e127, 1e128, 1e129, 1e130, 1e131, 1e132, - 1e133, 1e134, 1e135, 1e136, 1e137, 1e138, 1e139, 1e140, 1e141, - 1e142, 1e143, 1e144, 1e145, 1e146, 1e147, 1e148, 1e149, 1e150, - 1e151, 1e152, 1e153, 1e154, 1e155, 1e156, 1e157, 1e158, 1e159, - 1e160, 1e161, 1e162, 1e163, 1e164, 1e165, 1e166, 1e167, 1e168, - 1e169, 1e170, 1e171, 1e172, 1e173, 1e174, 1e175, 1e176, 1e177, - 1e178, 1e179, 1e180, 1e181, 1e182, 1e183, 1e184, 1e185, 1e186, - 1e187, 1e188, 1e189, 1e190, 1e191, 1e192, 1e193, 1e194, 1e195, - 1e196, 1e197, 1e198, 1e199, 1e200, 1e201, 1e202, 1e203, 1e204, - 1e205, 1e206, 1e207, 1e208, 1e209, 1e210, 1e211, 1e212, 1e213, - 1e214, 1e215, 1e216, 1e217, 1e218, 1e219, 1e220, 1e221, 1e222, - 1e223, 1e224, 1e225, 1e226, 1e227, 1e228, 1e229, 1e230, 1e231, - 1e232, 1e233, 1e234, 1e235, 1e236, 1e237, 1e238, 1e239, 1e240, - 1e241, 1e242, 1e243, 1e244, 1e245, 1e246, 1e247, 1e248, 1e249, - 1e250, 1e251, 1e252, 1e253, 1e254, 1e255, 1e256, 1e257, 1e258, - 1e259, 1e260, 1e261, 1e262, 1e263, 1e264, 1e265, 1e266, 1e267, - 1e268, 1e269, 1e270, 1e271, 1e272, 1e273, 1e274, 1e275, 1e276, - 1e277, 1e278, 1e279, 1e280, 1e281, 1e282, 1e283, 1e284, 1e285, - 1e286, 1e287, 1e288, 1e289, 1e290, 1e291, 1e292, 1e293, 1e294, - 1e295, 1e296, 1e297, 1e298, 1e299, 1e300, 1e301, 1e302, 1e303, - 1e304, 1e305, 1e306, 1e307, 1e308}; + +// Attempts to compute i * 10^(power) exactly; and if "negative" is +// true, negate the result. +// This function will only work in some cases, when it does not work, success is +// set to false. This should work *most of the time* (like 99% of the time). +// We assume that power is in the [FASTFLOAT_SMALLEST_POWER, +// FASTFLOAT_LARGEST_POWER] interval: the caller is responsible for this check. +really_inline double compute_float_64(int64_t power, uint64_t i, bool negative, + bool *success) { + // we start with a fast path + // It was described in + // Clinger WD. How to read floating point numbers accurately. + // ACM SIGPLAN Notices. 1990 + if (-22 <= power && power <= 22 && i <= 9007199254740991) { + // convert the integer into a double. This is lossless since + // 0 <= i <= 2^53 - 1. + double d = i; + // + // The general idea is as follows. + // If 0 <= s < 2^53 and if 10^0 <= p <= 10^22 then + // 1) Both s and p can be represented exactly as 64-bit floating-point + // values + // (binary64). + // 2) Because s and p can be represented exactly as floating-point values, + // then s * p + // and s / p will produce correctly rounded values. + // + if (power < 0) { + d = d / power_of_ten[-power]; + } else { + d = d * power_of_ten[power]; + } + if (negative) { + d = -d; + } + *success = true; + return d; + } + // When 22 < power && power < 22 + 16, we could + // hope for another, secondary fast path. It wa + // described by David M. Gay in "Correctly rounded + // binary-decimal and decimal-binary conversions." (1990) + // If you need to compute i * 10^(22 + x) for x < 16, + // first compute i * 10^x, if you know that result is exact + // (e.g., when i * 10^x < 2^53), + // then you can still proceed and do (i * 10^x) * 10^22. + // Is this worth your time? + // You need 22 < power *and* power < 22 + 16 *and* (i * 10^(x-22) < 2^53) + // for this second fast path to work. + // If you you have 22 < power *and* power < 22 + 16, and then you + // optimistically compute "i * 10^(x-22)", there is still a chance that you + // have wasted your time if i * 10^(x-22) >= 2^53. It makes the use cases of + // this optimization maybe less common than we would like. Source: + // http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ + // also used in RapidJSON: https://rapidjson.org/strtod_8h_source.html + + // The fast path has now failed, so we are failing back on the slower path. + + // In the slow path, we need to adjust i so that it is > 1<<63 which is always + // possible, except if i == 0, so we handle i == 0 separately. + if(i == 0) { + return 0.0; + } + + // We are going to need to do some 64-bit arithmetic to get a more precise product. + // We use a table lookup approach. + components c = + power_of_ten_components[power - FASTFLOAT_SMALLEST_POWER]; + // safe because + // power >= FASTFLOAT_SMALLEST_POWER + // and power <= FASTFLOAT_LARGEST_POWER + // we recover the mantissa of the power, it has a leading 1. It is always + // rounded down. + uint64_t factor_mantissa = c.mantissa; + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(i); + i <<= lz; + // We want the most significant 64 bits of the product. We know + // this will be non-zero because the most significant bit of i is + // 1. + value128 product = full_multiplication(i, factor_mantissa); + uint64_t lower = product.low; + uint64_t upper = product.high; + + // We know that upper has at most one leading zero because + // both i and factor_mantissa have a leading one. This means + // that the result is at least as large as ((1<<63)*(1<<63))/(1<<64). + + // As long as the first 9 bits of "upper" are not "1", then we + // know that we have an exact computed value for the leading + // 55 bits because any imprecision would play out as a +1, in + // the worst case. + if (unlikely((upper & 0x1FF) == 0x1FF) && (lower + i < lower)) { + uint64_t factor_mantissa_low = + mantissa_128[power - FASTFLOAT_SMALLEST_POWER]; + // next, we compute the 64-bit x 128-bit multiplication, getting a 192-bit + // result (three 64-bit values) + product = full_multiplication(i, factor_mantissa_low); + uint64_t product_low = product.low; + uint64_t product_middle2 = product.high; + uint64_t product_middle1 = lower; + uint64_t product_high = upper; + uint64_t product_middle = product_middle1 + product_middle2; + if (product_middle < product_middle1) { + product_high++; // overflow carry + } + // We want to check whether mantissa *i + i would affect our result. + // This does happen, e.g. with 7.3177701707893310e+15. + if (((product_middle + 1 == 0) && ((product_high & 0x1FF) == 0x1FF) && + (product_low + i < product_low))) { // let us be prudent and bail out. + *success = false; + return 0; + } + upper = product_high; + lower = product_middle; + } + // The final mantissa should be 53 bits with a leading 1. + // We shift it so that it occupies 54 bits with a leading 1. + /////// + uint64_t upperbit = upper >> 63; + uint64_t mantissa = upper >> (upperbit + 9); + lz += 1 ^ upperbit; + + // Here we have mantissa < (1<<54). + + // We have to round to even. The "to even" part + // is only a problem when we are right in between two floats + // which we guard against. + // If we have lots of trailing zeros, we may fall right between two + // floating-point values. + if (unlikely((lower == 0) && ((upper & 0x1FF) == 0) && + ((mantissa & 3) == 1))) { + // if mantissa & 1 == 1 we might need to round up. + // + // Scenarios: + // 1. We are not in the middle. Then we should round up. + // + // 2. We are right in the middle. Whether we round up depends + // on the last significant bit: if it is "one" then we round + // up (round to even) otherwise, we do not. + // + // So if the last significant bit is 1, we can safely round up. + // Hence we only need to bail out if (mantissa & 3) == 1. + // Otherwise we may need more accuracy or analysis to determine whether + // we are exactly between two floating-point numbers. + // It can be triggered with 1e23. + // Note: because the factor_mantissa and factor_mantissa_low are + // almost always rounded down (except for small positive powers), + // almost always should round up. + *success = false; + return 0; + } + + mantissa += mantissa & 1; + mantissa >>= 1; + + // Here we have mantissa < (1<<53), unless there was an overflow + if (mantissa >= (1ULL << 53)) { + ////////// + // This will happen when parsing values such as 7.2057594037927933e+16 + //////// + mantissa = (1ULL << 52); + lz--; // undo previous addition + } + mantissa &= ~(1ULL << 52); + uint64_t real_exponent = c.exp - lz; + // we have to check that real_exponent is in range, otherwise we bail out + if (unlikely((real_exponent < 1) || (real_exponent > 2046))) { + *success = false; + return 0; + } + mantissa |= real_exponent << 52; + mantissa |= (((uint64_t)negative) << 63); + double d; + memcpy(&d, &mantissa, sizeof(d)); + *success = true; + return d; +} + +static bool parse_float_strtod(const char *ptr, double *outDouble) { + char *endptr; + *outDouble = strtod(ptr, &endptr); + // Some libraries will set errno = ERANGE when the value is subnormal, + // yet we may want to be able to parse subnormal values. + // However, we do not want to tolerate NAN or infinite values. + // + // Values like infinity or NaN are not allowed in the JSON specification. + // If you consume a large value and you map it to "infinity", you will no + // longer be able to serialize back a standard-compliant JSON. And there is + // no realistic application where you might need values so large than they + // can't fit in binary64. The maximal value is about 1.7976931348623157 × + // 10^308 It is an unimaginable large number. There will never be any piece of + // engineering involving as many as 10^308 parts. It is estimated that there + // are about 10^80 atoms in the universe.  The estimate for the total number + // of electrons is similar. Using a double-precision floating-point value, we + // can represent easily the number of atoms in the universe. We could also + // represent the number of ways you can pick any three individual atoms at + // random in the universe. If you ever encounter a number much larger than + // 10^308, you know that you have a bug. RapidJSON will reject a document with + // a float that does not fit in binary64. JSON for Modern C++ (nlohmann/json) + // will flat out throw an exception. + // + if ((endptr == ptr) || (!std::isfinite(*outDouble))) { + return false; + } + return true; +} really_inline bool is_integer(char c) { return (c >= '0' && c <= '9'); @@ -8688,7 +11567,7 @@ really_inline bool is_integer(char c) { } // We need to check that the character following a zero is valid. This is -// probably frequent and it is hard than it looks. We are building all of this +// probably frequent and it is harder than it looks. We are building all of this // just to differentiate between 0x1 (invalid), 0,1 (valid) 0e1 (valid)... const bool structural_or_whitespace_or_exponent_or_decimal_negated[256] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -8726,162 +11605,6 @@ really_inline bool is_made_of_eight_digits_fast(const char *chars) { 0x3333333333333333); } - -// -// This function computes base * 10 ^ (- negative_exponent ). -// It is only even going to be used when negative_exponent is tiny. -really_inline double subnormal_power10(double base, int64_t negative_exponent) { - // avoid integer overflows in the pow expression, those values would - // become zero anyway. - if(negative_exponent < -1000) { - return 0; - } - - // this is probably not going to be fast - return base * 1e-308 * pow(10, negative_exponent + 308); -} - -// called by parse_number when we know that the output is a float, -// but where there might be some integer overflow. The trick here is to -// parse using floats from the start. -// Do not call this function directly as it skips some of the checks from -// parse_number -// -// This function will almost never be called!!! -// -// Note: a redesign could avoid this function entirely. -// -never_inline bool parse_float(const uint8_t *const buf, document::parser &parser, - const uint32_t offset, bool found_minus) { - const char *p = reinterpret_cast(buf + offset); - bool negative = false; - if (found_minus) { - ++p; - negative = true; - } - long double i; - if (*p == '0') { // 0 cannot be followed by an integer - ++p; - i = 0; - } else { - unsigned char digit = *p - '0'; - i = digit; - p++; - while (is_integer(*p)) { - digit = *p - '0'; - i = 10 * i + digit; - ++p; - } - } - if ('.' == *p) { - ++p; - int fractional_weight = 308; - if (is_integer(*p)) { - unsigned char digit = *p - '0'; - ++p; - - fractional_weight--; - i = i + digit * (fractional_weight >= 0 ? power_of_ten[fractional_weight] - : 0); - } else { -#ifdef JSON_TEST_NUMBERS // for unit testing - found_invalid_number(buf + offset); -#endif - return false; - } - while (is_integer(*p)) { - unsigned char digit = *p - '0'; - ++p; - fractional_weight--; - i = i + digit * (fractional_weight >= 0 ? power_of_ten[fractional_weight] - : 0); - } - } - if (('e' == *p) || ('E' == *p)) { - ++p; - bool neg_exp = false; - if ('-' == *p) { - neg_exp = true; - ++p; - } else if ('+' == *p) { - ++p; - } - if (!is_integer(*p)) { -#ifdef JSON_TEST_NUMBERS // for unit testing - found_invalid_number(buf + offset); -#endif - return false; - } - unsigned char digit = *p - '0'; - int64_t exp_number = digit; // exponential part - p++; - if (is_integer(*p)) { - digit = *p - '0'; - exp_number = 10 * exp_number + digit; - ++p; - } - if (is_integer(*p)) { - digit = *p - '0'; - exp_number = 10 * exp_number + digit; - ++p; - } - if (is_integer(*p)) { - digit = *p - '0'; - exp_number = 10 * exp_number + digit; - ++p; - } - while (is_integer(*p)) { - if (exp_number > 0x100000000) { // we need to check for overflows -// we refuse to parse this -#ifdef JSON_TEST_NUMBERS // for unit testing - found_invalid_number(buf + offset); -#endif - return false; - } - digit = *p - '0'; - exp_number = 10 * exp_number + digit; - ++p; - } - if (unlikely(exp_number > 308)) { - // this path is unlikely - if (neg_exp) { - // We either have zero or a subnormal. - // We expect this to be uncommon so we go through a slow path. - i = subnormal_power10(i, -exp_number); - } else { -// We know for sure that we have a number that is too large, -// we refuse to parse this -#ifdef JSON_TEST_NUMBERS // for unit testing - found_invalid_number(buf + offset); -#endif - return false; - } - } else { - int exponent = (neg_exp ? -exp_number : exp_number); - // we have that exp_number is [0,308] so that - // exponent is [-308,308] so that - // 308 + exponent is in [0, 2 * 308] - i *= power_of_ten[308 + exponent]; - } - } - if (is_not_structural_or_whitespace(*p)) { - return false; - } - // check that we can go from long double to double safely. - if(i > std::numeric_limits::max()) { -#ifdef JSON_TEST_NUMBERS // for unit testing - found_invalid_number(buf + offset); -#endif - return false; - } - double d = negative ? -i : i; - parser.on_number_double(d); -#ifdef JSON_TEST_NUMBERS // for unit testing - found_float(d, buf + offset); -#endif - return is_structural_or_whitespace(*p); -} - // called by parse_number when we know that the output is an integer, // but where there might be some integer overflow. // we want to catch overflows! @@ -8890,11 +11613,10 @@ never_inline bool parse_float(const uint8_t *const buf, document::parser &parser // // This function will almost never be called!!! // -never_inline bool parse_large_integer(const uint8_t *const buf, - document::parser &parser, - const uint32_t offset, - bool found_minus) { - const char *p = reinterpret_cast(buf + offset); +never_inline bool parse_large_integer(const uint8_t *const src, + document::parser &parser, + bool found_minus) { + const char *p = reinterpret_cast(src); bool negative = false; if (found_minus) { @@ -8915,13 +11637,13 @@ never_inline bool parse_large_integer(const uint8_t *const buf, digit = *p - '0'; if (mul_overflow(i, 10, &i)) { #ifdef JSON_TEST_NUMBERS // for unit testing - found_invalid_number(buf + offset); + found_invalid_number(src); #endif return false; // overflow } if (add_overflow(i, digit, &i)) { #ifdef JSON_TEST_NUMBERS // for unit testing - found_invalid_number(buf + offset); + found_invalid_number(src); #endif return false; // overflow } @@ -8930,40 +11652,40 @@ never_inline bool parse_large_integer(const uint8_t *const buf, } if (negative) { if (i > 0x8000000000000000) { - // overflows! + // overflows! #ifdef JSON_TEST_NUMBERS // for unit testing - found_invalid_number(buf + offset); + found_invalid_number(src); #endif return false; // overflow } else if (i == 0x8000000000000000) { // In two's complement, we cannot represent 0x8000000000000000 - // as a positive signed integer, but the negative version is + // as a positive signed integer, but the negative version is // possible. constexpr int64_t signed_answer = INT64_MIN; parser.on_number_s64(signed_answer); #ifdef JSON_TEST_NUMBERS // for unit testing - found_integer(signed_answer, buf + offset); + found_integer(signed_answer, src); #endif } else { // we can negate safely int64_t signed_answer = -static_cast(i); parser.on_number_s64(signed_answer); #ifdef JSON_TEST_NUMBERS // for unit testing - found_integer(signed_answer, buf + offset); + found_integer(signed_answer, src); #endif } } else { // we have a positive integer, the contract is that - // we try to represent it as a signed integer and only + // we try to represent it as a signed integer and only // fallback on unsigned integers if absolutely necessary. - if(i < 0x8000000000000000) { + if (i < 0x8000000000000000) { #ifdef JSON_TEST_NUMBERS // for unit testing - found_integer(i, buf + offset); + found_integer(i, src); #endif parser.on_number_s64(i); } else { #ifdef JSON_TEST_NUMBERS // for unit testing - found_unsigned_integer(i, buf + offset); + found_unsigned_integer(i, src); #endif parser.on_number_u64(i); } @@ -8971,7 +11693,22 @@ never_inline bool parse_large_integer(const uint8_t *const buf, return is_structural_or_whitespace(*p); } -// parse the number at buf + offset +bool slow_float_parsing(UNUSED const char * src, document::parser &parser) { + double d; + if (parse_float_strtod(src, &d)) { + parser.on_number_double(d); +#ifdef JSON_TEST_NUMBERS // for unit testing + found_float(d, (const uint8_t *)src); +#endif + return true; + } +#ifdef JSON_TEST_NUMBERS // for unit testing + found_invalid_number((const uint8_t *)src); +#endif + return false; +} + +// parse the number at src // define JSON_TEST_NUMBERS for unit testing // // It is assumed that the number is followed by a structural ({,},],[) character @@ -8979,24 +11716,23 @@ never_inline bool parse_large_integer(const uint8_t *const buf, // document is made of a single number), then it is necessary to copy the // content and append a space before calling this function. // -// Our objective is accurate parsing (ULP of 0 or 1) at high speed. -really_inline bool parse_number(UNUSED const uint8_t *const buf, - UNUSED const uint32_t offset, +// Our objective is accurate parsing (ULP of 0) at high speed. +really_inline bool parse_number(UNUSED const uint8_t *const src, UNUSED bool found_minus, document::parser &parser) { #ifdef SIMDJSON_SKIPNUMBERPARSING // for performance analysis, it is sometimes // useful to skip parsing - parser.on_number_s64(0); // always write zero + parser.on_number_s64(0); // always write zero return true; // always succeeds #else - const char *p = reinterpret_cast(buf + offset); + const char *p = reinterpret_cast(src); bool negative = false; if (found_minus) { ++p; negative = true; if (!is_integer(*p)) { // a negative sign must be followed by an integer #ifdef JSON_TEST_NUMBERS // for unit testing - found_invalid_number(buf + offset); + found_invalid_number(src); #endif return false; } @@ -9008,7 +11744,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const buf, ++p; if (is_not_structural_or_whitespace_or_exponent_or_decimal(*p)) { #ifdef JSON_TEST_NUMBERS // for unit testing - found_invalid_number(buf + offset); + found_invalid_number(src); #endif return false; } @@ -9016,7 +11752,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const buf, } else { if (!(is_integer(*p))) { // must start with an integer #ifdef JSON_TEST_NUMBERS // for unit testing - found_invalid_number(buf + offset); + found_invalid_number(src); #endif return false; } @@ -9051,7 +11787,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const buf, // we will handle the overflow later } else { #ifdef JSON_TEST_NUMBERS // for unit testing - found_invalid_number(buf + offset); + found_invalid_number(src); #endif return false; } @@ -9086,7 +11822,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const buf, } if (!is_integer(*p)) { #ifdef JSON_TEST_NUMBERS // for unit testing - found_invalid_number(buf + offset); + found_invalid_number(src); #endif return false; } @@ -9107,7 +11843,7 @@ really_inline bool parse_number(UNUSED const uint8_t *const buf, if (exp_number > 0x100000000) { // we need to check for overflows // we refuse to parse this #ifdef JSON_TEST_NUMBERS // for unit testing - found_invalid_number(buf + offset); + found_invalid_number(src); #endif return false; } @@ -9118,7 +11854,9 @@ really_inline bool parse_number(UNUSED const uint8_t *const buf, exponent += (neg_exp ? -exp_number : exp_number); } if (is_float) { - uint64_t power_index = 308 + exponent; + // If we frequently had to deal with long strings of digits, + // we could extend our code by using a 128-bit integer instead + // of a 64-bit integer. However, this is uncommon in practice. if (unlikely((digit_count >= 19))) { // this is uncommon // It is possible that the integer had an overflow. // We have to handle the case where we have 0.0000somenumber. @@ -9132,31 +11870,47 @@ really_inline bool parse_number(UNUSED const uint8_t *const buf, // Ok, chances are good that we had an overflow! // this is almost never going to get called!!! // we start anew, going slowly!!! - return parse_float(buf, parser, offset, found_minus); + // This will happen in the following examples: + // 10000000000000000000000000000000000000000000e+308 + // 3.1415926535897932384626433832795028841971693993751 + // + return slow_float_parsing((const char *) src, parser); } } - if (unlikely((power_index > 2 * 308))) { // this is uncommon!!! + if (unlikely(exponent < FASTFLOAT_SMALLEST_POWER) || + (exponent > FASTFLOAT_LARGEST_POWER)) { // this is uncommon!!! // this is almost never going to get called!!! // we start anew, going slowly!!! - return parse_float(buf, parser, offset, found_minus); + return slow_float_parsing((const char *) src, parser); } - double factor = power_of_ten[power_index]; - factor = negative ? -factor : factor; - double d = i * factor; - parser.on_number_double(d); + bool success = true; + double d = compute_float_64(exponent, i, negative, &success); + if (!success) { + // we are almost never going to get here. + success = parse_float_strtod((const char *)src, &d); + } + if (success) { + parser.on_number_double(d); #ifdef JSON_TEST_NUMBERS // for unit testing - found_float(d, buf + offset); + found_float(d, src); #endif + return true; + } else { +#ifdef JSON_TEST_NUMBERS // for unit testing + found_invalid_number(src); +#endif + return false; + } } else { if (unlikely(digit_count >= 18)) { // this is uncommon!!! // there is a good chance that we had an overflow, so we need // need to recover: we parse the whole thing again. - return parse_large_integer(buf, parser, offset, found_minus); + return parse_large_integer(src, parser, found_minus); } i = negative ? 0 - i : i; parser.on_number_s64(i); #ifdef JSON_TEST_NUMBERS // for unit testing - found_integer(i, buf + offset); + found_integer(i, src); #endif } return is_structural_or_whitespace(*p); @@ -9169,15 +11923,63 @@ really_inline bool parse_number(UNUSED const uint8_t *const buf, } // namespace simdjson::westmere UNTARGET_REGION - - -#endif // IS_X86_64 #endif // SIMDJSON_WESTMERE_NUMBERPARSING_H /* end file src/generic/numberparsing.h */ TARGET_WESTMERE namespace simdjson::westmere { +/* begin file src/generic/atomparsing.h */ +namespace atomparsing { + +really_inline uint32_t string_to_uint32(const char* str) { return *reinterpret_cast(str); } + +WARN_UNUSED +really_inline bool str4ncmp(const uint8_t *src, const char* atom) { + uint32_t srcval; // we want to avoid unaligned 64-bit loads (undefined in C/C++) + static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING); + std::memcpy(&srcval, src, sizeof(uint32_t)); + return srcval ^ string_to_uint32(atom); +} + +WARN_UNUSED +really_inline bool is_valid_true_atom(const uint8_t *src) { + return (str4ncmp(src, "true") | is_not_structural_or_whitespace(src[4])) == 0; +} + +WARN_UNUSED +really_inline bool is_valid_true_atom(const uint8_t *src, size_t len) { + if (len > 4) { return is_valid_true_atom(src); } + else if (len == 4) { return !str4ncmp(src, "true"); } + else { return false; } +} + +WARN_UNUSED +really_inline bool is_valid_false_atom(const uint8_t *src) { + return (str4ncmp(src+1, "alse") | is_not_structural_or_whitespace(src[5])) == 0; +} + +WARN_UNUSED +really_inline bool is_valid_false_atom(const uint8_t *src, size_t len) { + if (len > 5) { return is_valid_false_atom(src); } + else if (len == 5) { return !str4ncmp(src+1, "alse"); } + else { return false; } +} + +WARN_UNUSED +really_inline bool is_valid_null_atom(const uint8_t *src) { + return (str4ncmp(src, "null") | is_not_structural_or_whitespace(src[4])) == 0; +} + +WARN_UNUSED +really_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { + if (len > 4) { return is_valid_null_atom(src); } + else if (len == 4) { return !str4ncmp(src, "null"); } + else { return false; } +} + +} // namespace atomparsing +/* end file src/generic/atomparsing.h */ /* begin file src/generic/stage2_build_tape.h */ // This file contains the common code every implementation uses for stage2 // It is intended to be included multiple times and compiled multiple times @@ -9228,28 +12030,25 @@ struct unified_machine_addresses { #undef FAIL_IF #define FAIL_IF(EXPR) { if (EXPR) { return addresses.error; } } -struct structural_parser { - const uint8_t* const buf; - const size_t len; - document::parser &doc_parser; - size_t i; // next structural index - size_t idx; // location of the structural character in the input (buf) - uint8_t c; // used to track the (structural) character we are looking at - uint32_t depth = 0; // could have an arbitrary starting depth - - really_inline structural_parser( - const uint8_t *_buf, - size_t _len, - document::parser &_doc_parser, - uint32_t _i = 0 - ) : buf{_buf}, len{_len}, doc_parser{_doc_parser}, i{_i} {} - +class structural_iterator { +public: + really_inline structural_iterator(const uint8_t* _buf, size_t _len, const uint32_t *_structural_indexes, size_t next_structural_index) + : buf{_buf}, len{_len}, structural_indexes{_structural_indexes}, next_structural{next_structural_index} {} really_inline char advance_char() { - idx = doc_parser.structural_indexes[i++]; - c = buf[idx]; + idx = structural_indexes[next_structural]; + next_structural++; + c = *current(); return c; } - + really_inline char current_char() { + return c; + } + really_inline const uint8_t* current() { + return &buf[idx]; + } + really_inline size_t remaining_len() { + return len - idx; + } template really_inline bool with_space_terminated_copy(const F& f) { /** @@ -9275,6 +12074,35 @@ struct structural_parser { free(copy); return result; } + really_inline bool past_end(uint32_t n_structural_indexes) { + return next_structural+1 > n_structural_indexes; + } + really_inline bool at_end(uint32_t n_structural_indexes) { + return next_structural+1 == n_structural_indexes; + } + really_inline size_t next_structural_index() { + return next_structural; + } + + const uint8_t* const buf; + const size_t len; + const uint32_t* const structural_indexes; + size_t next_structural; // next structural index + size_t idx; // location of the structural character in the input (buf) + uint8_t c; // used to track the (structural) character we are looking at +}; + +struct structural_parser { + structural_iterator structurals; + document::parser &doc_parser; + uint32_t depth; + + really_inline structural_parser( + const uint8_t *buf, + size_t len, + document::parser &_doc_parser, + uint32_t next_structural = 0 + ) : structurals(buf, len, _doc_parser.structural_indexes.get(), next_structural), doc_parser{_doc_parser}, depth{0} {} WARN_UNUSED really_inline bool start_document(ret_address continue_state) { doc_parser.on_start_document(depth); @@ -9315,32 +12143,32 @@ struct structural_parser { WARN_UNUSED really_inline bool parse_string() { uint8_t *dst = doc_parser.on_start_string(); - dst = stringparsing::parse_string(buf, idx, dst); + dst = stringparsing::parse_string(structurals.current(), dst); if (dst == nullptr) { return true; } return !doc_parser.on_end_string(dst); } - WARN_UNUSED really_inline bool parse_number(const uint8_t *copy, uint32_t offset, bool found_minus) { - return !numberparsing::parse_number(copy, offset, found_minus, doc_parser); + WARN_UNUSED really_inline bool parse_number(const uint8_t *src, bool found_minus) { + return !numberparsing::parse_number(src, found_minus, doc_parser); } WARN_UNUSED really_inline bool parse_number(bool found_minus) { - return parse_number(buf, idx, found_minus); + return parse_number(structurals.current(), found_minus); } - WARN_UNUSED really_inline bool parse_atom(const uint8_t *copy, uint32_t offset) { - switch (c) { + WARN_UNUSED really_inline bool parse_atom() { + switch (structurals.current_char()) { case 't': - if (!is_valid_true_atom(copy + offset)) { return true; } + if (!atomparsing::is_valid_true_atom(structurals.current())) { return true; } doc_parser.on_true_atom(); break; case 'f': - if (!is_valid_false_atom(copy + offset)) { return true; } + if (!atomparsing::is_valid_false_atom(structurals.current())) { return true; } doc_parser.on_false_atom(); break; case 'n': - if (!is_valid_null_atom(copy + offset)) { return true; } + if (!atomparsing::is_valid_null_atom(structurals.current())) { return true; } doc_parser.on_null_atom(); break; default: @@ -9349,12 +12177,28 @@ struct structural_parser { return false; } - WARN_UNUSED really_inline bool parse_atom() { - return parse_atom(buf, idx); + WARN_UNUSED really_inline bool parse_single_atom() { + switch (structurals.current_char()) { + case 't': + if (!atomparsing::is_valid_true_atom(structurals.current(), structurals.remaining_len())) { return true; } + doc_parser.on_true_atom(); + break; + case 'f': + if (!atomparsing::is_valid_false_atom(structurals.current(), structurals.remaining_len())) { return true; } + doc_parser.on_false_atom(); + break; + case 'n': + if (!atomparsing::is_valid_null_atom(structurals.current(), structurals.remaining_len())) { return true; } + doc_parser.on_null_atom(); + break; + default: + return true; + } + return false; } WARN_UNUSED really_inline ret_address parse_value(const unified_machine_addresses &addresses, ret_address continue_state) { - switch (c) { + switch (structurals.current_char()) { case '"': FAIL_IF( parse_string() ); return continue_state; @@ -9381,7 +12225,7 @@ struct structural_parser { WARN_UNUSED really_inline error_code finish() { // the string might not be NULL terminated. - if ( i + 1 != doc_parser.n_structural_indexes ) { + if ( !structurals.at_end(doc_parser.n_structural_indexes) ) { return doc_parser.on_error(TAPE_ERROR); } end_document(); @@ -9409,7 +12253,7 @@ struct structural_parser { if (depth >= doc_parser.max_depth()) { return doc_parser.on_error(DEPTH_ERROR); } - switch (c) { + switch (structurals.current_char()) { case '"': return doc_parser.on_error(STRING_ERROR); case '0': @@ -9435,19 +12279,23 @@ struct structural_parser { } } - WARN_UNUSED really_inline error_code start(ret_address finish_state) { + WARN_UNUSED really_inline error_code start(size_t len, ret_address finish_state) { doc_parser.init_stage2(); // sets is_valid to false if (len > doc_parser.capacity()) { return CAPACITY; } // Advance to the first character as soon as possible - advance_char(); + structurals.advance_char(); // Push the root scope (there is always at least one scope) if (start_document(finish_state)) { return doc_parser.on_error(DEPTH_ERROR); } return SUCCESS; } + + really_inline char advance_char() { + return structurals.advance_char(); + } }; // Redefine FAIL_IF to use goto since it'll be used inside the function now @@ -9463,13 +12311,13 @@ struct structural_parser { WARN_UNUSED error_code implementation::stage2(const uint8_t *buf, size_t len, document::parser &doc_parser) const noexcept { static constexpr stage2::unified_machine_addresses addresses = INIT_ADDRESSES(); stage2::structural_parser parser(buf, len, doc_parser); - error_code result = parser.start(addresses.finish); + error_code result = parser.start(len, addresses.finish); if (result) { return result; } // // Read first value // - switch (parser.c) { + switch (parser.structurals.current_char()) { case '{': FAIL_IF( parser.start_object(addresses.finish) ); goto object_begin; @@ -9480,24 +12328,20 @@ WARN_UNUSED error_code implementation::stage2(const uint8_t *buf, size_t len, do FAIL_IF( parser.parse_string() ); goto finish; case 't': case 'f': case 'n': - FAIL_IF( - parser.with_space_terminated_copy([&](auto copy, auto idx) { - return parser.parse_atom(copy, idx); - }) - ); + FAIL_IF( parser.parse_single_atom() ); goto finish; case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': FAIL_IF( - parser.with_space_terminated_copy([&](auto copy, auto idx) { - return parser.parse_number(copy, idx, false); + parser.structurals.with_space_terminated_copy([&](auto copy, auto idx) { + return parser.parse_number(©[idx], false); }) ); goto finish; case '-': FAIL_IF( - parser.with_space_terminated_copy([&](auto copy, auto idx) { - return parser.parse_number(copy, idx, true); + parser.structurals.with_space_terminated_copy([&](auto copy, auto idx) { + return parser.parse_number(©[idx], true); }) ); goto finish; @@ -9509,8 +12353,7 @@ WARN_UNUSED error_code implementation::stage2(const uint8_t *buf, size_t len, do // Object parser states // object_begin: - parser.advance_char(); - switch (parser.c) { + switch (parser.advance_char()) { case '"': { FAIL_IF( parser.parse_string() ); goto object_key_state; @@ -9591,7 +12434,7 @@ struct streaming_structural_parser: structural_parser { really_inline streaming_structural_parser(const uint8_t *_buf, size_t _len, document::parser &_doc_parser, size_t _i) : structural_parser(_buf, _len, _doc_parser, _i) {} // override to add streaming - WARN_UNUSED really_inline error_code start(ret_address finish_parser) { + WARN_UNUSED really_inline error_code start(UNUSED size_t len, ret_address finish_parser) { doc_parser.init_stage2(); // sets is_valid to false // Capacity ain't no thang for streaming, so we don't check it. // Advance to the first character as soon as possible @@ -9605,7 +12448,7 @@ struct streaming_structural_parser: structural_parser { // override to add streaming WARN_UNUSED really_inline error_code finish() { - if ( i + 1 > doc_parser.n_structural_indexes ) { + if ( structurals.past_end(doc_parser.n_structural_indexes) ) { return doc_parser.on_error(TAPE_ERROR); } end_document(); @@ -9615,7 +12458,7 @@ struct streaming_structural_parser: structural_parser { if (doc_parser.containing_scope_offset[depth] != 0) { return doc_parser.on_error(TAPE_ERROR); } - bool finished = i + 1 == doc_parser.n_structural_indexes; + bool finished = structurals.at_end(doc_parser.n_structural_indexes); return doc_parser.on_success(finished ? SUCCESS : SUCCESS_AND_HAS_MORE); } }; @@ -9629,12 +12472,12 @@ struct streaming_structural_parser: structural_parser { WARN_UNUSED error_code implementation::stage2(const uint8_t *buf, size_t len, document::parser &doc_parser, size_t &next_json) const noexcept { static constexpr stage2::unified_machine_addresses addresses = INIT_ADDRESSES(); stage2::streaming_structural_parser parser(buf, len, doc_parser, next_json); - error_code result = parser.start(addresses.finish); + error_code result = parser.start(len, addresses.finish); if (result) { return result; } // // Read first value // - switch (parser.c) { + switch (parser.structurals.current_char()) { case '{': FAIL_IF( parser.start_object(addresses.finish) ); goto object_begin; @@ -9645,24 +12488,20 @@ WARN_UNUSED error_code implementation::stage2(const uint8_t *buf, size_t len, do FAIL_IF( parser.parse_string() ); goto finish; case 't': case 'f': case 'n': - FAIL_IF( - parser.with_space_terminated_copy([&](auto copy, auto idx) { - return parser.parse_atom(copy, idx); - }) - ); + FAIL_IF( parser.parse_single_atom() ); goto finish; case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': FAIL_IF( - parser.with_space_terminated_copy([&](auto copy, auto idx) { - return parser.parse_number(copy, idx, false); + parser.structurals.with_space_terminated_copy([&](auto copy, auto idx) { + return parser.parse_number(©[idx], false); }) ); goto finish; case '-': FAIL_IF( - parser.with_space_terminated_copy([&](auto copy, auto idx) { - return parser.parse_number(copy, idx, true); + parser.structurals.with_space_terminated_copy([&](auto copy, auto idx) { + return parser.parse_number(©[idx], true); }) ); goto finish; @@ -9674,8 +12513,7 @@ WARN_UNUSED error_code implementation::stage2(const uint8_t *buf, size_t len, do // Object parser parsers // object_begin: - parser.advance_char(); - switch (parser.c) { + switch (parser.advance_char()) { case '"': { FAIL_IF( parser.parse_string() ); goto object_key_parser; @@ -9735,7 +12573,7 @@ array_continue: } finish: - next_json = parser.i; + next_json = parser.structurals.next_structural_index(); return parser.finish(); error: @@ -9745,10 +12583,8 @@ error: } // namespace simdjson::westmere UNTARGET_REGION - -#endif // IS_X86_64 - #endif // SIMDJSON_WESTMERE_STAGE2_BUILD_TAPE_H /* end file src/generic/stage2_streaming_build_tape.h */ +#endif /* end file src/generic/stage2_streaming_build_tape.h */ /* end file src/generic/stage2_streaming_build_tape.h */ diff --git a/singleheader/simdjson.h b/singleheader/simdjson.h index a71ddee08..c2b8979d1 100644 --- a/singleheader/simdjson.h +++ b/singleheader/simdjson.h @@ -1,4 +1,4 @@ -/* auto-generated on Thu Mar 5 10:30:07 PST 2020. Do not edit! */ +/* auto-generated on Fri Mar 20 11:47:31 PDT 2020. Do not edit! */ /* begin file include/simdjson.h */ #ifndef SIMDJSON_H #define SIMDJSON_H @@ -32,68 +32,38 @@ // do not change by hand #ifndef SIMDJSON_SIMDJSON_VERSION_H #define SIMDJSON_SIMDJSON_VERSION_H + +/** The version of simdjson being used (major.minor.revision) */ #define SIMDJSON_VERSION 0.2.1 + namespace simdjson { enum { + /** + * The major version (MAJOR.minor.revision) of simdjson being used. + */ SIMDJSON_VERSION_MAJOR = 0, + /** + * The minor version (major.MINOR.revision) of simdjson being used. + */ SIMDJSON_VERSION_MINOR = 2, + /** + * The revision (major.minor.REVISION) of simdjson being used. + */ SIMDJSON_VERSION_REVISION = 1 }; -} +} // namespace simdjson + #endif // SIMDJSON_SIMDJSON_VERSION_H /* end file include/simdjson/simdjson_version.h */ /* begin file include/simdjson/error.h */ #ifndef SIMDJSON_ERROR_H #define SIMDJSON_ERROR_H -#include +/* begin file include/simdjson/common_defs.h */ +#ifndef SIMDJSON_COMMON_DEFS_H +#define SIMDJSON_COMMON_DEFS_H -namespace simdjson { - -enum error_code { - SUCCESS = 0, - SUCCESS_AND_HAS_MORE, //No errors and buffer still has more data - CAPACITY, // This parser can't support a document that big - MEMALLOC, // Error allocating memory, most likely out of memory - TAPE_ERROR, // Something went wrong while writing to the tape (stage 2), this - // is a generic error - DEPTH_ERROR, // Your document exceeds the user-specified depth limitation - STRING_ERROR, // Problem while parsing a string - T_ATOM_ERROR, // Problem while parsing an atom starting with the letter 't' - F_ATOM_ERROR, // Problem while parsing an atom starting with the letter 'f' - N_ATOM_ERROR, // Problem while parsing an atom starting with the letter 'n' - NUMBER_ERROR, // Problem while parsing a number - UTF8_ERROR, // the input is not valid UTF-8 - UNINITIALIZED, // unknown error, or uninitialized document - EMPTY, // no structural element found - UNESCAPED_CHARS, // found unescaped characters in a string. - UNCLOSED_STRING, // missing quote at the end - UNSUPPORTED_ARCHITECTURE, // unsupported architecture - INCORRECT_TYPE, // JSON element has a different type than user expected - NUMBER_OUT_OF_RANGE, // JSON number does not fit in 64 bits - NO_SUCH_FIELD, // JSON field not found in object - UNEXPECTED_ERROR // indicative of a bug in simdjson -}; - -const std::string &error_message(error_code error) noexcept; - -struct invalid_json : public std::exception { - invalid_json(error_code _error) : error{_error} { } - const char *what() const noexcept { return error_message(error); } - error_code error; -}; - -// TODO these are deprecated, remove -using ErrorValues = error_code; -inline const std::string &error_message(int error) noexcept { return error_message(error_code(error)); } - -} // namespace simdjson - -#endif // SIMDJSON_ERROR_H -/* end file include/simdjson/error.h */ -/* begin file include/simdjson/padded_string.h */ -#ifndef SIMDJSON_PADDING_STRING_H -#define SIMDJSON_PADDING_STRING_H +#include /* begin file include/simdjson/portability.h */ #ifndef SIMDJSON_PORTABILITY_H #define SIMDJSON_PORTABILITY_H @@ -118,6 +88,28 @@ inline const std::string &error_message(int error) noexcept { return error_messa #define STRINGIFY_IMPLEMENTATION_(a) #a #define STRINGIFY(a) STRINGIFY_IMPLEMENTATION_(a) +#ifndef SIMDJSON_IMPLEMENTATION_FALLBACK +#define SIMDJSON_IMPLEMENTATION_FALLBACK 1 +#endif + +#if IS_ARM64 +#ifndef SIMDJSON_IMPLEMENTATION_ARM64 +#define SIMDJSON_IMPLEMENTATION_ARM64 1 +#endif +#define SIMDJSON_IMPLEMENTATION_HASWELL 0 +#define SIMDJSON_IMPLEMENTATION_WESTMERE 0 +#endif // IS_ARM64 + +#if IS_X86_64 +#ifndef SIMDJSON_IMPLEMENTATION_HASWELL +#define SIMDJSON_IMPLEMENTATION_HASWELL 1 +#endif +#ifndef SIMDJSON_IMPLEMENTATION_WESTMERE +#define SIMDJSON_IMPLEMENTATION_WESTMERE 1 +#endif +#define SIMDJSON_IMPLEMENTATION_ARM64 0 +#endif // IS_X86_64 + // we are going to use runtime dispatch #ifdef IS_X86_64 #ifdef __clang__ @@ -215,20 +207,38 @@ static inline void aligned_free_char(char *mem_block) { } // namespace simdjson #endif // SIMDJSON_PORTABILITY_H /* end file include/simdjson/portability.h */ -/* begin file include/simdjson/common_defs.h */ -#ifndef SIMDJSON_COMMON_DEFS_H -#define SIMDJSON_COMMON_DEFS_H -#include +namespace simdjson { -// we support documents up to 4GB -#define SIMDJSON_MAXSIZE_BYTES 0xFFFFFFFF +#ifndef SIMDJSON_EXCEPTIONS +#if __cpp_exceptions +#define SIMDJSON_EXCEPTIONS 1 +#else +#define SIMDJSON_EXCEPTIONS 0 +#endif +#endif -// the input buf should be readable up to buf + SIMDJSON_PADDING -// this is a stopgap; there should be a better description of the -// main loop and its behavior that abstracts over this -// See https://github.com/lemire/simdjson/issues/174 -#define SIMDJSON_PADDING 32 +/** The maximum document size supported by simdjson. */ +constexpr size_t SIMDJSON_MAXSIZE_BYTES = 0xFFFFFFFF; + +/** + * The amount of padding needed in a buffer to parse JSON. + * + * the input buf should be readable up to buf + SIMDJSON_PADDING + * this is a stopgap; there should be a better description of the + * main loop and its behavior that abstracts over this + * See https://github.com/lemire/simdjson/issues/174 + */ +constexpr size_t SIMDJSON_PADDING = 32; + +/** + * By default, simdjson supports this many nested objects and arrays. + * + * This is the default for document::parser::max_depth(). + */ +constexpr size_t DEFAULT_MAX_DEPTH = 1024; + +} // namespace simdjson #if defined(__GNUC__) // Marks a block with a name so that MCA analysis can see it. @@ -287,104 +297,294 @@ static inline void aligned_free_char(char *mem_block) { #endif // MSC_VER #endif // SIMDJSON_COMMON_DEFS_H -/* end file include/simdjson/common_defs.h */ +/* end file include/simdjson/portability.h */ +#include +#include + +namespace simdjson { + +/** + * All possible errors returned by simdjson. + */ +enum error_code { + SUCCESS = 0, ///< No error + SUCCESS_AND_HAS_MORE, ///< No error and buffer still has more data + CAPACITY, ///< This parser can't support a document that big + MEMALLOC, ///< Error allocating memory, most likely out of memory + TAPE_ERROR, ///< Something went wrong while writing to the tape (stage 2), this is a generic error + DEPTH_ERROR, ///< Your document exceeds the user-specified depth limitation + STRING_ERROR, ///< Problem while parsing a string + T_ATOM_ERROR, ///< Problem while parsing an atom starting with the letter 't' + F_ATOM_ERROR, ///< Problem while parsing an atom starting with the letter 'f' + N_ATOM_ERROR, ///< Problem while parsing an atom starting with the letter 'n' + NUMBER_ERROR, ///< Problem while parsing a number + UTF8_ERROR, ///< the input is not valid UTF-8 + UNINITIALIZED, ///< unknown error, or uninitialized document + EMPTY, ///< no structural element found + UNESCAPED_CHARS, ///< found unescaped characters in a string. + UNCLOSED_STRING, ///< missing quote at the end + UNSUPPORTED_ARCHITECTURE, ///< unsupported architecture + INCORRECT_TYPE, ///< JSON element has a different type than user expected + NUMBER_OUT_OF_RANGE, ///< JSON number does not fit in 64 bits + NO_SUCH_FIELD, ///< JSON field not found in object + IO_ERROR, ///< Error reading a file + UNEXPECTED_ERROR, ///< indicative of a bug in simdjson + /** @private Number of error codes */ + NUM_ERROR_CODES +}; + +/** + * Get the error message for the given error code. + * + * auto [doc, error] = document::parse("foo"); + * if (error) { printf("Error: %s\n", error_message(error)); } + * + * @return The error message. + */ +inline const char *error_message(error_code error) noexcept; + +/** + * Write the error message to the output stream + */ +inline std::ostream& operator<<(std::ostream& out, error_code error) noexcept; + +/** + * Exception thrown when an exception-supporting simdjson method is called + */ +struct simdjson_error : public std::exception { + /** + * Create an exception from a simdjson error code. + * @param error The error code + */ + simdjson_error(error_code error) noexcept : _error{error} { } + /** The error message */ + const char *what() const noexcept { return error_message(error()); } + /** The error code */ + error_code error() const noexcept { return _error; } +private: + /** The error code that was used */ + error_code _error; +}; + +/** + * The result of a simd operation that could fail. + * + * Gives the option of reading error codes, or throwing an exception by casting to the desired result. + */ +template +struct simdjson_result : public std::pair { + /** + * The error. + */ + error_code error() const { return this->second; } + +#if SIMDJSON_EXCEPTIONS + + /** + * The value of the function. + * + * @throw simdjson_error if there was an error. + */ + T get() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return this->first; + }; + + /** + * Cast to the value (will throw on error). + * + * @throw simdjson_error if there was an error. + */ + operator T() noexcept(false) { return get(); } + +#endif // SIMDJSON_EXCEPTIONS + + /** + * Create a new error result. + */ + simdjson_result(error_code _error) noexcept : std::pair({}, _error) {} + + /** + * Create a new successful result. + */ + simdjson_result(T _value) noexcept : std::pair(_value, SUCCESS) {} + + /** + * Create a new result with both things (use if you don't want to branch when creating the result). + */ + simdjson_result(T value, error_code error) noexcept : std::pair(value, error) {} +}; + +/** + * The result of a simd operation that could fail. + * + * This class is for values that must be *moved*, like padded_string and document. + * + * Gives the option of reading error codes, or throwing an exception by casting to the desired result. + */ +template +struct simdjson_move_result : std::pair { + /** + * Move the value and the error to the provided variables. + */ + void tie(T& t, error_code & e) { + // on the clang compiler that comes with current macOS (Apple clang version 11.0.0), + // std::tie(this->json, error) = padded_string::load(filename); + // fails with "benchmark/benchmarker.h:266:33: error: no viable overloaded '='"" + t = std::move(this->first); + e = std::move(this->second); + } + /** + * The error. + */ + error_code error() const { return this->second; } + +#if SIMDJSON_EXCEPTIONS + + /** + * The value of the function. + * + * @throw simdjson_error if there was an error. + */ + T move() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return std::move(this->first); + }; + + /** + * Cast to the value (will throw on error). + * + * @throw simdjson_error if there was an error. + */ + operator T() noexcept(false) { return move(); } + +#endif + + /** + * Create a new error result. + */ + simdjson_move_result(error_code error) noexcept : std::pair(T(), error) {} + + /** + * Create a new successful result. + */ + simdjson_move_result(T value) noexcept : std::pair(std::move(value), SUCCESS) {} + + /** + * Create a new result with both things (use if you don't want to branch when creating the result). + */ + simdjson_move_result(T value, error_code error) noexcept : std::pair(std::move(value), error) {} +}; + +/** + * @deprecated This is an alias and will be removed, use error_code instead + */ +using ErrorValues = error_code; + +/** + * @deprecated Error codes should be stored and returned as `error_code`, use `error_message()` instead. + */ +inline const std::string &error_message(int error) noexcept; + +} // namespace simdjson + +#endif // SIMDJSON_ERROR_H +/* end file include/simdjson/portability.h */ +/* begin file include/simdjson/padded_string.h */ +#ifndef SIMDJSON_PADDED_STRING_H +#define SIMDJSON_PADDED_STRING_H #include #include #include - namespace simdjson { -// low-level function to allocate memory with padding so we can read past the -// "length" bytes safely. if you must provide a pointer to some data, create it -// with this function: length is the max. size in bytes of the string caller is -// responsible to free the memory (free(...)) -inline char *allocate_padded_buffer(size_t length) noexcept { - // we could do a simple malloc - // return (char *) malloc(length + SIMDJSON_PADDING); - // However, we might as well align to cache lines... - size_t totalpaddedlength = length + SIMDJSON_PADDING; - char *padded_buffer = aligned_malloc_char(64, totalpaddedlength); -#ifndef NDEBUG - if (padded_buffer == nullptr) { - return nullptr; - } -#endif // NDEBUG - memset(padded_buffer + length, 0, totalpaddedlength - length); - return padded_buffer; -} // allocate_padded_buffer -// Simple string with padded allocation. -// We deliberately forbid copies, users should rely on swap or move -// constructors. +/** + * String with extra allocation for ease of use with document::parser::parse() + * + * This is a move-only class, it cannot be copied. + */ struct padded_string final { - explicit padded_string() noexcept : viable_size(0), data_ptr(nullptr) {} + /** + * Create a new, empty padded string. + */ + explicit inline padded_string() noexcept; + /** + * Create a new padded string buffer. + * + * @param length the size of the string. + */ + explicit inline padded_string(size_t length) noexcept; + /** + * Create a new padded string by copying the given input. + * + * @param data the buffer to copy + * @param length the number of bytes to copy + */ + explicit inline padded_string(const char *data, size_t length) noexcept; + /** + * Create a new padded string by copying the given input. + * + * @param str_ the string to copy + */ + inline padded_string(const std::string & str_ ) noexcept; + /** + * Create a new padded string by copying the given input. + * + * @param str_ the string to copy + */ + inline padded_string(std::string_view sv_) noexcept; + /** + * Move one padded string into another. + * + * The original padded string will be reduced to zero capacity. + * + * @param o the string to move. + */ + inline padded_string(padded_string &&o) noexcept; + /** + * Move one padded string into another. + * + * The original padded string will be reduced to zero capacity. + * + * @param o the string to move. + */ + inline padded_string &operator=(padded_string &&o) noexcept; + inline void swap(padded_string &o) noexcept; + ~padded_string() noexcept; - explicit padded_string(size_t length) noexcept - : viable_size(length), data_ptr(allocate_padded_buffer(length)) { - if (data_ptr != nullptr) - data_ptr[length] = '\0'; // easier when you need a c_str - } + /** + * The length of the string. + * + * Does not include padding. + */ + size_t size() const noexcept; - explicit padded_string(const char *data, size_t length) noexcept - : viable_size(length), data_ptr(allocate_padded_buffer(length)) { - if ((data != nullptr) and (data_ptr != nullptr)) { - memcpy(data_ptr, data, length); - data_ptr[length] = '\0'; // easier when you need a c_str - } - } + /** + * The length of the string. + * + * Does not include padding. + */ + size_t length() const noexcept; - // note: do not pass std::string arguments by value - padded_string(const std::string & str_ ) noexcept - : viable_size(str_.size()), data_ptr(allocate_padded_buffer(str_.size())) { - if (data_ptr != nullptr) { - memcpy(data_ptr, str_.data(), str_.size()); - data_ptr[str_.size()] = '\0'; // easier when you need a c_str - } - } + /** + * The string data. + **/ + const char *data() const noexcept; - // note: do pass std::string_view arguments by value - padded_string(std::string_view sv_) noexcept - : viable_size(sv_.size()), data_ptr(allocate_padded_buffer(sv_.size())) { - if (data_ptr != nullptr) { - memcpy(data_ptr, sv_.data(), sv_.size()); - data_ptr[sv_.size()] = '\0'; // easier when you need a c_str - } - } + /** + * The string data. + **/ + char *data() noexcept; - padded_string(padded_string &&o) noexcept - : viable_size(o.viable_size), data_ptr(o.data_ptr) { - o.data_ptr = nullptr; // we take ownership - } - - padded_string &operator=(padded_string &&o) { - aligned_free_char(data_ptr); - data_ptr = o.data_ptr; - viable_size = o.viable_size; - o.data_ptr = nullptr; // we take ownership - o.viable_size = 0; - return *this; - } - - void swap(padded_string &o) { - size_t tmp_viable_size = viable_size; - char *tmp_data_ptr = data_ptr; - viable_size = o.viable_size; - data_ptr = o.data_ptr; - o.data_ptr = tmp_data_ptr; - o.viable_size = tmp_viable_size; - } - - ~padded_string() { - aligned_free_char(data_ptr); - } - - size_t size() const { return viable_size; } - - size_t length() const { return viable_size; } - - char *data() const { return data_ptr; } + /** + * Load this padded string from a file. + * + * @param path the path to the file. + **/ + inline static simdjson_move_result load(const std::string &path) noexcept; private: padded_string &operator=(const padded_string &o) = delete; @@ -397,8 +597,18 @@ private: } // namespace simdjson -#endif -/* end file include/simdjson/common_defs.h */ +namespace simdjson::internal { + +// low-level function to allocate memory with padding so we can read past the +// "length" bytes safely. if you must provide a pointer to some data, create it +// with this function: length is the max. size in bytes of the string caller is +// responsible to free the memory (free(...)) +inline char *allocate_padded_buffer(size_t length) noexcept; + +} // namespace simdjson::internal; + +#endif // SIMDJSON_PADDED_STRING_H +/* end file include/simdjson/padded_string.h */ /* begin file include/simdjson/implementation.h */ #ifndef SIMDJSON_IMPLEMENTATION_H #define SIMDJSON_IMPLEMENTATION_H @@ -415,6 +625,7 @@ private: #include #include #include +#include /* begin file include/simdjson/simdjson.h */ /** * @file @@ -427,8 +638,11 @@ private: #endif // SIMDJSON_H /* end file include/simdjson/simdjson.h */ -#define JSON_VALUE_MASK 0x00FFFFFFFFFFFFFF -#define DEFAULT_MAX_DEPTH 1024 // a JSON document with a depth exceeding 1024 is probably de facto invalid +namespace simdjson::internal { +constexpr const uint64_t JSON_VALUE_MASK = 0x00FFFFFFFFFFFFFF; +enum class tape_type; +class tape_ref; +} // namespace simdjson::internal namespace simdjson { @@ -464,17 +678,23 @@ public: document &operator=(document &&other) noexcept = default; document &operator=(const document &) = delete; // Disallow copying + /** The default batch size for parse_many and load_many */ + static constexpr size_t DEFAULT_BATCH_SIZE = 1000000; + // Nested classes class element; class array; class object; class key_value_pair; class parser; + class stream; - template - class element_result; + class doc_move_result; class doc_result; - class doc_ref_result; + class element_result; + class array_result; + class object_result; + class stream_result; // Nested classes. See definitions later in file. using iterator = document_iterator; @@ -495,20 +715,23 @@ public: * Get the root element of this document. */ operator element() const noexcept; + +#if SIMDJSON_EXCEPTIONS /** * Read the root element of this document as a JSON array. * * @return The JSON array. - * @exception invalid_json(UNEXPECTED_TYPE) if the JSON element is not an array + * @exception simdjson_error(UNEXPECTED_TYPE) if the JSON element is not an array */ operator array() const noexcept(false); /** * Read this element as a JSON object (key/value pairs). * * @return The JSON object. - * @exception invalid_json(UNEXPECTED_TYPE) if the JSON element is not an object + * @exception simdjson_error(UNEXPECTED_TYPE) if the JSON element is not an object */ operator object() const noexcept(false); +#endif // SIMDJSON_EXCEPTIONS /** * Get the value associated with the given key. @@ -537,14 +760,6 @@ public: */ element_result operator[](const char *s) const noexcept; - /** - * Print this JSON to a std::ostream. - * - * @param os the stream to output to. - * @param max_depth the maximum JSON depth to output. - * @return false if the tape is likely wrong (e.g., you did not parse a valid JSON). - */ - bool print_json(std::ostream &os, size_t max_depth=DEFAULT_MAX_DEPTH) const noexcept; /** * Dump the raw tape for debugging. * @@ -553,6 +768,25 @@ public: */ bool dump_raw_tape(std::ostream &os) const noexcept; + /** + * Load a JSON document from a file and return it. + * + * document doc = document::load("jsonexamples/twitter.json"); + * + * ### Parser Capacity + * + * If the parser's current capacity is less than the file length, it will allocate enough capacity + * to handle it (up to max_capacity). + * + * @param path The path to load. + * @return The document, or an error: + * - IO_ERROR if there was an error opening or reading the file. + * - MEMALLOC if the parser does not have enough capacity and memory allocation fails. + * - CAPACITY if the parser does not have enough capacity and len > max_capacity. + * - other json errors if parsing fails. + */ + inline static doc_move_result load(const std::string& path) noexcept; + /** * Parse a JSON document and return a reference to it. * @@ -567,7 +801,7 @@ public: * @param realloc_if_needed Whether to reallocate and enlarge the JSON buffer to add padding. * @return the document, or an error if the JSON is invalid. */ - inline static doc_result parse(const uint8_t *buf, size_t len, bool realloc_if_needed = true) noexcept; + inline static doc_move_result parse(const uint8_t *buf, size_t len, bool realloc_if_needed = true) noexcept; /** * Parse a JSON document. @@ -583,7 +817,7 @@ public: * @param realloc_if_needed Whether to reallocate and enlarge the JSON buffer to add padding. * @return the document, or an error if the JSON is invalid. */ - really_inline static doc_result parse(const char *buf, size_t len, bool realloc_if_needed = true) noexcept; + really_inline static doc_move_result parse(const char *buf, size_t len, bool realloc_if_needed = true) noexcept; /** * Parse a JSON document. @@ -596,7 +830,7 @@ public: * a new string will be created with the extra padding. * @return the document, or an error if the JSON is invalid. */ - really_inline static doc_result parse(const std::string &s) noexcept; + really_inline static doc_move_result parse(const std::string &s) noexcept; /** * Parse a JSON document. @@ -604,26 +838,29 @@ public: * @param s The JSON to parse. * @return the document, or an error if the JSON is invalid. */ - really_inline static doc_result parse(const padded_string &s) noexcept; + really_inline static doc_move_result parse(const padded_string &s) noexcept; // We do not want to allow implicit conversion from C string to std::string. - doc_ref_result parse(const char *buf, bool realloc_if_needed = true) noexcept = delete; + doc_result parse(const char *buf, bool realloc_if_needed = true) noexcept = delete; std::unique_ptr tape; std::unique_ptr string_buf;// should be at least byte_capacity private: - class tape_ref; - enum class tape_type; - bool set_capacity(size_t len); + inline error_code set_capacity(size_t len) noexcept; + template + friend class minify; }; // class document +template +class minify; + /** * A parsed, *owned* document, or an error if the parse failed. * * document &doc = document::parse(json); * - * Returns an owned `document`. When the doc_result (or the document retrieved from it) goes out of + * Returns an owned `document`. When the doc_move_result (or the document retrieved from it) goes out of * scope, the document's memory is deallocated. * * ## Error Codes vs. Exceptions @@ -640,38 +877,58 @@ private: * document doc = document::parse(json); * */ -class document::doc_result { +class document::doc_move_result : public simdjson_move_result { public: - /** - * The parsed document. This is *invalid* if there is an error. - */ - document doc; - /** - * The error code, or SUCCESS (0) if there is no error. - */ - error_code error; /** - * Return the document, or throw an exception if it is invalid. + * Read this document as a JSON objec. * - * @return the document. - * @exception invalid_json if the document is invalid or there was an error parsing it. + * @return The object value, or: + * - UNEXPECTED_TYPE if the JSON document is not an object */ - operator document() noexcept(false); + inline object_result as_object() const noexcept; /** - * Get the error message for the error. + * Read this document as a JSON array. + * + * @return The array value, or: + * - UNEXPECTED_TYPE if the JSON document is not an array */ - const std::string &get_error_message() const noexcept; + inline array_result as_array() const noexcept; - ~doc_result() noexcept=default; + /** + * Get the value associated with the given key. + * + * The key will be matched against **unescaped** JSON: + * + * document::parse(R"({ "a\n": 1 })")["a\n"].as_uint64_t().value == 1 + * document::parse(R"({ "a\n": 1 })")["a\\n"].as_uint64_t().error == NO_SUCH_FIELD + * + * @return The value associated with this field, or: + * - NO_SUCH_FIELD if the field does not exist in the object + * - UNEXPECTED_TYPE if the document is not an object + */ + inline element_result operator[](const std::string_view &key) const noexcept; + /** + * Get the value associated with the given key. + * + * The key will be matched against **unescaped** JSON: + * + * document::parse(R"({ "a\n": 1 })")["a\n"].as_uint64_t().value == 1 + * document::parse(R"({ "a\n": 1 })")["a\\n"].as_uint64_t().error == NO_SUCH_FIELD + * + * @return The value associated with this field, or: + * - NO_SUCH_FIELD if the field does not exist in the object + * - UNEXPECTED_TYPE if the document is not an object + */ + inline element_result operator[](const char *key) const noexcept; -private: - doc_result(document &&_doc, error_code _error) noexcept; - doc_result(document &&_doc) noexcept; - doc_result(error_code _error) noexcept; + ~doc_move_result() noexcept=default; + doc_move_result(document &&doc, error_code error) noexcept; + doc_move_result(document &&doc) noexcept; + doc_move_result(error_code error) noexcept; friend class document; -}; // class document::doc_result +}; // class document::doc_move_result /** * A parsed document reference, or an error if the parse failed. @@ -704,76 +961,102 @@ private: * document &doc = document::parse(json); * */ -class document::doc_ref_result { +class document::doc_result : public simdjson_result { public: /** - * The parsed document. This is *invalid* if there is an error. - */ - document &doc; - /** - * The error code, or SUCCESS (0) if there is no error. - */ - error_code error; - - /** - * A reference to the document, or throw an exception if it is invalid. + * Read this document as a JSON objec. * - * @return the document. - * @exception invalid_json if the document is invalid or there was an error parsing it. + * @return The object value, or: + * - UNEXPECTED_TYPE if the JSON document is not an object */ - operator document&() noexcept(false); + inline object_result as_object() const noexcept; /** - * Get the error message for the error. + * Read this document as a JSON array. + * + * @return The array value, or: + * - UNEXPECTED_TYPE if the JSON document is not an array */ - const std::string &get_error_message() const noexcept; + inline array_result as_array() const noexcept; - ~doc_ref_result()=default; + /** + * Get the value associated with the given key. + * + * The key will be matched against **unescaped** JSON: + * + * document::parse(R"({ "a\n": 1 })")["a\n"].as_uint64_t().value == 1 + * document::parse(R"({ "a\n": 1 })")["a\\n"].as_uint64_t().error == NO_SUCH_FIELD + * + * @return The value associated with this field, or: + * - NO_SUCH_FIELD if the field does not exist in the object + * - UNEXPECTED_TYPE if the document is not an object + */ + inline element_result operator[](const std::string_view &key) const noexcept; -private: - doc_ref_result(document &_doc, error_code _error) noexcept; + /** + * Get the value associated with the given key. + * + * The key will be matched against **unescaped** JSON: + * + * document::parse(R"({ "a\n": 1 })")["a\n"].as_uint64_t().value == 1 + * document::parse(R"({ "a\n": 1 })")["a\\n"].as_uint64_t().error == NO_SUCH_FIELD + * + * @return The value associated with this field, or: + * - NO_SUCH_FIELD if the field does not exist in the object + * - UNEXPECTED_TYPE if the document is not an object + */ + inline element_result operator[](const char *key) const noexcept; + + ~doc_result()=default; + doc_result(document &doc, error_code error) noexcept; friend class document::parser; -}; // class document::doc_ref_result + friend class document::stream; +}; // class document::doc_result -/** - * The possible types in the tape. Internal only. +namespace internal { + /** + * The possible types in the tape. Internal only. + */ + enum class tape_type { + ROOT = 'r', + START_ARRAY = '[', + START_OBJECT = '{', + END_ARRAY = ']', + END_OBJECT = '}', + STRING = '"', + INT64 = 'l', + UINT64 = 'u', + DOUBLE = 'd', + TRUE_VALUE = 't', + FALSE_VALUE = 'f', + NULL_VALUE = 'n' + }; + + /** + * A reference to an element on the tape. Internal only. */ -enum class document::tape_type { - ROOT = 'r', - START_ARRAY = '[', - START_OBJECT = '{', - END_ARRAY = ']', - END_OBJECT = '}', - STRING = '"', - INT64 = 'l', - UINT64 = 'u', - DOUBLE = 'd', - TRUE_VALUE = 't', - FALSE_VALUE = 'f', - NULL_VALUE = 'n' -}; + class tape_ref { + protected: + really_inline tape_ref() noexcept; + really_inline tape_ref(const document *_doc, size_t _json_index) noexcept; + inline size_t after_element() const noexcept; + really_inline tape_type type() const noexcept; + really_inline uint64_t tape_value() const noexcept; + template + really_inline T next_tape_value() const noexcept; + inline std::string_view get_string_view() const noexcept; -/** - * A reference to an element on the tape. Internal only. - */ -class document::tape_ref { -protected: - really_inline tape_ref() noexcept; - really_inline tape_ref(const document *_doc, size_t _json_index) noexcept; - inline size_t after_element() const noexcept; - really_inline tape_type type() const noexcept; - really_inline uint64_t tape_value() const noexcept; - template - really_inline T next_tape_value() const noexcept; + /** The document this element references. */ + const document *doc; - /** The document this element references. */ - const document *doc; + /** The index of this element on `doc.tape[]` */ + size_t json_index; - /** The index of this element on `doc.tape[]` */ - size_t json_index; - - friend class document::key_value_pair; -}; + friend class simdjson::document::key_value_pair; + template + friend class simdjson::minify; + }; +} // namespace simdjson::internal /** * A JSON element. @@ -781,8 +1064,11 @@ protected: * References an element in a JSON document, representing a JSON null, boolean, string, number, * array or object. */ -class document::element : protected document::tape_ref { +class document::element : protected internal::tape_ref { public: + /** Create a new, invalid element. */ + really_inline element() noexcept; + /** Whether this element is a json `null`. */ really_inline bool is_null() const noexcept; /** Whether this is a JSON `true` or `false` */ @@ -870,11 +1156,12 @@ public: */ inline object_result as_object() const noexcept; +#if SIMDJSON_EXCEPTIONS /** * Read this element as a boolean. * * @return The boolean value - * @exception invalid_json(UNEXPECTED_TYPE) if the JSON element is not a boolean. + * @exception simdjson_error(UNEXPECTED_TYPE) if the JSON element is not a boolean. */ inline operator bool() const noexcept(false); @@ -885,7 +1172,7 @@ public: * an actual string. * * @return The string value. - * @exception invalid_json(UNEXPECTED_TYPE) if the JSON element is not a string. + * @exception simdjson_error(UNEXPECTED_TYPE) if the JSON element is not a string. */ inline explicit operator const char*() const noexcept(false); @@ -896,7 +1183,7 @@ public: * an actual string. * * @return The string value. - * @exception invalid_json(UNEXPECTED_TYPE) if the JSON element is not a string. + * @exception simdjson_error(UNEXPECTED_TYPE) if the JSON element is not a string. */ inline operator std::string_view() const noexcept(false); @@ -904,40 +1191,41 @@ public: * Read this element as an unsigned integer. * * @return The integer value. - * @exception invalid_json(UNEXPECTED_TYPE) if the JSON element is not an integer - * @exception invalid_json(NUMBER_OUT_OF_RANGE) if the integer doesn't fit in 64 bits or is negative + * @exception simdjson_error(UNEXPECTED_TYPE) if the JSON element is not an integer + * @exception simdjson_error(NUMBER_OUT_OF_RANGE) if the integer doesn't fit in 64 bits or is negative */ inline operator uint64_t() const noexcept(false); /** * Read this element as an signed integer. * * @return The integer value. - * @exception invalid_json(UNEXPECTED_TYPE) if the JSON element is not an integer - * @exception invalid_json(NUMBER_OUT_OF_RANGE) if the integer doesn't fit in 64 bits + * @exception simdjson_error(UNEXPECTED_TYPE) if the JSON element is not an integer + * @exception simdjson_error(NUMBER_OUT_OF_RANGE) if the integer doesn't fit in 64 bits */ inline operator int64_t() const noexcept(false); /** * Read this element as an double. * * @return The double value. - * @exception invalid_json(UNEXPECTED_TYPE) if the JSON element is not a number - * @exception invalid_json(NUMBER_OUT_OF_RANGE) if the integer doesn't fit in 64 bits or is negative + * @exception simdjson_error(UNEXPECTED_TYPE) if the JSON element is not a number + * @exception simdjson_error(NUMBER_OUT_OF_RANGE) if the integer doesn't fit in 64 bits or is negative */ inline operator double() const noexcept(false); /** * Read this element as a JSON array. * * @return The JSON array. - * @exception invalid_json(UNEXPECTED_TYPE) if the JSON element is not an array + * @exception simdjson_error(UNEXPECTED_TYPE) if the JSON element is not an array */ inline operator document::array() const noexcept(false); /** * Read this element as a JSON object (key/value pairs). * * @return The JSON object. - * @exception invalid_json(UNEXPECTED_TYPE) if the JSON element is not an object + * @exception simdjson_error(UNEXPECTED_TYPE) if the JSON element is not an object */ inline operator document::object() const noexcept(false); +#endif // SIMDJSON_EXCEPTIONS /** * Get the value associated with the given key. @@ -952,6 +1240,7 @@ public: * - UNEXPECTED_TYPE if the document is not an object */ inline element_result operator[](const std::string_view &s) const noexcept; + /** * Get the value associated with the given key. * @@ -967,18 +1256,21 @@ public: inline element_result operator[](const char *s) const noexcept; private: - really_inline element() noexcept; really_inline element(const document *_doc, size_t _json_index) noexcept; friend class document; - template friend class document::element_result; + template + friend class minify; }; /** * Represents a JSON array. */ -class document::array : protected document::tape_ref { +class document::array : protected internal::tape_ref { public: + /** Create a new, invalid array */ + really_inline array() noexcept; + class iterator : tape_ref { public: /** @@ -1016,19 +1308,22 @@ public: inline iterator end() const noexcept; private: - really_inline array() noexcept; really_inline array(const document *_doc, size_t _json_index) noexcept; friend class document::element; - template friend class document::element_result; + template + friend class minify; }; /** * Represents a JSON object. */ -class document::object : protected document::tape_ref { +class document::object : protected internal::tape_ref { public: - class iterator : protected document::tape_ref { + /** Create a new, invalid object */ + really_inline object() noexcept; + + class iterator : protected internal::tape_ref { public: /** * Get the actual key/value pair @@ -1088,6 +1383,7 @@ public: * - NO_SUCH_FIELD if the field does not exist in the object */ inline element_result operator[](const std::string_view &s) const noexcept; + /** * Get the value associated with the given key. * @@ -1102,11 +1398,11 @@ public: inline element_result operator[](const char *s) const noexcept; private: - really_inline object() noexcept; really_inline object(const document *_doc, size_t _json_index) noexcept; friend class document::element; - template friend class document::element_result; + template + friend class minify; }; /** @@ -1123,47 +1419,11 @@ private: }; -/** - * The result of a JSON navigation or conversion, or an error (if the navigation or conversion - * failed). Allows the user to pick whether to use exceptions or not. - * - * Use like this to avoid exceptions: - * - * auto [str, error] = document::parse(json).root().as_string(); - * if (error) { exit(1); } - * cout << str; - * - * Use like this if you'd prefer to use exceptions: - * - * string str = document::parse(json).root(); - * cout << str; - * - */ -template -class document::element_result { + /** The result of a JSON navigation that may fail. */ +class document::element_result : public simdjson_result { public: - /** The value */ - T value; - /** The error code (or 0 if there is no error) */ - error_code error; - - inline operator T() const noexcept(false); - -private: - really_inline element_result(T value) noexcept; - really_inline element_result(error_code _error) noexcept; - friend class document; - friend class element; -}; - -// Add exception-throwing navigation / conversion methods to element_result -template<> -class document::element_result { -public: - /** The value */ - element value; - /** The error code (or 0 if there is no error) */ - error_code error; + really_inline element_result(element value) noexcept; + really_inline element_result(error_code error) noexcept; /** Whether this is a JSON `null` */ inline simdjson_result is_null() const noexcept; @@ -1176,6 +1436,10 @@ public: inline array_result as_array() const noexcept; inline object_result as_object() const noexcept; + inline element_result operator[](const std::string_view &s) const noexcept; + inline element_result operator[](const char *s) const noexcept; + +#if SIMDJSON_EXCEPTIONS inline operator bool() const noexcept(false); inline explicit operator const char*() const noexcept(false); inline operator std::string_view() const noexcept(false); @@ -1184,67 +1448,41 @@ public: inline operator double() const noexcept(false); inline operator array() const noexcept(false); inline operator object() const noexcept(false); - - inline element_result operator[](const std::string_view &s) const noexcept; - inline element_result operator[](const char *s) const noexcept; - -private: - really_inline element_result(element value) noexcept; - really_inline element_result(error_code _error) noexcept; - friend class document; - friend class element; +#endif // SIMDJSON_EXCEPTIONS }; -// Add exception-throwing navigation methods to array_result -template<> -class document::array_result { +/** The result of a JSON conversion that may fail. */ +class document::array_result : public simdjson_result { public: - /** The value */ - array value; - /** The error code (or 0 if there is no error) */ - error_code error; - - inline operator array() const noexcept(false); + really_inline array_result(array value) noexcept; + really_inline array_result(error_code error) noexcept; +#if SIMDJSON_EXCEPTIONS inline array::iterator begin() const noexcept(false); inline array::iterator end() const noexcept(false); - -private: - really_inline element_result(array value) noexcept; - really_inline element_result(error_code _error) noexcept; - friend class document; - friend class element; +#endif // SIMDJSON_EXCEPTIONS }; -// Add exception-throwing navigation methods to object_result -template<> -class document::object_result { +/** The result of a JSON conversion that may fail. */ +class document::object_result : public simdjson_result { public: - /** The value */ - object value; - /** The error code (or 0 if there is no error) */ - error_code error; - - inline operator object() const noexcept(false); - - inline object::iterator begin() const noexcept(false); - inline object::iterator end() const noexcept(false); + really_inline object_result(object value) noexcept; + really_inline object_result(error_code error) noexcept; inline element_result operator[](const std::string_view &s) const noexcept; inline element_result operator[](const char *s) const noexcept; -private: - really_inline element_result(object value) noexcept; - really_inline element_result(error_code _error) noexcept; - friend class document; - friend class element; +#if SIMDJSON_EXCEPTIONS + inline object::iterator begin() const noexcept(false); + inline object::iterator end() const noexcept(false); +#endif // SIMDJSON_EXCEPTIONS }; /** * A persistent document parser. * - * Use this if you intend to parse more than one document. It holds the internal memory necessary - * to do parsing, as well as memory for a single document that is overwritten on each parse. + * The parser is designed to be reused, holding the internal buffers necessary to do parsing, + * as well as memory for a single document. The parsed document is overwritten on each parse. * * This class cannot be copied, only moved, to avoid unintended allocations. * @@ -1253,9 +1491,20 @@ private: class document::parser { public: /** - * Create a JSON parser with zero capacity. Call allocate_capacity() to initialize it. + * Create a JSON parser. + * + * The new parser will have zero capacity. + * + * @param max_capacity The maximum document length the parser can automatically handle. The parser + * will allocate more capacity on an as needed basis (when it sees documents too big to handle) + * up to this amount. The parser still starts with zero capacity no matter what this number is: + * to allocate an initial capacity, call set_capacity() after constructing the parser. Defaults + * to SIMDJSON_MAXSIZE_BYTES (the largest single document simdjson can process). + * @param max_depth The maximum depth--number of nested objects and arrays--this parser can handle. + * This will not be allocated until parse() is called for the first time. Defaults to + * DEFAULT_MAX_DEPTH. */ - parser()=default; + really_inline parser(size_t max_capacity = SIMDJSON_MAXSIZE_BYTES, size_t max_depth = DEFAULT_MAX_DEPTH) noexcept; ~parser()=default; /** @@ -1274,96 +1523,540 @@ public: parser &operator=(const document::parser &) = delete; // Disallow copying /** - * Parse a JSON document and return a reference to it. + * Load a JSON document from a file and return a reference to it. + * + * document::parser parser; + * const document &doc = parser.load("jsonexamples/twitter.json"); + * + * ### IMPORTANT: Document Lifetime * * The JSON document still lives in the parser: this is the most efficient way to parse JSON * documents because it reuses the same buffers, but you *must* use the document before you * destroy the parser or call parse() again. * + * ### Parser Capacity + * + * If the parser's current capacity is less than the file length, it will allocate enough capacity + * to handle it (up to max_capacity). + * + * @param path The path to load. + * @return The document, or an error: + * - IO_ERROR if there was an error opening or reading the file. + * - MEMALLOC if the parser does not have enough capacity and memory allocation fails. + * - CAPACITY if the parser does not have enough capacity and len > max_capacity. + * - other json errors if parsing fails. + */ + inline doc_result load(const std::string& path) noexcept; + + /** + * Load a file containing many JSON documents. + * + * document::parser parser; + * for (const document &doc : parser.parse_many(path)) { + * cout << std::string(doc["title"]) << endl; + * } + * + * ### Format + * + * The file must contain a series of one or more JSON documents, concatenated into a single + * buffer, separated by whitespace. It effectively parses until it has a fully valid document, + * then starts parsing the next document at that point. (It does this with more parallelism and + * lookahead than you might think, though.) + * + * documents that consist of an object or array may omit the whitespace between them, concatenating + * with no separator. documents that consist of a single primitive (i.e. documents that are not + * arrays or objects) MUST be separated with whitespace. + * + * ### Error Handling + * + * All errors are returned during iteration: if there is a global error such as memory allocation, + * it will be yielded as the first result. Iteration always stops after the first error. + * + * As with all other simdjson methods, non-exception error handling is readily available through + * the same interface, requiring you to check the error before using the document: + * + * document::parser parser; + * for (auto [doc, error] : parser.load_many(path)) { + * if (error) { cerr << error << endl; exit(1); } + * cout << std::string(doc["title"]) << endl; + * } + * + * ### Threads + * + * When compiled with SIMDJSON_THREADS_ENABLED, this method will use a single thread under the + * hood to do some lookahead. + * + * ### Parser Capacity + * + * If the parser's current capacity is less than batch_size, it will allocate enough capacity + * to handle it (up to max_capacity). + * + * @param s The concatenated JSON to parse. Must have at least len + SIMDJSON_PADDING allocated bytes. + * @param batch_size The batch size to use. MUST be larger than the largest document. The sweet + * spot is cache-related: small enough to fit in cache, yet big enough to + * parse as many documents as possible in one tight loop. + * Defaults to 10MB, which has been a reasonable sweet spot in our tests. + * @return The stream. If there is an error, it will be returned during iteration. An empty input + * will yield 0 documents rather than an EMPTY error. Errors: + * - IO_ERROR if there was an error opening or reading the file. + * - MEMALLOC if the parser does not have enough capacity and memory allocation fails. + * - CAPACITY if the parser does not have enough capacity and batch_size > max_capacity. + * - other json errors if parsing fails. + */ + inline document::stream load_many(const std::string& path, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept; + + /** + * Parse a JSON document and return a temporary reference to it. + * + * document::parser parser; + * const document &doc = parser.parse(buf, len); + * + * ### IMPORTANT: Document Lifetime + * + * The JSON document still lives in the parser: this is the most efficient way to parse JSON + * documents because it reuses the same buffers, but you *must* use the document before you + * destroy the parser or call parse() again. + * + * ### REQUIRED: Buffer Padding + * * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what - * those bytes are initialized to, as long as they are allocated. If realloc_if_needed is true, - * it is assumed that the buffer does *not* have enough padding, and it is reallocated, enlarged - * and copied before parsing. + * those bytes are initialized to, as long as they are allocated. + * + * If realloc_if_needed is true, it is assumed that the buffer does *not* have enough padding, + * and it is copied into an enlarged temporary buffer before parsing. + * + * ### Parser Capacity + * + * If the parser's current capacity is less than len, it will allocate enough capacity + * to handle it (up to max_capacity). * * @param buf The JSON to parse. Must have at least len + SIMDJSON_PADDING allocated bytes, unless * realloc_if_needed is true. * @param len The length of the JSON. * @param realloc_if_needed Whether to reallocate and enlarge the JSON buffer to add padding. - * @return the document, or an error if the JSON is invalid. + * @return The document, or an error: + * - MEMALLOC if realloc_if_needed is true or the parser does not have enough capacity, + * and memory allocation fails. + * - CAPACITY if the parser does not have enough capacity and len > max_capacity. + * - other json errors if parsing fails. */ - inline doc_ref_result parse(const uint8_t *buf, size_t len, bool realloc_if_needed = true) noexcept; + inline doc_result parse(const uint8_t *buf, size_t len, bool realloc_if_needed = true) noexcept; /** - * Parse a JSON document and return a reference to it. + * Parse a JSON document and return a temporary reference to it. + * + * document::parser parser; + * const document &doc = parser.parse(buf, len); + * + * ### IMPORTANT: Document Lifetime * * The JSON document still lives in the parser: this is the most efficient way to parse JSON * documents because it reuses the same buffers, but you *must* use the document before you * destroy the parser or call parse() again. * + * ### REQUIRED: Buffer Padding + * * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what - * those bytes are initialized to, as long as they are allocated. If realloc_if_needed is true, - * it is assumed that the buffer does *not* have enough padding, and it is reallocated, enlarged - * and copied before parsing. + * those bytes are initialized to, as long as they are allocated. + * + * If realloc_if_needed is true, it is assumed that the buffer does *not* have enough padding, + * and it is copied into an enlarged temporary buffer before parsing. + * + * ### Parser Capacity + * + * If the parser's current capacity is less than len, it will allocate enough capacity + * to handle it (up to max_capacity). * * @param buf The JSON to parse. Must have at least len + SIMDJSON_PADDING allocated bytes, unless * realloc_if_needed is true. * @param len The length of the JSON. * @param realloc_if_needed Whether to reallocate and enlarge the JSON buffer to add padding. - * @return the document, or an error if the JSON is invalid. + * @return The document, or an error: + * - MEMALLOC if realloc_if_needed is true or the parser does not have enough capacity, + * and memory allocation fails. + * - CAPACITY if the parser does not have enough capacity and len > max_capacity. + * - other json errors if parsing fails. */ - really_inline doc_ref_result parse(const char *buf, size_t len, bool realloc_if_needed = true) noexcept; + really_inline doc_result parse(const char *buf, size_t len, bool realloc_if_needed = true) noexcept; /** - * Parse a JSON document and return a reference to it. + * Parse a JSON document and return a temporary reference to it. + * + * document::parser parser; + * const document &doc = parser.parse(s); + * + * ### IMPORTANT: Document Lifetime * * The JSON document still lives in the parser: this is the most efficient way to parse JSON * documents because it reuses the same buffers, but you *must* use the document before you * destroy the parser or call parse() again. * + * ### REQUIRED: Buffer Padding + * * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what - * those bytes are initialized to, as long as they are allocated. If `str.capacity() - str.size() - * < SIMDJSON_PADDING`, the string will be copied to a string with larger capacity before parsing. + * those bytes are initialized to, as long as they are allocated. + * + * If s.capacity() is less than SIMDJSON_PADDING, the string will be copied into an enlarged + * temporary buffer before parsing. + * + * ### Parser Capacity + * + * If the parser's current capacity is less than len, it will allocate enough capacity + * to handle it (up to max_capacity). * * @param s The JSON to parse. Must have at least len + SIMDJSON_PADDING allocated bytes, or * a new string will be created with the extra padding. - * @return the document, or an error if the JSON is invalid. + * @return The document, or an error: + * - MEMALLOC if the string does not have enough padding or the parser does not have + * enough capacity, and memory allocation fails. + * - CAPACITY if the parser does not have enough capacity and len > max_capacity. + * - other json errors if parsing fails. */ - really_inline doc_ref_result parse(const std::string &s) noexcept; + really_inline doc_result parse(const std::string &s) noexcept; /** - * Parse a JSON document and return a reference to it. + * Parse a JSON document and return a temporary reference to it. + * + * document::parser parser; + * const document &doc = parser.parse(s); + * + * ### IMPORTANT: Document Lifetime * * The JSON document still lives in the parser: this is the most efficient way to parse JSON * documents because it reuses the same buffers, but you *must* use the document before you * destroy the parser or call parse() again. * + * ### Parser Capacity + * + * If the parser's current capacity is less than batch_size, it will allocate enough capacity + * to handle it (up to max_capacity). + * * @param s The JSON to parse. - * @return the document, or an error if the JSON is invalid. + * @return The document, or an error: + * - MEMALLOC if the parser does not have enough capacity and memory allocation fails. + * - CAPACITY if the parser does not have enough capacity and len > max_capacity. + * - other json errors if parsing fails. */ - really_inline doc_ref_result parse(const padded_string &s) noexcept; + really_inline doc_result parse(const padded_string &s) noexcept; // We do not want to allow implicit conversion from C string to std::string. - really_inline doc_ref_result parse(const char *buf) noexcept = delete; + really_inline doc_result parse(const char *buf) noexcept = delete; /** - * Current capacity: the largest document this parser can support without reallocating. + * Parse a buffer containing many JSON documents. + * + * document::parser parser; + * for (const document &doc : parser.parse_many(buf, len)) { + * cout << std::string(doc["title"]) << endl; + * } + * + * ### Format + * + * The buffer must contain a series of one or more JSON documents, concatenated into a single + * buffer, separated by whitespace. It effectively parses until it has a fully valid document, + * then starts parsing the next document at that point. (It does this with more parallelism and + * lookahead than you might think, though.) + * + * documents that consist of an object or array may omit the whitespace between them, concatenating + * with no separator. documents that consist of a single primitive (i.e. documents that are not + * arrays or objects) MUST be separated with whitespace. + * + * ### Error Handling + * + * All errors are returned during iteration: if there is a global error such as memory allocation, + * it will be yielded as the first result. Iteration always stops after the first error. + * + * As with all other simdjson methods, non-exception error handling is readily available through + * the same interface, requiring you to check the error before using the document: + * + * document::parser parser; + * for (auto [doc, error] : parser.parse_many(buf, len)) { + * if (error) { cerr << error << endl; exit(1); } + * cout << std::string(doc["title"]) << endl; + * } + * + * ### REQUIRED: Buffer Padding + * + * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what + * those bytes are initialized to, as long as they are allocated. + * + * ### Threads + * + * When compiled with SIMDJSON_THREADS_ENABLED, this method will use a single thread under the + * hood to do some lookahead. + * + * ### Parser Capacity + * + * If the parser's current capacity is less than batch_size, it will allocate enough capacity + * to handle it (up to max_capacity). + * + * @param buf The concatenated JSON to parse. Must have at least len + SIMDJSON_PADDING allocated bytes. + * @param len The length of the concatenated JSON. + * @param batch_size The batch size to use. MUST be larger than the largest document. The sweet + * spot is cache-related: small enough to fit in cache, yet big enough to + * parse as many documents as possible in one tight loop. + * Defaults to 10MB, which has been a reasonable sweet spot in our tests. + * @return The stream. If there is an error, it will be returned during iteration. An empty input + * will yield 0 documents rather than an EMPTY error. Errors: + * - MEMALLOC if the parser does not have enough capacity and memory allocation fails + * - CAPACITY if the parser does not have enough capacity and batch_size > max_capacity. + * - other json errors if parsing fails. + */ + inline stream parse_many(const uint8_t *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept; + + /** + * Parse a buffer containing many JSON documents. + * + * document::parser parser; + * for (const document &doc : parser.parse_many(buf, len)) { + * cout << std::string(doc["title"]) << endl; + * } + * + * ### Format + * + * The buffer must contain a series of one or more JSON documents, concatenated into a single + * buffer, separated by whitespace. It effectively parses until it has a fully valid document, + * then starts parsing the next document at that point. (It does this with more parallelism and + * lookahead than you might think, though.) + * + * documents that consist of an object or array may omit the whitespace between them, concatenating + * with no separator. documents that consist of a single primitive (i.e. documents that are not + * arrays or objects) MUST be separated with whitespace. + * + * ### Error Handling + * + * All errors are returned during iteration: if there is a global error such as memory allocation, + * it will be yielded as the first result. Iteration always stops after the first error. + * + * As with all other simdjson methods, non-exception error handling is readily available through + * the same interface, requiring you to check the error before using the document: + * + * document::parser parser; + * for (auto [doc, error] : parser.parse_many(buf, len)) { + * if (error) { cerr << error << endl; exit(1); } + * cout << std::string(doc["title"]) << endl; + * } + * + * ### REQUIRED: Buffer Padding + * + * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what + * those bytes are initialized to, as long as they are allocated. + * + * ### Threads + * + * When compiled with SIMDJSON_THREADS_ENABLED, this method will use a single thread under the + * hood to do some lookahead. + * + * ### Parser Capacity + * + * If the parser's current capacity is less than batch_size, it will allocate enough capacity + * to handle it (up to max_capacity). + * + * @param buf The concatenated JSON to parse. Must have at least len + SIMDJSON_PADDING allocated bytes. + * @param len The length of the concatenated JSON. + * @param batch_size The batch size to use. MUST be larger than the largest document. The sweet + * spot is cache-related: small enough to fit in cache, yet big enough to + * parse as many documents as possible in one tight loop. + * Defaults to 10MB, which has been a reasonable sweet spot in our tests. + * @return The stream. If there is an error, it will be returned during iteration. An empty input + * will yield 0 documents rather than an EMPTY error. Errors: + * - MEMALLOC if the parser does not have enough capacity and memory allocation fails + * - CAPACITY if the parser does not have enough capacity and batch_size > max_capacity. + * - other json errors if parsing fails + */ + inline stream parse_many(const char *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept; + + /** + * Parse a buffer containing many JSON documents. + * + * document::parser parser; + * for (const document &doc : parser.parse_many(buf, len)) { + * cout << std::string(doc["title"]) << endl; + * } + * + * ### Format + * + * The buffer must contain a series of one or more JSON documents, concatenated into a single + * buffer, separated by whitespace. It effectively parses until it has a fully valid document, + * then starts parsing the next document at that point. (It does this with more parallelism and + * lookahead than you might think, though.) + * + * documents that consist of an object or array may omit the whitespace between them, concatenating + * with no separator. documents that consist of a single primitive (i.e. documents that are not + * arrays or objects) MUST be separated with whitespace. + * + * ### Error Handling + * + * All errors are returned during iteration: if there is a global error such as memory allocation, + * it will be yielded as the first result. Iteration always stops after the first error. + * + * As with all other simdjson methods, non-exception error handling is readily available through + * the same interface, requiring you to check the error before using the document: + * + * document::parser parser; + * for (auto [doc, error] : parser.parse_many(buf, len)) { + * if (error) { cerr << error << endl; exit(1); } + * cout << std::string(doc["title"]) << endl; + * } + * + * ### REQUIRED: Buffer Padding + * + * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what + * those bytes are initialized to, as long as they are allocated. + * + * ### Threads + * + * When compiled with SIMDJSON_THREADS_ENABLED, this method will use a single thread under the + * hood to do some lookahead. + * + * ### Parser Capacity + * + * If the parser's current capacity is less than batch_size, it will allocate enough capacity + * to handle it (up to max_capacity). + * + * @param s The concatenated JSON to parse. Must have at least len + SIMDJSON_PADDING allocated bytes. + * @param batch_size The batch size to use. MUST be larger than the largest document. The sweet + * spot is cache-related: small enough to fit in cache, yet big enough to + * parse as many documents as possible in one tight loop. + * Defaults to 10MB, which has been a reasonable sweet spot in our tests. + * @return The stream. If there is an error, it will be returned during iteration. An empty input + * will yield 0 documents rather than an EMPTY error. Errors: + * - MEMALLOC if the parser does not have enough capacity and memory allocation fails + * - CAPACITY if the parser does not have enough capacity and batch_size > max_capacity. + * - other json errors if parsing fails + */ + inline stream parse_many(const std::string &s, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept; + + /** + * Parse a buffer containing many JSON documents. + * + * document::parser parser; + * for (const document &doc : parser.parse_many(buf, len)) { + * cout << std::string(doc["title"]) << endl; + * } + * + * ### Format + * + * The buffer must contain a series of one or more JSON documents, concatenated into a single + * buffer, separated by whitespace. It effectively parses until it has a fully valid document, + * then starts parsing the next document at that point. (It does this with more parallelism and + * lookahead than you might think, though.) + * + * documents that consist of an object or array may omit the whitespace between them, concatenating + * with no separator. documents that consist of a single primitive (i.e. documents that are not + * arrays or objects) MUST be separated with whitespace. + * + * ### Error Handling + * + * All errors are returned during iteration: if there is a global error such as memory allocation, + * it will be yielded as the first result. Iteration always stops after the first error. + * + * As with all other simdjson methods, non-exception error handling is readily available through + * the same interface, requiring you to check the error before using the document: + * + * document::parser parser; + * for (auto [doc, error] : parser.parse_many(buf, len)) { + * if (error) { cerr << error << endl; exit(1); } + * cout << std::string(doc["title"]) << endl; + * } + * + * ### Threads + * + * When compiled with SIMDJSON_THREADS_ENABLED, this method will use a single thread under the + * hood to do some lookahead. + * + * ### Parser Capacity + * + * If the parser's current capacity is less than batch_size, it will allocate enough capacity + * to handle it (up to max_capacity). + * + * @param s The concatenated JSON to parse. + * @param batch_size The batch size to use. MUST be larger than the largest document. The sweet + * spot is cache-related: small enough to fit in cache, yet big enough to + * parse as many documents as possible in one tight loop. + * Defaults to 10MB, which has been a reasonable sweet spot in our tests. + * @return The stream. If there is an error, it will be returned during iteration. An empty input + * will yield 0 documents rather than an EMPTY error. Errors: + * - MEMALLOC if the parser does not have enough capacity and memory allocation fails + * - CAPACITY if the parser does not have enough capacity and batch_size > max_capacity. + * - other json errors if parsing fails + */ + inline stream parse_many(const padded_string &s, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept; + + // We do not want to allow implicit conversion from C string to std::string. + really_inline doc_result parse_many(const char *buf, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept = delete; + + /** + * The largest document this parser can automatically support. + * + * The parser may reallocate internal buffers as needed up to this amount. + * + * @return Maximum capacity, in bytes. + */ + really_inline size_t max_capacity() const noexcept; + + /** + * The largest document this parser can support without reallocating. + * + * @return Current capacity, in bytes. */ really_inline size_t capacity() const noexcept; /** * The maximum level of nested object and arrays supported by this parser. + * + * @return Maximum depth, in bytes. */ really_inline size_t max_depth() const noexcept; + /** + * Set max_capacity. This is the largest document this parser can automatically support. + * + * The parser may reallocate internal buffers as needed up to this amount. + * + * This call will not allocate or deallocate, even if capacity is currently above max_capacity. + * + * @param max_capacity The new maximum capacity, in bytes. + */ + really_inline void set_max_capacity(size_t max_capacity) noexcept; + + /** + * Set capacity. This is the largest document this parser can support without reallocating. + * + * This will allocate or deallocate as necessary. + * + * @param capacity The new capacity, in bytes. + * + * @return MEMALLOC if unsuccessful, SUCCESS otherwise. + */ + WARN_UNUSED inline error_code set_capacity(size_t capacity) noexcept; + + /** + * Set the maximum level of nested object and arrays supported by this parser. + * + * This will allocate or deallocate as necessary. + * + * @param max_depth The new maximum depth, in bytes. + * + * @return MEMALLOC if unsuccessful, SUCCESS otherwise. + */ + WARN_UNUSED inline error_code set_max_depth(size_t max_depth) noexcept; + /** * Ensure this parser has enough memory to process JSON documents up to `capacity` bytes in length * and `max_depth` depth. + * + * Equivalent to calling set_capacity() and set_max_depth(). + * + * @param capacity The new capacity. + * @param max_depth The new max_depth. Defaults to DEFAULT_MAX_DEPTH. + * @return true if successful, false if allocation failed. */ - WARN_UNUSED inline bool allocate_capacity(size_t capacity, size_t max_depth = DEFAULT_MAX_DEPTH); + WARN_UNUSED inline bool allocate_capacity(size_t capacity, size_t max_depth = DEFAULT_MAX_DEPTH) noexcept; // type aliases for backcompat using Iterator = document::iterator; - using InvalidJSON = invalid_json; + using InvalidJSON = simdjson_error; // Next location to write to in the tape uint32_t current_loc{0}; @@ -1434,13 +2127,6 @@ public: really_inline bool on_number_s64(int64_t value) noexcept; really_inline bool on_number_u64(uint64_t value) noexcept; really_inline bool on_number_double(double value) noexcept; - // - // Called before a parse is initiated. - // - // - Returns CAPACITY if the document is too large - // - Returns MEMALLOC if we needed to allocate memory and could not - // - WARN_UNUSED inline error_code init_parse(size_t len) noexcept; private: // @@ -1450,12 +2136,19 @@ private: // size_t _capacity{0}; + // + // The maximum document length this parser will automatically support. + // + // The parser will not be automatically allocated above this amount. + // + size_t _max_capacity; + // // The maximum depth (number of nested objects and arrays) supported by this parser. // // Defaults to DEFAULT_MAX_DEPTH. // - size_t _max_depth{0}; + size_t _max_depth; // all nodes are stored on the doc.tape using a 64-bit word. // @@ -1471,33 +2164,178 @@ private: // // - inline void write_tape(uint64_t val, tape_type t) noexcept; + inline void write_tape(uint64_t val, internal::tape_type t) noexcept; inline void annotate_previous_loc(uint32_t saved_loc, uint64_t val) noexcept; - // - // Set the current capacity: the largest document this parser can support without reallocating. - // - // This will allocate *or deallocate* as necessary. - // - // Returns false if allocation fails. - // - inline WARN_UNUSED bool set_capacity(size_t capacity); - - // - // Set the maximum level of nested object and arrays supported by this parser. - // - // This will allocate *or deallocate* as necessary. - // - // Returns false if allocation fails. - // - inline WARN_UNUSED bool set_max_depth(size_t max_depth); + // Ensure we have enough capacity to handle at least desired_capacity bytes, + // and auto-allocate if not. + inline error_code ensure_capacity(size_t desired_capacity) noexcept; +#if SIMDJSON_EXCEPTIONS // Used internally to get the document inline const document &get_document() const noexcept(false); +#endif // SIMDJSON_EXCEPTIONS template friend class document_iterator; + friend class document::stream; }; // class parser +/** + * Minifies a JSON element or document, printing the smallest possible valid JSON. + * + * document doc = document::parse(" [ 1 , 2 , 3 ] "_pad); + * cout << minify(doc) << endl; // prints [1,2,3] + * + */ +template +class minify { +public: + /** + * Create a new minifier. + * + * @param _value The document or element to minify. + */ + inline minify(const T &_value) noexcept : value{_value} {} + + /** + * Minify JSON to a string. + */ + inline operator std::string() const noexcept { std::stringstream s; s << *this; return s.str(); } + + /** + * Minify JSON to an output stream. + */ + inline std::ostream& print(std::ostream& out); +private: + const T &value; +}; + +/** + * Minify JSON to an output stream. + * + * @param out The output stream. + * @param formatter The minifier. + * @throw if there is an error with the underlying output stream. simdjson itself will not throw. + */ +template +inline std::ostream& operator<<(std::ostream& out, minify formatter) { return formatter.print(out); } + +/** + * Print JSON to an output stream. + * + * By default, the document will be printed minified. + * + * @param out The output stream. + * @param value The document to print. + * @throw if there is an error with the underlying output stream. simdjson itself will not throw. + */ +inline std::ostream& operator<<(std::ostream& out, const document &value) { return out << minify(value); } +/** + * Print JSON to an output stream. + * + * By default, the value will be printed minified. + * + * @param out The output stream. + * @param value The value to print. + * @throw if there is an error with the underlying output stream. simdjson itself will not throw. + */ +inline std::ostream& operator<<(std::ostream& out, const document::element &value) { return out << minify(value); }; +/** + * Print JSON to an output stream. + * + * By default, the value will be printed minified. + * + * @param out The output stream. + * @param value The value to print. + * @throw if there is an error with the underlying output stream. simdjson itself will not throw. + */ +inline std::ostream& operator<<(std::ostream& out, const document::array &value) { return out << minify(value); } +/** + * Print JSON to an output stream. + * + * By default, the value will be printed minified. + * + * @param out The output stream. + * @param value The value to print. + * @throw if there is an error with the underlying output stream. simdjson itself will not throw. + */ +inline std::ostream& operator<<(std::ostream& out, const document::object &value) { return out << minify(value); } +/** + * Print JSON to an output stream. + * + * By default, the value will be printed minified. + * + * @param out The output stream. + * @param value The value to print. + * @throw if there is an error with the underlying output stream. simdjson itself will not throw. + */ +inline std::ostream& operator<<(std::ostream& out, const document::key_value_pair &value) { return out << minify(value); } + +#if SIMDJSON_EXCEPTIONS + +/** + * Print JSON to an output stream. + * + * By default, the value will be printed minified. + * + * @param out The output stream. + * @param value The value to print. + * @throw simdjson_error if the result being printed has an error. If there is an error with the + * underlying output stream, that error will be propagated (simdjson_error will not be + * thrown). + */ +inline std::ostream& operator<<(std::ostream& out, const document::doc_move_result &value) noexcept(false) { return out << minify(value); } +/** + * Print JSON to an output stream. + * + * By default, the value will be printed minified. + * + * @param out The output stream. + * @param value The value to print. + * @throw simdjson_error if the result being printed has an error. If there is an error with the + * underlying output stream, that error will be propagated (simdjson_error will not be + * thrown). + */ +inline std::ostream& operator<<(std::ostream& out, const document::doc_result &value) noexcept(false) { return out << minify(value); } +/** + * Print JSON to an output stream. + * + * By default, the value will be printed minified. + * + * @param out The output stream. + * @param value The value to print. + * @throw simdjson_error if the result being printed has an error. If there is an error with the + * underlying output stream, that error will be propagated (simdjson_error will not be + * thrown). + */ +inline std::ostream& operator<<(std::ostream& out, const document::element_result &value) noexcept(false) { return out << minify(value); } +/** + * Print JSON to an output stream. + * + * By default, the value will be printed minified. + * + * @param out The output stream. + * @param value The value to print. + * @throw simdjson_error if the result being printed has an error. If there is an error with the + * underlying output stream, that error will be propagated (simdjson_error will not be + * thrown). + */ +inline std::ostream& operator<<(std::ostream& out, const document::array_result &value) noexcept(false) { return out << minify(value); } +/** + * Print JSON to an output stream. + * + * By default, the value will be printed minified. + * + * @param out The output stream. + * @param value The value to print. + * @throw simdjson_error if the result being printed has an error. If there is an error with the + * underlying output stream, that error will be propagated (simdjson_error will not be + * thrown). + */ +inline std::ostream& operator<<(std::ostream& out, const document::object_result &value) noexcept(false) { return out << minify(value); } + +#endif + } // namespace simdjson #endif // SIMDJSON_DOCUMENT_H @@ -1541,7 +2379,7 @@ public: virtual uint32_t required_instruction_sets() const { return _required_instruction_sets; }; /** - * Run a full document parse (init_parse, stage1 and stage2). + * Run a full document parse (ensure_capacity, stage1 and stage2). * * Overridden by each implementation. * @@ -1560,7 +2398,7 @@ public: * @param buf the json document to parse. *MUST* be allocated up to len + SIMDJSON_PADDING bytes. * @param len the length of the json document. * @param parser the parser with the buffers to use. *MUST* have allocated up to at least len capacity. - * @param streaming whether this is being called by a JsonStream parser. + * @param streaming whether this is being called by document::parser::parse_many. * @return the error code, or SUCCESS if there was no error. */ WARN_UNUSED virtual error_code stage1(const uint8_t *buf, size_t len, document::parser &parser, bool streaming) const noexcept = 0; @@ -1578,7 +2416,7 @@ public: WARN_UNUSED virtual error_code stage2(const uint8_t *buf, size_t len, document::parser &parser) const noexcept = 0; /** - * Stage 2 of the document parser for JsonStream. + * Stage 2 of the document parser for document::parser::parse_many. * * Overridden by each implementation. * @@ -1731,92 +2569,84 @@ inline internal::atomic_ptr active_implementation = &inter #endif // SIMDJSON_IMPLEMENTATION_H /* end file include/simdjson/simdjson.h */ -/* begin file include/simdjson/jsonstream.h */ -#ifndef SIMDJSON_JSONSTREAM_H -#define SIMDJSON_JSONSTREAM_H +/* begin file include/simdjson/document_stream.h */ +#ifndef SIMDJSON_DOCUMENT_STREAM_H +#define SIMDJSON_DOCUMENT_STREAM_H #include namespace simdjson { -/************************************************************************************* - * The main motivation for this piece of software is to achieve maximum speed - *and offer - * good quality of life while parsing files containing multiple JSON documents. - * - * Since we want to offer flexibility and not restrict ourselves to a specific - *file - * format, we support any file that contains any valid JSON documents separated - *by one - * or more character that is considered a whitespace by the JSON spec. - * Namely: space, nothing, linefeed, carriage return, horizontal tab. - * Anything that is not whitespace will be parsed as a JSON document and could - *lead - * to failure. - * - * To offer maximum parsing speed, our implementation processes the data inside - *the - * buffer by batches and their size is defined by the parameter "batch_size". - * By loading data in batches, we can optimize the time spent allocating data in - *the - * parser and can also open the possibility of multi-threading. - * The batch_size must be at least as large as the biggest document in the file, - *but - * not too large in order to submerge the chached memory. We found that 1MB is - * somewhat a sweet spot for now. Eventually, this batch_size could be fully - * automated and be optimal at all times. - ************************************************************************************/ +template class JsonStream; + /** -* The template parameter (string_container) must -* support the data() and size() methods, returning a pointer -* to a char* and to the number of bytes respectively. -* The simdjson parser may read up to SIMDJSON_PADDING bytes beyond the end -* of the string, so if you do not use a padded_string container, -* you have the responsability to overallocated. If you fail to -* do so, your software may crash if you cross a page boundary, -* and you should expect memory checkers to object. -* Most users should use a simdjson::padded_string. -*/ -template class JsonStream { + * A forward-only stream of documents. + * + * Produced by document::parser::parse_many. + * + */ +class document::stream { public: - /* Create a JsonStream object that can be used to parse sequentially the valid - * JSON documents found in the buffer "buf". - * - * The batch_size must be at least as large as the biggest document in the - * file, but - * not too large to submerge the cached memory. We found that 1MB is - * somewhat a sweet spot for now. - * - * The user is expected to call the following json_parse method to parse the - * next - * valid JSON document found in the buffer. This method can and is expected - * to be - * called in a loop. - * - * Various methods are offered to keep track of the status, like - * get_current_buffer_loc, - * get_n_parsed_docs, get_n_bytes_parsed, etc. - * - * */ - JsonStream(const string_container &s, size_t batch_size = 1000000); + really_inline ~stream() noexcept; - ~JsonStream(); + /** + * An iterator through a forward-only stream of documents. + */ + class iterator { + public: + /** + * Get the current document (or error). + */ + really_inline doc_result operator*() noexcept; + /** + * Advance to the next document. + */ + inline iterator& operator++() noexcept; + /** + * Check if we're at the end yet. + * @param other the end iterator to compare to. + */ + really_inline bool operator!=(const iterator &other) const noexcept; - /* Parse the next document found in the buffer previously given to JsonStream. + private: + iterator(stream& stream, bool finished) noexcept; + /** The stream parser we're iterating through. */ + stream& _stream; + /** Whether we're finished or not. */ + bool finished; + friend class stream; + }; + /** + * Start iterating the documents in the stream. + */ + really_inline iterator begin() noexcept; + /** + * The end of the stream, for iterator comparison purposes. + */ + really_inline iterator end() noexcept; + +private: + + stream &operator=(const document::stream &) = delete; // Disallow copying + + stream(document::stream &other) = delete; // Disallow copying + + really_inline stream(document::parser &parser, const uint8_t *buf, size_t len, size_t batch_size, error_code error = SUCCESS) noexcept; + + /** + * Parse the next document found in the buffer previously given to stream. + * * The content should be a valid JSON document encoded as UTF-8. If there is a * UTF-8 BOM, the caller is responsible for omitting it, UTF-8 BOM are * discouraged. * * You do NOT need to pre-allocate a parser. This function takes care of - * pre-allocating a capacity defined by the batch_size defined when creating - the - * JsonStream object. + * pre-allocating a capacity defined by the batch_size defined when creating the + * stream object. * - * The function returns simdjson::SUCCESS_AND_HAS_MORE (an integer = 1) in - case - * of success and indicates that the buffer still contains more data to be - parsed, + * The function returns simdjson::SUCCESS_AND_HAS_MORE (an integer = 1) in case + * of success and indicates that the buffer still contains more data to be parsed, * meaning this function can be called again to return the next JSON document * after this one. * @@ -1824,43 +2654,46 @@ public: * and indicates that the buffer has successfully been parsed to the end. * Every document it contained has been parsed without error. * - * The function returns an error code from simdjson/simdjson.h in case of - failure - * such as simdjson::CAPACITY, simdjson::MEMALLOC, simdjson::DEPTH_ERROR and - so forth; - * the simdjson::error_message function converts these error codes into a - * string). + * The function returns an error code from simdjson/simdjson.h in case of failure + * such as simdjson::CAPACITY, simdjson::MEMALLOC, simdjson::DEPTH_ERROR and so forth; + * the simdjson::error_message function converts these error codes into a string). * - * You can also check validity by calling parser.is_valid(). The same parser - can + * You can also check validity by calling parser.is_valid(). The same parser can * and should be reused for the other documents in the buffer. */ - int json_parse(document::parser &parser); + inline error_code json_parse() noexcept; - /* Returns the location (index) of where the next document should be in the + /** + * Returns the location (index) of where the next document should be in the * buffer. * Can be used for debugging, it tells the user the position of the end of the * last - * valid JSON document parsed*/ + * valid JSON document parsed + */ inline size_t get_current_buffer_loc() const { return current_buffer_loc; } - /* Returns the total amount of complete documents parsed by the JsonStream, - * in the current buffer, at the given time.*/ + /** + * Returns the total amount of complete documents parsed by the stream, + * in the current buffer, at the given time. + */ inline size_t get_n_parsed_docs() const { return n_parsed_docs; } - /* Returns the total amount of data (in bytes) parsed by the JsonStream, - * in the current buffer, at the given time.*/ + /** + * Returns the total amount of data (in bytes) parsed by the stream, + * in the current buffer, at the given time. + */ inline size_t get_n_bytes_parsed() const { return n_bytes_parsed; } -private: - inline const uint8_t *buf() const { return reinterpret_cast(str.data()) + str_start; } + inline const uint8_t *buf() const { return _buf + buf_start; } - inline void advance(size_t offset) { str_start += offset; } + inline void advance(size_t offset) { buf_start += offset; } - inline size_t remaining() const { return str.size() - str_start; } + inline size_t remaining() const { return _len - buf_start; } - const string_container &str; + document::parser &parser; + const uint8_t *_buf; + const size_t _len; size_t _batch_size; // this is actually variable! - size_t str_start{0}; + size_t buf_start{0}; size_t next_json{0}; bool load_next_batch{true}; size_t current_buffer_loc{0}; @@ -1869,17 +2702,19 @@ private: #endif size_t n_parsed_docs{0}; size_t n_bytes_parsed{0}; - simdjson::implementation *stage_parser; + error_code error{SUCCESS_AND_HAS_MORE}; #ifdef SIMDJSON_THREADS_ENABLED error_code stage1_is_ok_thread{SUCCESS}; std::thread stage_1_thread; document::parser parser_thread; #endif -}; // end of class JsonStream + template friend class JsonStream; + friend class document::parser; +}; // class document::stream } // end of namespace simdjson -#endif // SIMDJSON_JSONSTREAM_H -/* end file include/simdjson/jsonstream.h */ +#endif // SIMDJSON_DOCUMENT_STREAM_H +/* end file include/simdjson/document_stream.h */ /* begin file include/simdjson/jsonminifier.h */ #ifndef SIMDJSON_JSONMINIFIER_H #define SIMDJSON_JSONMINIFIER_H @@ -1908,8 +2743,10 @@ static inline size_t json_minify(const std::string_view &p, char *out) { static inline size_t json_minify(const padded_string &p, char *out) { return json_minify(p.data(), p.size(), out); } + } // namespace simdjson -#endif + +#endif // SIMDJSON_JSONMINIFIER_H /* end file include/simdjson/jsonminifier.h */ // Deprecated API @@ -1936,110 +2773,59 @@ static inline size_t json_minify(const padded_string &p, char *out) { #include #include +#include namespace simdjson::internal { -// ends with zero char -static inline void print_with_escapes(const unsigned char *src, std::ostream &os) { - while (*src) { - switch (*src) { +class escape_json_string; + +inline std::ostream& operator<<(std::ostream& out, const escape_json_string &str); + +class escape_json_string { +public: + escape_json_string(std::string_view _str) noexcept : str{_str} {} + operator std::string() const noexcept { std::stringstream s; s << *this; return s.str(); } +private: + std::string_view str; + friend std::ostream& operator<<(std::ostream& out, const escape_json_string &unescaped); +}; + +inline std::ostream& operator<<(std::ostream& out, const escape_json_string &unescaped) { + for (size_t i=0; i(*src); - os.flags(f); + if ((unsigned char)unescaped.str[i] <= 0x1F) { + // TODO can this be done once at the beginning, or will it mess up << char? + std::ios::fmtflags f(out.flags()); + out << "\\u" << std::hex << std::setw(4) << std::setfill('0') << static_cast(unescaped.str[i]); + out.flags(f); } else { - os << *src; + out << unescaped.str[i]; } } - src++; } -} - - -// print len chars -static inline void print_with_escapes(const unsigned char *src, - std::ostream &os, size_t len) { - const unsigned char *finalsrc = src + len; - while (src < finalsrc) { - switch (*src) { - case '\b': - os << '\\'; - os << 'b'; - break; - case '\f': - os << '\\'; - os << 'f'; - break; - case '\n': - os << '\\'; - os << 'n'; - break; - case '\r': - os << '\\'; - os << 'r'; - break; - case '\"': - os << '\\'; - os << '"'; - break; - case '\t': - os << '\\'; - os << 't'; - break; - case '\\': - os << '\\'; - os << '\\'; - break; - default: - if (*src <= 0x1F) { - std::ios::fmtflags f(os.flags()); - os << std::hex << std::setw(4) << std::setfill('0') - << static_cast(*src); - os.flags(f); - } else { - os << *src; - } - } - src++; - } -} - -static inline void print_with_escapes(const char *src, std::ostream &os) { - print_with_escapes(reinterpret_cast(src), os); -} - -static inline void print_with_escapes(const char *src, std::ostream &os, size_t len) { - print_with_escapes(reinterpret_cast(src), os, len); + return out; } } // namespace simdjson::internal @@ -2051,7 +2837,9 @@ namespace simdjson { template class document_iterator { public: - document_iterator(const document::parser &parser); +#if SIMDJSON_EXCEPTIONS + document_iterator(const document::parser &parser) noexcept(false); +#endif document_iterator(const document &doc) noexcept; document_iterator(const document_iterator &o) noexcept; document_iterator &operator=(const document_iterator &o) noexcept; @@ -2103,7 +2891,7 @@ public: // within the string: get_string_length determines the true string length. inline const char *get_string() const { return reinterpret_cast( - doc.string_buf.get() + (current_val & JSON_VALUE_MASK) + sizeof(uint32_t)); + doc.string_buf.get() + (current_val & internal::JSON_VALUE_MASK) + sizeof(uint32_t)); } // return the length of the string in bytes @@ -2111,7 +2899,7 @@ public: uint32_t answer; memcpy(&answer, reinterpret_cast(doc.string_buf.get() + - (current_val & JSON_VALUE_MASK)), + (current_val & internal::JSON_VALUE_MASK)), sizeof(uint32_t)); return answer; } @@ -2336,24 +3124,17 @@ using ParsedJson = document::parser; namespace simdjson { -// load a file in memory... -// get a corpus; pad out to cache line so we can always use SIMD -// throws exceptions in case of failure -// first element of the pair is a string (null terminated) -// whereas the second element is the length. -// caller is responsible to free (aligned_free((void*)result.data()))) -// -// throws an exception if the file cannot be opened, use try/catch -// try { -// p = get_corpus(filename); -// } catch (const std::exception& e) { -// aligned_free((void*)p.data()); -// std::cout << "Could not load the file " << filename << std::endl; -// } -padded_string get_corpus(const std::string &filename); +#if SIMDJSON_EXCEPTIONS + +inline padded_string get_corpus(const std::string &filename) { + return padded_string::load(filename); +} + +#endif // SIMDJSON_EXCEPTIONS + } // namespace simdjson -#endif +#endif // SIMDJSON_JSONIOUTIL_H /* end file include/simdjson/jsonioutil.h */ namespace simdjson { @@ -2363,7 +3144,7 @@ namespace simdjson { // inline int json_parse(const uint8_t *buf, size_t len, document::parser &parser, bool realloc_if_needed = true) noexcept { - error_code code = parser.parse(buf, len, realloc_if_needed).error; + error_code code = parser.parse(buf, len, realloc_if_needed).error(); // The deprecated json_parse API is a signal that the user plans to *use* the error code / valid // bits in the parser instead of heeding the result code. The normal parser unsets those in // anticipation of making the error code ephemeral. @@ -2384,11 +3165,6 @@ inline int json_parse(const padded_string &s, document::parser &parser) noexcept WARN_UNUSED static document::parser build_parsed_json(const uint8_t *buf, size_t len, bool realloc_if_needed = true) noexcept { document::parser parser; - if (!parser.allocate_capacity(len)) { - parser.valid = false; - parser.error = MEMALLOC; - return parser; - } json_parse(buf, len, parser, realloc_if_needed); return parser; } @@ -2410,6 +3186,131 @@ document::parser build_parsed_json(const char *buf) noexcept = delete; #endif /* end file include/simdjson/jsonioutil.h */ +/* begin file include/simdjson/jsonstream.h */ +// TODO Remove this -- deprecated API and files + +#ifndef SIMDJSON_JSONSTREAM_H +#define SIMDJSON_JSONSTREAM_H + + +namespace simdjson { + +/** + * @deprecated use document::stream instead. + * + * The main motivation for this piece of software is to achieve maximum speed and offer + * good quality of life while parsing files containing multiple JSON documents. + * + * Since we want to offer flexibility and not restrict ourselves to a specific file + * format, we support any file that contains any valid JSON documents separated by one + * or more character that is considered a whitespace by the JSON spec. + * Namely: space, nothing, linefeed, carriage return, horizontal tab. + * Anything that is not whitespace will be parsed as a JSON document and could lead + * to failure. + * + * To offer maximum parsing speed, our implementation processes the data inside the + * buffer by batches and their size is defined by the parameter "batch_size". + * By loading data in batches, we can optimize the time spent allocating data in the + * parser and can also open the possibility of multi-threading. + * The batch_size must be at least as large as the biggest document in the file, but + * not too large in order to submerge the chached memory. We found that 1MB is + * somewhat a sweet spot for now. Eventually, this batch_size could be fully + * automated and be optimal at all times. + * + * The template parameter (string_container) must + * support the data() and size() methods, returning a pointer + * to a char* and to the number of bytes respectively. + * The simdjson parser may read up to SIMDJSON_PADDING bytes beyond the end + * of the string, so if you do not use a padded_string container, + * you have the responsability to overallocated. If you fail to + * do so, your software may crash if you cross a page boundary, + * and you should expect memory checkers to object. + * Most users should use a simdjson::padded_string. + */ +template class JsonStream { +public: + /* Create a JsonStream object that can be used to parse sequentially the valid + * JSON documents found in the buffer "buf". + * + * The batch_size must be at least as large as the biggest document in the + * file, but + * not too large to submerge the cached memory. We found that 1MB is + * somewhat a sweet spot for now. + * + * The user is expected to call the following json_parse method to parse the + * next + * valid JSON document found in the buffer. This method can and is expected + * to be + * called in a loop. + * + * Various methods are offered to keep track of the status, like + * get_current_buffer_loc, + * get_n_parsed_docs, get_n_bytes_parsed, etc. + * + * */ + JsonStream(const string_container &s, size_t _batch_size = 1000000) noexcept; + + ~JsonStream() noexcept; + + /* Parse the next document found in the buffer previously given to JsonStream. + + * The content should be a valid JSON document encoded as UTF-8. If there is a + * UTF-8 BOM, the caller is responsible for omitting it, UTF-8 BOM are + * discouraged. + * + * You do NOT need to pre-allocate a parser. This function takes care of + * pre-allocating a capacity defined by the batch_size defined when creating + the + * JsonStream object. + * + * The function returns simdjson::SUCCESS_AND_HAS_MORE (an integer = 1) in + case + * of success and indicates that the buffer still contains more data to be + parsed, + * meaning this function can be called again to return the next JSON document + * after this one. + * + * The function returns simdjson::SUCCESS (as integer = 0) in case of success + * and indicates that the buffer has successfully been parsed to the end. + * Every document it contained has been parsed without error. + * + * The function returns an error code from simdjson/simdjson.h in case of + failure + * such as simdjson::CAPACITY, simdjson::MEMALLOC, simdjson::DEPTH_ERROR and + so forth; + * the simdjson::error_message function converts these error codes into a + * string). + * + * You can also check validity by calling parser.is_valid(). The same parser + can + * and should be reused for the other documents in the buffer. */ + int json_parse(document::parser &parser) noexcept; + + /* Returns the location (index) of where the next document should be in the + * buffer. + * Can be used for debugging, it tells the user the position of the end of the + * last + * valid JSON document parsed*/ + inline size_t get_current_buffer_loc() const noexcept { return stream ? stream->current_buffer_loc : 0; } + + /* Returns the total amount of complete documents parsed by the JsonStream, + * in the current buffer, at the given time.*/ + inline size_t get_n_parsed_docs() const noexcept { return stream ? stream->n_parsed_docs : 0; } + + /* Returns the total amount of data (in bytes) parsed by the JsonStream, + * in the current buffer, at the given time.*/ + inline size_t get_n_bytes_parsed() const noexcept { return stream ? stream->n_bytes_parsed : 0; } + +private: + const string_container &str; + const size_t batch_size; + document::stream *stream{nullptr}; +}; // end of class JsonStream + +} // end of namespace simdjson + +#endif // SIMDJSON_JSONSTREAM_H +/* end file include/simdjson/jsonstream.h */ // Inline functions /* begin file include/simdjson/inline/document.h */ @@ -2423,129 +3324,58 @@ document::parser build_parsed_json(const char *buf) noexcept = delete; namespace simdjson { // -// document::element_result inline implementation +// element_result inline implementation // -template -inline document::element_result::element_result(T _value) noexcept : value(_value), error{SUCCESS} {} -template -inline document::element_result::element_result(error_code _error) noexcept : value(), error{_error} {} -template<> -inline document::simdjson_result::operator std::string_view() const noexcept(false) { - if (error) { throw invalid_json(error); } - return value; +really_inline document::element_result::element_result(element value) noexcept : simdjson_result(value) {} +really_inline document::element_result::element_result(error_code error) noexcept : simdjson_result(error) {} +inline simdjson_result document::element_result::is_null() const noexcept { + if (error()) { return error(); } + return first.is_null(); } -template<> -inline document::simdjson_result::operator const char *() const noexcept(false) { - if (error) { throw invalid_json(error); } - return value; +inline simdjson_result document::element_result::as_bool() const noexcept { + if (error()) { return error(); } + return first.as_bool(); } -template<> -inline document::simdjson_result::operator bool() const noexcept(false) { - if (error) { throw invalid_json(error); } - return value; +inline simdjson_result document::element_result::as_c_str() const noexcept { + if (error()) { return error(); } + return first.as_c_str(); } -template<> -inline document::simdjson_result::operator uint64_t() const noexcept(false) { - if (error) { throw invalid_json(error); } - return value; +inline simdjson_result document::element_result::as_string() const noexcept { + if (error()) { return error(); } + return first.as_string(); } -template<> -inline document::simdjson_result::operator int64_t() const noexcept(false) { - if (error) { throw invalid_json(error); } - return value; +inline simdjson_result document::element_result::as_uint64_t() const noexcept { + if (error()) { return error(); } + return first.as_uint64_t(); } -template<> -inline document::simdjson_result::operator double() const noexcept(false) { - if (error) { throw invalid_json(error); } - return value; +inline simdjson_result document::element_result::as_int64_t() const noexcept { + if (error()) { return error(); } + return first.as_int64_t(); } - -// -// document::array_result inline implementation -// -inline document::array_result::element_result(document::array _value) noexcept : value(_value), error{SUCCESS} {} -inline document::array_result::element_result(error_code _error) noexcept : value(), error{_error} {} -inline document::array_result::operator document::array() const noexcept(false) { - if (error) { throw invalid_json(error); } - return value; -} -inline document::array::iterator document::array_result::begin() const noexcept(false) { - if (error) { throw invalid_json(error); } - return value.begin(); -} -inline document::array::iterator document::array_result::end() const noexcept(false) { - if (error) { throw invalid_json(error); } - return value.end(); -} - -// -// document::object_result inline implementation -// -inline document::object_result::element_result(document::object _value) noexcept : value(_value), error{SUCCESS} {} -inline document::object_result::element_result(error_code _error) noexcept : value(), error{_error} {} -inline document::object_result::operator document::object() const noexcept(false) { - if (error) { throw invalid_json(error); } - return value; -} -inline document::element_result document::object_result::operator[](const std::string_view &key) const noexcept { - if (error) { return error; } - return value[key]; -} -inline document::element_result document::object_result::operator[](const char *key) const noexcept { - if (error) { return error; } - return value[key]; -} -inline document::object::iterator document::object_result::begin() const noexcept(false) { - if (error) { throw invalid_json(error); } - return value.begin(); -} -inline document::object::iterator document::object_result::end() const noexcept(false) { - if (error) { throw invalid_json(error); } - return value.end(); -} - -// -// document::element_result inline implementation -// -inline document::element_result::element_result(document::element _value) noexcept : value(_value), error{SUCCESS} {} -inline document::element_result::element_result(error_code _error) noexcept : value(), error{_error} {} -inline document::simdjson_result document::element_result::is_null() const noexcept { - if (error) { return error; } - return value.is_null(); -} -inline document::simdjson_result document::element_result::as_bool() const noexcept { - if (error) { return error; } - return value.as_bool(); -} -inline document::element_result document::element_result::as_c_str() const noexcept { - if (error) { return error; } - return value.as_c_str(); -} -inline document::simdjson_result document::element_result::as_string() const noexcept { - if (error) { return error; } - return value.as_string(); -} -inline document::simdjson_result document::element_result::as_uint64_t() const noexcept { - if (error) { return error; } - return value.as_uint64_t(); -} -inline document::simdjson_result document::element_result::as_int64_t() const noexcept { - if (error) { return error; } - return value.as_int64_t(); -} -inline document::simdjson_result document::element_result::as_double() const noexcept { - if (error) { return error; } - return value.as_double(); +inline simdjson_result document::element_result::as_double() const noexcept { + if (error()) { return error(); } + return first.as_double(); } inline document::array_result document::element_result::as_array() const noexcept { - if (error) { return error; } - return value.as_array(); + if (error()) { return error(); } + return first.as_array(); } inline document::object_result document::element_result::as_object() const noexcept { - if (error) { return error; } - return value.as_object(); + if (error()) { return error(); } + return first.as_object(); } +inline document::element_result document::element_result::operator[](const std::string_view &key) const noexcept { + if (error()) { return *this; } + return first[key]; +} +inline document::element_result document::element_result::operator[](const char *key) const noexcept { + if (error()) { return *this; } + return first[key]; +} + +#if SIMDJSON_EXCEPTIONS + inline document::element_result::operator bool() const noexcept(false) { return as_bool(); } @@ -2570,20 +3400,61 @@ inline document::element_result::operator document::array() const noexcept(false inline document::element_result::operator document::object() const noexcept(false) { return as_object(); } -inline document::element_result document::element_result::operator[](const std::string_view &key) const noexcept { - if (error) { return *this; } - return value[key]; + +#endif + +// +// array_result inline implementation +// +really_inline document::array_result::array_result(array value) noexcept : simdjson_result(value) {} +really_inline document::array_result::array_result(error_code error) noexcept : simdjson_result(error) {} + +#if SIMDJSON_EXCEPTIONS + +inline document::array::iterator document::array_result::begin() const noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first.begin(); } -inline document::element_result document::element_result::operator[](const char *key) const noexcept { - if (error) { return *this; } - return value[key]; +inline document::array::iterator document::array_result::end() const noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first.end(); } +#endif // SIMDJSON_EXCEPTIONS + +// +// object_result inline implementation +// +really_inline document::object_result::object_result(object value) noexcept : simdjson_result(value) {} +really_inline document::object_result::object_result(error_code error) noexcept : simdjson_result(error) {} + +inline document::element_result document::object_result::operator[](const std::string_view &key) const noexcept { + if (error()) { return error(); } + return first[key]; +} +inline document::element_result document::object_result::operator[](const char *key) const noexcept { + if (error()) { return error(); } + return first[key]; +} + +#if SIMDJSON_EXCEPTIONS + +inline document::object::iterator document::object_result::begin() const noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first.begin(); +} +inline document::object::iterator document::object_result::end() const noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first.end(); +} + +#endif // SIMDJSON_EXCEPTIONS + // // document inline implementation // inline document::element document::root() const noexcept { - return document::element(this, 1); + return element(this, 1); } inline document::array_result document::as_array() const noexcept { return root().as_array(); @@ -2591,15 +3462,21 @@ inline document::array_result document::as_array() const noexcept { inline document::object_result document::as_object() const noexcept { return root().as_object(); } -inline document::operator document::element() const noexcept { +inline document::operator element() const noexcept { return root(); } + +#if SIMDJSON_EXCEPTIONS + inline document::operator document::array() const noexcept(false) { return root(); } inline document::operator document::object() const noexcept(false) { return root(); } + +#endif + inline document::element_result document::operator[](const std::string_view &key) const noexcept { return root()[key]; } @@ -2607,30 +3484,33 @@ inline document::element_result document::operator[](const char *key) const noex return root()[key]; } -inline document::doc_result document::parse(const uint8_t *buf, size_t len, bool realloc_if_needed) noexcept { +inline document::doc_move_result document::load(const std::string &path) noexcept { + document::parser parser; + auto [doc, error] = parser.load(path); + return doc_move_result((document &&)doc, error); +} + +inline document::doc_move_result document::parse(const uint8_t *buf, size_t len, bool realloc_if_needed) noexcept { document::parser parser; - if (!parser.allocate_capacity(len)) { - return MEMALLOC; - } auto [doc, error] = parser.parse(buf, len, realloc_if_needed); - return document::doc_result((document &&)doc, error); + return doc_move_result((document &&)doc, error); } -really_inline document::doc_result document::parse(const char *buf, size_t len, bool realloc_if_needed) noexcept { - return parse((const uint8_t *)buf, len, realloc_if_needed); +really_inline document::doc_move_result document::parse(const char *buf, size_t len, bool realloc_if_needed) noexcept { + return parse((const uint8_t *)buf, len, realloc_if_needed); } -really_inline document::doc_result document::parse(const std::string &s) noexcept { - return parse(s.data(), s.length(), s.capacity() - s.length() < SIMDJSON_PADDING); +really_inline document::doc_move_result document::parse(const std::string &s) noexcept { + return parse(s.data(), s.length(), s.capacity() - s.length() < SIMDJSON_PADDING); } -really_inline document::doc_result document::parse(const padded_string &s) noexcept { - return parse(s.data(), s.length(), false); +really_inline document::doc_move_result document::parse(const padded_string &s) noexcept { + return parse(s.data(), s.length(), false); } WARN_UNUSED -inline bool document::set_capacity(size_t capacity) { +inline error_code document::set_capacity(size_t capacity) noexcept { if (capacity == 0) { string_buf.reset(); tape.reset(); - return true; + return SUCCESS; } // a pathological input like "[[[[..." would generate len tape elements, so @@ -2644,113 +3524,7 @@ inline bool document::set_capacity(size_t capacity) { size_t string_capacity = ROUNDUP_N(5 * capacity / 3 + 32, 64); string_buf.reset( new (std::nothrow) uint8_t[string_capacity]); tape.reset(new (std::nothrow) uint64_t[tape_capacity]); - return string_buf && tape; -} - -inline bool document::print_json(std::ostream &os, size_t max_depth) const noexcept { - uint32_t string_length; - size_t tape_idx = 0; - uint64_t tape_val = tape[tape_idx]; - uint8_t type = (tape_val >> 56); - size_t how_many = 0; - if (type == 'r') { - how_many = tape_val & JSON_VALUE_MASK; - } else { - // Error: no starting root node? - return false; - } - tape_idx++; - std::unique_ptr in_object(new bool[max_depth]); - std::unique_ptr in_object_idx(new size_t[max_depth]); - int depth = 1; // only root at level 0 - in_object_idx[depth] = 0; - in_object[depth] = false; - for (; tape_idx < how_many; tape_idx++) { - tape_val = tape[tape_idx]; - uint64_t payload = tape_val & JSON_VALUE_MASK; - type = (tape_val >> 56); - if (!in_object[depth]) { - if ((in_object_idx[depth] > 0) && (type != ']')) { - os << ","; - } - in_object_idx[depth]++; - } else { // if (in_object) { - if ((in_object_idx[depth] > 0) && ((in_object_idx[depth] & 1) == 0) && - (type != '}')) { - os << ","; - } - if (((in_object_idx[depth] & 1) == 1)) { - os << ":"; - } - in_object_idx[depth]++; - } - switch (type) { - case '"': // we have a string - os << '"'; - memcpy(&string_length, string_buf.get() + payload, sizeof(uint32_t)); - internal::print_with_escapes( - (const unsigned char *)(string_buf.get() + payload + sizeof(uint32_t)), - os, string_length); - os << '"'; - break; - case 'l': // we have a long int - if (tape_idx + 1 >= how_many) { - return false; - } - os << static_cast(tape[++tape_idx]); - break; - case 'u': - if (tape_idx + 1 >= how_many) { - return false; - } - os << tape[++tape_idx]; - break; - case 'd': // we have a double - if (tape_idx + 1 >= how_many) { - return false; - } - double answer; - memcpy(&answer, &tape[++tape_idx], sizeof(answer)); - os << answer; - break; - case 'n': // we have a null - os << "null"; - break; - case 't': // we have a true - os << "true"; - break; - case 'f': // we have a false - os << "false"; - break; - case '{': // we have an object - os << '{'; - depth++; - in_object[depth] = true; - in_object_idx[depth] = 0; - break; - case '}': // we end an object - depth--; - os << '}'; - break; - case '[': // we start an array - os << '['; - depth++; - in_object[depth] = false; - in_object_idx[depth] = 0; - break; - case ']': // we end an array - depth--; - os << ']'; - break; - case 'r': // we start and end with the root node - // should we be hitting the root node? - return false; - default: - // bug? - return false; - } - } - return true; + return string_buf && tape ? SUCCESS : MEMALLOC; } inline bool document::dump_raw_tape(std::ostream &os) const noexcept { @@ -2762,7 +3536,7 @@ inline bool document::dump_raw_tape(std::ostream &os) const noexcept { tape_idx++; size_t how_many = 0; if (type == 'r') { - how_many = tape_val & JSON_VALUE_MASK; + how_many = tape_val & internal::JSON_VALUE_MASK; } else { // Error: no starting root node? return false; @@ -2772,16 +3546,16 @@ inline bool document::dump_raw_tape(std::ostream &os) const noexcept { for (; tape_idx < how_many; tape_idx++) { os << tape_idx << " : "; tape_val = tape[tape_idx]; - payload = tape_val & JSON_VALUE_MASK; + payload = tape_val & internal::JSON_VALUE_MASK; type = (tape_val >> 56); switch (type) { case '"': // we have a string os << "string \""; memcpy(&string_length, string_buf.get() + payload, sizeof(uint32_t)); - internal::print_with_escapes( - (const unsigned char *)(string_buf.get() + payload + sizeof(uint32_t)), - os, - string_length); + os << internal::escape_json_string(std::string_view( + (const char *)(string_buf.get() + payload + sizeof(uint32_t)), + string_length + )); os << '"'; os << '\n'; break; @@ -2839,7 +3613,7 @@ inline bool document::dump_raw_tape(std::ostream &os) const noexcept { } } tape_val = tape[tape_idx]; - payload = tape_val & JSON_VALUE_MASK; + payload = tape_val & internal::JSON_VALUE_MASK; type = (tape_val >> 56); os << tape_idx << " : " << type << "\t// pointing to " << payload << " (start root)\n"; @@ -2847,100 +3621,153 @@ inline bool document::dump_raw_tape(std::ostream &os) const noexcept { } // -// document::doc_ref_result inline implementation +// doc_result inline implementation // -inline document::doc_ref_result::doc_ref_result(document &_doc, error_code _error) noexcept : doc(_doc), error(_error) { } -inline document::doc_ref_result::operator document&() noexcept(false) { - if (error) { - throw invalid_json(error); - } - return doc; +inline document::doc_result::doc_result(document &doc, error_code error) noexcept : simdjson_result(doc, error) { } + +inline document::array_result document::doc_result::as_array() const noexcept { + if (error()) { return error(); } + return first.root().as_array(); } -inline const std::string &document::doc_ref_result::get_error_message() const noexcept { - return error_message(error); +inline document::object_result document::doc_result::as_object() const noexcept { + if (error()) { return error(); } + return first.root().as_object(); +} + +inline document::element_result document::doc_result::operator[](const std::string_view &key) const noexcept { + if (error()) { return error(); } + return first[key]; +} +inline document::element_result document::doc_result::operator[](const char *key) const noexcept { + if (error()) { return error(); } + return first[key]; } // -// document::doc_result inline implementation +// doc_move_result inline implementation // -inline document::doc_result::doc_result(document &&_doc, error_code _error) noexcept : doc(std::move(_doc)), error(_error) { } -inline document::doc_result::doc_result(document &&_doc) noexcept : doc(std::move(_doc)), error(SUCCESS) { } -inline document::doc_result::doc_result(error_code _error) noexcept : doc(), error(_error) { } -inline document::doc_result::operator document() noexcept(false) { - if (error) { - throw invalid_json(error); - } - return std::move(doc); +inline document::doc_move_result::doc_move_result(document &&doc, error_code error) noexcept : simdjson_move_result(std::move(doc), error) { } +inline document::doc_move_result::doc_move_result(document &&doc) noexcept : simdjson_move_result(std::move(doc)) { } +inline document::doc_move_result::doc_move_result(error_code error) noexcept : simdjson_move_result(error) { } + +inline document::array_result document::doc_move_result::as_array() const noexcept { + if (error()) { return error(); } + return first.root().as_array(); } -inline const std::string &document::doc_result::get_error_message() const noexcept { - return error_message(error); +inline document::object_result document::doc_move_result::as_object() const noexcept { + if (error()) { return error(); } + return first.root().as_object(); +} + +inline document::element_result document::doc_move_result::operator[](const std::string_view &key) const noexcept { + if (error()) { return error(); } + return first[key]; +} +inline document::element_result document::doc_move_result::operator[](const char *key) const noexcept { + if (error()) { return error(); } + return first[key]; } // // document::parser inline implementation // +really_inline document::parser::parser(size_t max_capacity, size_t max_depth) noexcept + : _max_capacity{max_capacity}, _max_depth{max_depth} { + +} inline bool document::parser::is_valid() const noexcept { return valid; } inline int document::parser::get_error_code() const noexcept { return error; } -inline std::string document::parser::get_error_message() const noexcept { return error_message(error); } +inline std::string document::parser::get_error_message() const noexcept { return error_message(int(error)); } inline bool document::parser::print_json(std::ostream &os) const noexcept { - return is_valid() ? doc.print_json(os) : false; + if (!is_valid()) { return false; } + os << minify(doc); + return true; } inline bool document::parser::dump_raw_tape(std::ostream &os) const noexcept { return is_valid() ? doc.dump_raw_tape(os) : false; } + +#if SIMDJSON_EXCEPTIONS + inline const document &document::parser::get_document() const noexcept(false) { if (!is_valid()) { - throw invalid_json(error); + throw simdjson_error(error); } return doc; } -inline document::doc_ref_result document::parser::parse(const uint8_t *buf, size_t len, bool realloc_if_needed) noexcept { - error_code code = init_parse(len); - if (code) { return document::doc_ref_result(doc, code); } +#endif // SIMDJSON_EXCEPTIONS + +inline document::doc_result document::parser::load(const std::string &path) noexcept { + auto [json, _error] = padded_string::load(path); + if (_error) { return doc_result(doc, _error); } + return parse(json); +} + +inline document::stream document::parser::load_many(const std::string &path, size_t batch_size) noexcept { + auto [json, _error] = padded_string::load(path); + return stream(*this, reinterpret_cast(json.data()), json.length(), batch_size, _error); +} + +inline document::doc_result document::parser::parse(const uint8_t *buf, size_t len, bool realloc_if_needed) noexcept { + error_code code = ensure_capacity(len); + if (code) { return doc_result(doc, code); } if (realloc_if_needed) { const uint8_t *tmp_buf = buf; - buf = (uint8_t *)allocate_padded_buffer(len); + buf = (uint8_t *)internal::allocate_padded_buffer(len); if (buf == nullptr) - return document::doc_ref_result(doc, MEMALLOC); + return doc_result(doc, MEMALLOC); memcpy((void *)buf, tmp_buf, len); } code = simdjson::active_implementation->parse(buf, len, *this); - // We're indicating validity via the doc_ref_result, so set the parse state back to invalid + // We're indicating validity via the doc_result, so set the parse state back to invalid valid = false; error = UNINITIALIZED; if (realloc_if_needed) { aligned_free((void *)buf); // must free before we exit } - return document::doc_ref_result(doc, code); + return doc_result(doc, code); } -really_inline document::doc_ref_result document::parser::parse(const char *buf, size_t len, bool realloc_if_needed) noexcept { +really_inline document::doc_result document::parser::parse(const char *buf, size_t len, bool realloc_if_needed) noexcept { return parse((const uint8_t *)buf, len, realloc_if_needed); } -really_inline document::doc_ref_result document::parser::parse(const std::string &s) noexcept { +really_inline document::doc_result document::parser::parse(const std::string &s) noexcept { return parse(s.data(), s.length(), s.capacity() - s.length() < SIMDJSON_PADDING); } -really_inline document::doc_ref_result document::parser::parse(const padded_string &s) noexcept { +really_inline document::doc_result document::parser::parse(const padded_string &s) noexcept { return parse(s.data(), s.length(), false); } +inline document::stream document::parser::parse_many(const uint8_t *buf, size_t len, size_t batch_size) noexcept { + return stream(*this, buf, len, batch_size); +} +inline document::stream document::parser::parse_many(const char *buf, size_t len, size_t batch_size) noexcept { + return parse_many((const uint8_t *)buf, len, batch_size); +} +inline document::stream document::parser::parse_many(const std::string &s, size_t batch_size) noexcept { + return parse_many(s.data(), s.length(), batch_size); +} +inline document::stream document::parser::parse_many(const padded_string &s, size_t batch_size) noexcept { + return parse_many(s.data(), s.length(), batch_size); +} + really_inline size_t document::parser::capacity() const noexcept { return _capacity; } +really_inline size_t document::parser::max_capacity() const noexcept { + return _max_capacity; +} really_inline size_t document::parser::max_depth() const noexcept { return _max_depth; } -WARN_UNUSED inline bool document::parser::allocate_capacity(size_t capacity, size_t max_depth) { - return set_capacity(capacity) && set_max_depth(max_depth); -} WARN_UNUSED -inline bool document::parser::set_capacity(size_t capacity) { +inline error_code document::parser::set_capacity(size_t capacity) noexcept { if (_capacity == capacity) { - return true; + return SUCCESS; } // Set capacity to 0 until we finish, in case there's an error @@ -2949,16 +3776,15 @@ inline bool document::parser::set_capacity(size_t capacity) { // // Reallocate the document // - if (!doc.set_capacity(capacity)) { - return false; - } + error_code err = doc.set_capacity(capacity); + if (err) { return err; } // // Don't allocate 0 bytes, just return. // if (capacity == 0) { structural_indexes.reset(); - return true; + return SUCCESS; } // @@ -2967,20 +3793,26 @@ inline bool document::parser::set_capacity(size_t capacity) { uint32_t max_structures = ROUNDUP_N(capacity, 64) + 2 + 7; structural_indexes.reset( new (std::nothrow) uint32_t[max_structures]); // TODO realloc if (!structural_indexes) { - return false; + return MEMALLOC; } _capacity = capacity; - return true; + return SUCCESS; } -WARN_UNUSED inline bool document::parser::set_max_depth(size_t max_depth) { +really_inline void document::parser::set_max_capacity(size_t max_capacity) noexcept { + _max_capacity = max_capacity; +} + +WARN_UNUSED inline error_code document::parser::set_max_depth(size_t max_depth) noexcept { + if (max_depth == _max_depth && ret_address) { return SUCCESS; } + _max_depth = 0; if (max_depth == 0) { ret_address.reset(); containing_scope_offset.reset(); - return true; + return SUCCESS; } // @@ -2995,34 +3827,48 @@ WARN_UNUSED inline bool document::parser::set_max_depth(size_t max_depth) { if (!ret_address || !containing_scope_offset) { // Could not allocate memory - return false; + return MEMALLOC; } _max_depth = max_depth; - return true; + return SUCCESS; } -WARN_UNUSED -inline error_code document::parser::init_parse(size_t len) noexcept { - if (len > capacity()) { - return error = CAPACITY; +WARN_UNUSED inline bool document::parser::allocate_capacity(size_t capacity, size_t max_depth) noexcept { + return !set_capacity(capacity) && !set_max_depth(max_depth); +} + +inline error_code document::parser::ensure_capacity(size_t desired_capacity) noexcept { + // If we don't have enough capacity, (try to) automatically bump it. + if (unlikely(desired_capacity > capacity())) { + if (desired_capacity > max_capacity()) { + return error = CAPACITY; + } + + error = set_capacity(desired_capacity); + if (error) { return error; } } + + // Allocate depth-based buffers if they aren't already. + error = set_max_depth(max_depth()); + if (error) { return error; } + // If the last doc was taken, we need to allocate a new one if (!doc.tape) { - if (!doc.set_capacity(len)) { - return error = MEMALLOC; - } + error = doc.set_capacity(desired_capacity); + if (error) { return error; } } + return SUCCESS; } // -// document::tape_ref inline implementation +// tape_ref inline implementation // -really_inline document::tape_ref::tape_ref() noexcept : doc{nullptr}, json_index{0} {} -really_inline document::tape_ref::tape_ref(const document *_doc, size_t _json_index) noexcept : doc{_doc}, json_index{_json_index} {} +really_inline internal::tape_ref::tape_ref() noexcept : doc{nullptr}, json_index{0} {} +really_inline internal::tape_ref::tape_ref(const document *_doc, size_t _json_index) noexcept : doc{_doc}, json_index{_json_index} {} -inline size_t document::tape_ref::after_element() const noexcept { +inline size_t internal::tape_ref::after_element() const noexcept { switch (type()) { case tape_type::START_ARRAY: case tape_type::START_OBJECT: @@ -3035,23 +3881,32 @@ inline size_t document::tape_ref::after_element() const noexcept { return json_index + 1; } } -really_inline document::tape_type document::tape_ref::type() const noexcept { +really_inline internal::tape_type internal::tape_ref::type() const noexcept { return static_cast(doc->tape[json_index] >> 56); } -really_inline uint64_t document::tape_ref::tape_value() const noexcept { - return doc->tape[json_index] & JSON_VALUE_MASK; +really_inline uint64_t internal::tape_ref::tape_value() const noexcept { + return doc->tape[json_index] & internal::JSON_VALUE_MASK; } template -really_inline T document::tape_ref::next_tape_value() const noexcept { +really_inline T internal::tape_ref::next_tape_value() const noexcept { static_assert(sizeof(T) == sizeof(uint64_t)); return *reinterpret_cast(&doc->tape[json_index + 1]); } +inline std::string_view internal::tape_ref::get_string_view() const noexcept { + size_t string_buf_index = tape_value(); + uint32_t len; + memcpy(&len, &doc->string_buf[string_buf_index], sizeof(len)); + return std::string_view( + reinterpret_cast(&doc->string_buf[string_buf_index + sizeof(uint32_t)]), + len + ); +} // -// document::array inline implementation +// array inline implementation // -really_inline document::array::array() noexcept : tape_ref() {} -really_inline document::array::array(const document *_doc, size_t _json_index) noexcept : tape_ref(_doc, _json_index) {} +really_inline document::array::array() noexcept : internal::tape_ref() {} +really_inline document::array::array(const document *_doc, size_t _json_index) noexcept : internal::tape_ref(_doc, _json_index) {} inline document::array::iterator document::array::begin() const noexcept { return iterator(doc, json_index + 1); } @@ -3063,7 +3918,7 @@ inline document::array::iterator document::array::end() const noexcept { // // document::array::iterator inline implementation // -really_inline document::array::iterator::iterator(const document *_doc, size_t _json_index) noexcept : tape_ref(_doc, _json_index) { } +really_inline document::array::iterator::iterator(const document *_doc, size_t _json_index) noexcept : internal::tape_ref(_doc, _json_index) { } inline document::element document::array::iterator::operator*() const noexcept { return element(doc, json_index); } @@ -3075,10 +3930,10 @@ inline void document::array::iterator::operator++() noexcept { } // -// document::object inline implementation +// object inline implementation // -really_inline document::object::object() noexcept : tape_ref() {} -really_inline document::object::object(const document *_doc, size_t _json_index) noexcept : tape_ref(_doc, _json_index) { }; +really_inline document::object::object() noexcept : internal::tape_ref() {} +really_inline document::object::object(const document *_doc, size_t _json_index) noexcept : internal::tape_ref(_doc, _json_index) { }; inline document::object::iterator document::object::begin() const noexcept { return iterator(doc, json_index + 1); } @@ -3107,7 +3962,7 @@ inline document::element_result document::object::operator[](const char *key) co // // document::object::iterator inline implementation // -really_inline document::object::iterator::iterator(const document *_doc, size_t _json_index) noexcept : tape_ref(_doc, _json_index) { } +really_inline document::object::iterator::iterator(const document *_doc, size_t _json_index) noexcept : internal::tape_ref(_doc, _json_index) { } inline const document::key_value_pair document::object::iterator::operator*() const noexcept { return key_value_pair(key(), value()); } @@ -3137,35 +3992,39 @@ inline document::element document::object::iterator::value() const noexcept { // // document::key_value_pair inline implementation // -inline document::key_value_pair::key_value_pair(std::string_view _key, document::element _value) noexcept : +inline document::key_value_pair::key_value_pair(std::string_view _key, element _value) noexcept : key(_key), value(_value) {} // -// document::element inline implementation +// element inline implementation // -really_inline document::element::element() noexcept : tape_ref() {} -really_inline document::element::element(const document *_doc, size_t _json_index) noexcept : tape_ref(_doc, _json_index) { } +really_inline document::element::element() noexcept : internal::tape_ref() {} +really_inline document::element::element(const document *_doc, size_t _json_index) noexcept : internal::tape_ref(_doc, _json_index) { } + really_inline bool document::element::is_null() const noexcept { - return type() == tape_type::NULL_VALUE; + return type() == internal::tape_type::NULL_VALUE; } really_inline bool document::element::is_bool() const noexcept { - return type() == tape_type::TRUE_VALUE || type() == tape_type::FALSE_VALUE; + return type() == internal::tape_type::TRUE_VALUE || type() == internal::tape_type::FALSE_VALUE; } really_inline bool document::element::is_number() const noexcept { - return type() == tape_type::UINT64 || type() == tape_type::INT64 || type() == tape_type::DOUBLE; + return type() == internal::tape_type::UINT64 || type() == internal::tape_type::INT64 || type() == internal::tape_type::DOUBLE; } really_inline bool document::element::is_integer() const noexcept { - return type() == tape_type::UINT64 || type() == tape_type::INT64; + return type() == internal::tape_type::UINT64 || type() == internal::tape_type::INT64; } really_inline bool document::element::is_string() const noexcept { - return type() == tape_type::STRING; + return type() == internal::tape_type::STRING; } really_inline bool document::element::is_array() const noexcept { - return type() == tape_type::START_ARRAY; + return type() == internal::tape_type::START_ARRAY; } really_inline bool document::element::is_object() const noexcept { - return type() == tape_type::START_OBJECT; + return type() == internal::tape_type::START_OBJECT; } + +#if SIMDJSON_EXCEPTIONS + inline document::element::operator bool() const noexcept(false) { return as_bool(); } inline document::element::operator const char*() const noexcept(false) { return as_c_str(); } inline document::element::operator std::string_view() const noexcept(false) { return as_string(); } @@ -3174,19 +4033,22 @@ inline document::element::operator int64_t() const noexcept(false) { return as_i inline document::element::operator double() const noexcept(false) { return as_double(); } inline document::element::operator document::array() const noexcept(false) { return as_array(); } inline document::element::operator document::object() const noexcept(false) { return as_object(); } -inline document::simdjson_result document::element::as_bool() const noexcept { + +#endif + +inline simdjson_result document::element::as_bool() const noexcept { switch (type()) { - case tape_type::TRUE_VALUE: + case internal::tape_type::TRUE_VALUE: return true; - case tape_type::FALSE_VALUE: + case internal::tape_type::FALSE_VALUE: return false; default: return INCORRECT_TYPE; } } -inline document::simdjson_result document::element::as_c_str() const noexcept { +inline simdjson_result document::element::as_c_str() const noexcept { switch (type()) { - case tape_type::STRING: { + case internal::tape_type::STRING: { size_t string_buf_index = tape_value(); return reinterpret_cast(&doc->string_buf[string_buf_index + sizeof(uint32_t)]); } @@ -3194,26 +4056,19 @@ inline document::simdjson_result document::element::as_c_str() con return INCORRECT_TYPE; } } -inline document::simdjson_result document::element::as_string() const noexcept { +inline simdjson_result document::element::as_string() const noexcept { switch (type()) { - case tape_type::STRING: { - size_t string_buf_index = tape_value(); - uint32_t len; - memcpy(&len, &doc->string_buf[string_buf_index], sizeof(len)); - return std::string_view( - reinterpret_cast(&doc->string_buf[string_buf_index + sizeof(uint32_t)]), - len - ); - } + case internal::tape_type::STRING: + return get_string_view(); default: return INCORRECT_TYPE; } } -inline document::simdjson_result document::element::as_uint64_t() const noexcept { +inline simdjson_result document::element::as_uint64_t() const noexcept { switch (type()) { - case tape_type::UINT64: + case internal::tape_type::UINT64: return next_tape_value(); - case tape_type::INT64: { + case internal::tape_type::INT64: { int64_t result = next_tape_value(); if (result < 0) { return NUMBER_OUT_OF_RANGE; @@ -3224,9 +4079,9 @@ inline document::simdjson_result document::element::as_uint64_t() cons return INCORRECT_TYPE; } } -inline document::simdjson_result document::element::as_int64_t() const noexcept { +inline simdjson_result document::element::as_int64_t() const noexcept { switch (type()) { - case tape_type::UINT64: { + case internal::tape_type::UINT64: { uint64_t result = next_tape_value(); // Wrapping max in parens to handle Windows issue: https://stackoverflow.com/questions/11544073/how-do-i-deal-with-the-max-macro-in-windows-h-colliding-with-max-in-std if (result > (std::numeric_limits::max)()) { @@ -3234,18 +4089,17 @@ inline document::simdjson_result document::element::as_int64_t() const } return static_cast(result); } - case tape_type::INT64: + case internal::tape_type::INT64: return next_tape_value(); default: - std::cout << "Incorrect " << json_index << " = " << char(type()) << std::endl; return INCORRECT_TYPE; } } -inline document::simdjson_result document::element::as_double() const noexcept { +inline simdjson_result document::element::as_double() const noexcept { switch (type()) { - case tape_type::UINT64: + case internal::tape_type::UINT64: return next_tape_value(); - case tape_type::INT64: { + case internal::tape_type::INT64: { return next_tape_value(); int64_t result = tape_value(); if (result < 0) { @@ -3253,7 +4107,7 @@ inline document::simdjson_result document::element::as_double() const no } return result; } - case tape_type::DOUBLE: + case internal::tape_type::DOUBLE: return next_tape_value(); default: return INCORRECT_TYPE; @@ -3261,7 +4115,7 @@ inline document::simdjson_result document::element::as_double() const no } inline document::array_result document::element::as_array() const noexcept { switch (type()) { - case tape_type::START_ARRAY: + case internal::tape_type::START_ARRAY: return array(doc, json_index); default: return INCORRECT_TYPE; @@ -3269,7 +4123,7 @@ inline document::array_result document::element::as_array() const noexcept { } inline document::object_result document::element::as_object() const noexcept { switch (type()) { - case tape_type::START_OBJECT: + case internal::tape_type::START_OBJECT: return object(doc, json_index); default: return INCORRECT_TYPE; @@ -3286,6 +4140,198 @@ inline document::element_result document::element::operator[](const char *key) c return obj[key]; } +// +// minify inline implementation +// + +template<> +inline std::ostream& minify::print(std::ostream& out) { + return out << minify(value.root()); +} +template<> +inline std::ostream& minify::print(std::ostream& out) { + using tape_type=internal::tape_type; + size_t depth = 0; + constexpr size_t MAX_DEPTH = 16; + bool is_object[MAX_DEPTH]; + is_object[0] = false; + bool after_value = false; + + internal::tape_ref iter(value.doc, value.json_index); + do { + // print commas after each value + if (after_value) { + out << ","; + } + // If we are in an object, print the next key and :, and skip to the next value. + if (is_object[depth]) { + out << '"' << internal::escape_json_string(iter.get_string_view()) << "\":"; + iter.json_index++; + } + switch (iter.type()) { + + // Arrays + case tape_type::START_ARRAY: { + // If we're too deep, we need to recurse to go deeper. + depth++; + if (unlikely(depth >= MAX_DEPTH)) { + out << minify(document::array(iter.doc, iter.json_index)); + iter.json_index = iter.tape_value() - 1; // Jump to the ] + depth--; + break; + } + + // Output start [ + out << '['; + iter.json_index++; + + // Handle empty [] (we don't want to come back around and print commas) + if (iter.type() == tape_type::END_ARRAY) { + out << ']'; + depth--; + break; + } + + is_object[depth] = false; + after_value = false; + continue; + } + + // Objects + case tape_type::START_OBJECT: { + // If we're too deep, we need to recurse to go deeper. + depth++; + if (unlikely(depth >= MAX_DEPTH)) { + out << minify(document::object(iter.doc, iter.json_index)); + iter.json_index = iter.tape_value() - 1; // Jump to the } + depth--; + break; + } + + // Output start { + out << '{'; + iter.json_index++; + + // Handle empty {} (we don't want to come back around and print commas) + if (iter.type() == tape_type::END_OBJECT) { + out << '}'; + depth--; + break; + } + + is_object[depth] = true; + after_value = false; + continue; + } + + // Scalars + case tape_type::STRING: + out << '"' << internal::escape_json_string(iter.get_string_view()) << '"'; + break; + case tape_type::INT64: + out << iter.next_tape_value(); + iter.json_index++; // numbers take up 2 spots, so we need to increment extra + break; + case tape_type::UINT64: + out << iter.next_tape_value(); + iter.json_index++; // numbers take up 2 spots, so we need to increment extra + break; + case tape_type::DOUBLE: + out << iter.next_tape_value(); + iter.json_index++; // numbers take up 2 spots, so we need to increment extra + break; + case tape_type::TRUE_VALUE: + out << "true"; + break; + case tape_type::FALSE_VALUE: + out << "false"; + break; + case tape_type::NULL_VALUE: + out << "null"; + break; + + // These are impossible + case tape_type::END_ARRAY: + case tape_type::END_OBJECT: + case tape_type::ROOT: + abort(); + } + iter.json_index++; + after_value = true; + + // Handle multiple ends in a row + while (depth != 0 && (iter.type() == tape_type::END_ARRAY || iter.type() == tape_type::END_OBJECT)) { + out << char(iter.type()); + depth--; + iter.json_index++; + } + + // Stop when we're at depth 0 + } while (depth != 0); + + return out; +} +template<> +inline std::ostream& minify::print(std::ostream& out) { + out << '{'; + auto pair = value.begin(); + auto end = value.end(); + if (pair != end) { + out << minify(*pair); + for (++pair; pair != end; ++pair) { + out << "," << minify(*pair); + } + } + return out << '}'; +} +template<> +inline std::ostream& minify::print(std::ostream& out) { + out << '['; + auto element = value.begin(); + auto end = value.end(); + if (element != end) { + out << minify(*element); + for (++element; element != end; ++element) { + out << "," << minify(*element); + } + } + return out << ']'; +} +template<> +inline std::ostream& minify::print(std::ostream& out) { + return out << '"' << internal::escape_json_string(value.key) << "\":" << value.value; +} + +#if SIMDJSON_EXCEPTIONS + +template<> +inline std::ostream& minify::print(std::ostream& out) { + if (value.error()) { throw simdjson_error(value.error()); } + return out << minify(value.first); +} +template<> +inline std::ostream& minify::print(std::ostream& out) { + if (value.error()) { throw simdjson_error(value.error()); } + return out << minify(value.first); +} +template<> +inline std::ostream& minify::print(std::ostream& out) { + if (value.error()) { throw simdjson_error(value.error()); } + return out << minify(value.first); +} +template<> +inline std::ostream& minify::print(std::ostream& out) { + if (value.error()) { throw simdjson_error(value.error()); } + return out << minify(value.first); +} +template<> +inline std::ostream& minify::print(std::ostream& out) { + if (value.error()) { throw simdjson_error(value.error()); } + return out << minify(value.first); +} + +#endif + } // namespace simdjson #endif // SIMDJSON_INLINE_DOCUMENT_H @@ -3442,7 +4488,7 @@ template bool document_iterator::prev() { oldnpos = npos; if ((current_type == '[') || (current_type == '{')) { // we need to jump - npos = (current_val & JSON_VALUE_MASK); + npos = (current_val & internal::JSON_VALUE_MASK); } else { npos = npos + ((current_type == 'd' || current_type == 'l') ? 2 : 1); } @@ -3471,7 +4517,7 @@ template bool document_iterator::down() { return false; } if ((current_type == '[') || (current_type == '{')) { - size_t npos = (current_val & JSON_VALUE_MASK); + size_t npos = (current_val & internal::JSON_VALUE_MASK); if (npos == location + 2) { return false; // we have an empty scope } @@ -3498,7 +4544,7 @@ template bool document_iterator::next() { size_t npos; if ((current_type == '[') || (current_type == '{')) { // we need to jump - npos = (current_val & JSON_VALUE_MASK); + npos = (current_val & internal::JSON_VALUE_MASK); } else { npos = location + (is_number() ? 2 : 1); } @@ -3520,7 +4566,7 @@ document_iterator::document_iterator(const document &doc_) noexcept current_val = doc.tape[location++]; current_type = (current_val >> 56); depth_index[0].scope_type = current_type; - tape_length = current_val & JSON_VALUE_MASK; + tape_length = current_val & internal::JSON_VALUE_MASK; if (location < tape_length) { // If we make it here, then depth_capacity must >=2, but the compiler // may not know this. @@ -3533,10 +4579,14 @@ document_iterator::document_iterator(const document &doc_) noexcept } } +#if SIMDJSON_EXCEPTIONS + template -document_iterator::document_iterator(const document::parser &parser) +document_iterator::document_iterator(const document::parser &parser) noexcept(false) : document_iterator(parser.get_document()) {} +#endif + template document_iterator::document_iterator( const document_iterator &o) noexcept @@ -3568,7 +4618,7 @@ bool document_iterator::print(std::ostream &os, bool escape_strings) case '"': // we have a string os << '"'; if (escape_strings) { - internal::print_with_escapes(get_string(), os, get_string_length()); + os << internal::escape_json_string(std::string_view(get_string(), get_string_length())); } else { // was: os << get_string();, but given that we can include null chars, we // have to do something crazier: @@ -3616,7 +4666,9 @@ bool document_iterator::move_to(const char *pointer, uint32_t new_length = 0; for (uint32_t i = 1; i < length; i++) { if (pointer[i] == '%' && pointer[i + 1] == 'x') { +#if __cpp_exceptions try { +#endif int fragment = std::stoi(std::string(&pointer[i + 2], 2), nullptr, 16); if (fragment == '\\' || fragment == '"' || (fragment <= 0x1F)) { @@ -3626,10 +4678,12 @@ bool document_iterator::move_to(const char *pointer, } new_pointer[new_length] = fragment; i += 3; +#if __cpp_exceptions } catch (std::invalid_argument &) { delete[] new_pointer; return false; // the fragment is invalid } +#endif } else { new_pointer[new_length] = pointer[i]; } @@ -3748,7 +4802,7 @@ bool document_iterator::relative_move_to(const char *pointer, size_t npos; if ((current_type == '[') || (current_type == '{')) { // we need to jump - npos = (current_val & JSON_VALUE_MASK); + npos = (current_val & internal::JSON_VALUE_MASK); } else { npos = location + ((current_type == 'd' || current_type == 'l') ? 2 : 1); @@ -3782,9 +4836,9 @@ bool document_iterator::relative_move_to(const char *pointer, #endif // SIMDJSON_INLINE_DOCUMENT_ITERATOR_H /* end file include/simdjson/inline/document_iterator.h */ -/* begin file include/simdjson/inline/jsonstream.h */ -#ifndef SIMDJSON_INLINE_JSONSTREAM_H -#define SIMDJSON_INLINE_JSONSTREAM_H +/* begin file include/simdjson/inline/document_stream.h */ +#ifndef SIMDJSON_INLINE_DOCUMENT_STREAM_H +#define SIMDJSON_INLINE_DOCUMENT_STREAM_H #include #include @@ -3793,7 +4847,8 @@ bool document_iterator::relative_move_to(const char *pointer, namespace simdjson::internal { -/* This algorithm is used to quickly identify the buffer position of +/** + * This algorithm is used to quickly identify the buffer position of * the last JSON document inside the current batch. * * It does its work by finding the last pair of structural characters @@ -3880,13 +4935,17 @@ static inline size_t trimmed_length_safe_utf8(const char * c, size_t len) { namespace simdjson { -template -JsonStream::JsonStream(const string_container &s, - size_t batchSize) - : str(s), _batch_size(batchSize) { +really_inline document::stream::stream( + document::parser &_parser, + const uint8_t *buf, + size_t len, + size_t batch_size, + error_code _error +) noexcept : parser{_parser}, _buf{buf}, _len{len}, _batch_size(batch_size), error{_error} { + if (!error) { error = json_parse(); } } -template JsonStream::~JsonStream() { +inline document::stream::~stream() noexcept { #ifdef SIMDJSON_THREADS_ENABLED if (stage_1_thread.joinable()) { stage_1_thread.join(); @@ -3894,42 +4953,61 @@ template JsonStream::~JsonStream() { #endif } +really_inline document::stream::iterator document::stream::begin() noexcept { + return iterator(*this, false); +} + +really_inline document::stream::iterator document::stream::end() noexcept { + return iterator(*this, true); +} + +really_inline document::stream::iterator::iterator(stream& stream, bool _is_end) noexcept + : _stream{stream}, finished{_is_end} { +} + +really_inline document::doc_result document::stream::iterator::operator*() noexcept { + return doc_result(_stream.parser.doc, _stream.error == SUCCESS_AND_HAS_MORE ? SUCCESS : _stream.error); +} + +really_inline document::stream::iterator& document::stream::iterator::operator++() noexcept { + if (_stream.error == SUCCESS_AND_HAS_MORE) { + _stream.error = _stream.json_parse(); + } else { + finished = true; + } + return *this; +} + +really_inline bool document::stream::iterator::operator!=(const document::stream::iterator &other) const noexcept { + return finished != other.finished; +} + #ifdef SIMDJSON_THREADS_ENABLED // threaded version of json_parse // todo: simplify this code further -template -int JsonStream::json_parse(document::parser &parser) { - if (unlikely(parser.capacity() == 0)) { - const bool allocok = parser.allocate_capacity(_batch_size); - if (!allocok) { - return parser.error = simdjson::MEMALLOC; - } - } else if (unlikely(parser.capacity() < _batch_size)) { - return parser.error = simdjson::CAPACITY; - } - if (unlikely(parser_thread.capacity() < _batch_size)) { - const bool allocok_thread = parser_thread.allocate_capacity(_batch_size); - if (!allocok_thread) { - return parser.error = simdjson::MEMALLOC; - } - } +inline error_code document::stream::json_parse() noexcept { + error = parser.ensure_capacity(_batch_size); + if (error) { return error; } + error = parser_thread.ensure_capacity(_batch_size); + if (error) { return error; } + if (unlikely(load_next_batch)) { // First time loading if (!stage_1_thread.joinable()) { _batch_size = (std::min)(_batch_size, remaining()); _batch_size = internal::trimmed_length_safe_utf8((const char *)buf(), _batch_size); if (_batch_size == 0) { - return parser.error = simdjson::UTF8_ERROR; + return simdjson::UTF8_ERROR; } auto stage1_is_ok = error_code(simdjson::active_implementation->stage1(buf(), _batch_size, parser, true)); if (stage1_is_ok != simdjson::SUCCESS) { - return parser.error = stage1_is_ok; + return stage1_is_ok; } size_t last_index = internal::find_last_json_buf_idx(buf(), _batch_size, parser); if (last_index == 0) { if (parser.n_structural_indexes == 0) { - return parser.error = simdjson::EMPTY; + return simdjson::EMPTY; } } else { parser.n_structural_indexes = last_index + 1; @@ -3939,7 +5017,7 @@ int JsonStream::json_parse(document::parser &parser) { else { stage_1_thread.join(); if (stage1_is_ok_thread != simdjson::SUCCESS) { - return parser.error = stage1_is_ok_thread; + return stage1_is_ok_thread; } std::swap(parser.structural_indexes, parser_thread.structural_indexes); parser.n_structural_indexes = parser_thread.n_structural_indexes; @@ -3955,7 +5033,7 @@ int JsonStream::json_parse(document::parser &parser) { _batch_size = internal::trimmed_length_safe_utf8( (const char *)(buf() + last_json_buffer_loc), _batch_size); if (_batch_size == 0) { - return parser.error = simdjson::UTF8_ERROR; + return simdjson::UTF8_ERROR; } // let us capture read-only variables const uint8_t *const b = buf() + last_json_buffer_loc; @@ -3971,7 +5049,7 @@ int JsonStream::json_parse(document::parser &parser) { next_json = 0; load_next_batch = false; } // load_next_batch - int res = simdjson::active_implementation->stage2(buf(), remaining(), parser, next_json); + error_code res = simdjson::active_implementation->stage2(buf(), remaining(), parser, next_json); if (res == simdjson::SUCCESS_AND_HAS_MORE) { n_parsed_docs++; current_buffer_loc = parser.structural_indexes[next_json]; @@ -3990,18 +5068,10 @@ int JsonStream::json_parse(document::parser &parser) { #else // SIMDJSON_THREADS_ENABLED // single-threaded version of json_parse -template -int JsonStream::json_parse(document::parser &parser) { - if (unlikely(parser.capacity() == 0)) { - const bool allocok = parser.allocate_capacity(_batch_size); - if (!allocok) { - parser.valid = false; - return parser.error = MEMALLOC; - } - } else if (unlikely(parser.capacity() < _batch_size)) { - parser.valid = false; - return parser.error = CAPACITY; - } +inline error_code document::stream::json_parse() noexcept { + error = parser.ensure_capacity(_batch_size); + if (error) { return error; } + if (unlikely(load_next_batch)) { advance(current_buffer_loc); n_bytes_parsed += current_buffer_loc; @@ -4009,21 +5079,19 @@ int JsonStream::json_parse(document::parser &parser) { _batch_size = internal::trimmed_length_safe_utf8((const char *)buf(), _batch_size); auto stage1_is_ok = (error_code)simdjson::active_implementation->stage1(buf(), _batch_size, parser, true); if (stage1_is_ok != simdjson::SUCCESS) { - parser.valid = false; - return parser.error = stage1_is_ok; + return stage1_is_ok; } size_t last_index = internal::find_last_json_buf_idx(buf(), _batch_size, parser); if (last_index == 0) { if (parser.n_structural_indexes == 0) { - parser.valid = false; - return parser.error = EMPTY; + return EMPTY; } } else { parser.n_structural_indexes = last_index + 1; } load_next_batch = false; } // load_next_batch - int res = simdjson::active_implementation->stage2(buf(), remaining(), parser, next_json); + error_code res = simdjson::active_implementation->stage2(buf(), remaining(), parser, next_json); if (likely(res == simdjson::SUCCESS_AND_HAS_MORE)) { n_parsed_docs++; current_buffer_loc = parser.structural_indexes[next_json]; @@ -4035,16 +5103,248 @@ int JsonStream::json_parse(document::parser &parser) { load_next_batch = true; res = simdjson::SUCCESS_AND_HAS_MORE; } - } else { - printf("E\n"); } return res; } #endif // SIMDJSON_THREADS_ENABLED } // end of namespace simdjson +#endif // SIMDJSON_INLINE_DOCUMENT_STREAM_H +/* end file include/simdjson/inline/document_stream.h */ +/* begin file include/simdjson/inline/error.h */ +#ifndef SIMDJSON_INLINE_ERROR_H +#define SIMDJSON_INLINE_ERROR_H + +#include + +namespace simdjson::internal { + // We store the error code so we can validate the error message is associated with the right code + struct error_code_info { + error_code code; + std::string message; + }; + // These MUST match the codes in error_code. We check this constraint in basictests. + inline const error_code_info error_codes[] { + { SUCCESS, "No error" }, + { SUCCESS_AND_HAS_MORE, "No error and buffer still has more data" }, + { CAPACITY, "This parser can't support a document that big" }, + { MEMALLOC, "Error allocating memory, we're most likely out of memory" }, + { TAPE_ERROR, "Something went wrong while writing to the tape" }, + { DEPTH_ERROR, "The JSON document was too deep (too many nested objects and arrays)" }, + { STRING_ERROR, "Problem while parsing a string" }, + { T_ATOM_ERROR, "Problem while parsing an atom starting with the letter 't'" }, + { F_ATOM_ERROR, "Problem while parsing an atom starting with the letter 'f'" }, + { N_ATOM_ERROR, "Problem while parsing an atom starting with the letter 'n'" }, + { NUMBER_ERROR, "Problem while parsing a number" }, + { UTF8_ERROR, "The input is not valid UTF-8" }, + { UNINITIALIZED, "Uninitialized" }, + { EMPTY, "Empty: no JSON found" }, + { UNESCAPED_CHARS, "Within strings, some characters must be escaped, we found unescaped characters" }, + { UNCLOSED_STRING, "A string is opened, but never closed." }, + { UNSUPPORTED_ARCHITECTURE, "simdjson does not have an implementation supported by this CPU architecture (perhaps it's a non-SIMD CPU?)." }, + { INCORRECT_TYPE, "The JSON element does not have the requested type." }, + { NUMBER_OUT_OF_RANGE, "The JSON number is too large or too small to fit within the requested type." }, + { NO_SUCH_FIELD, "The JSON field referenced does not exist in this object." }, + { IO_ERROR, "Error reading the file." }, + { UNEXPECTED_ERROR, "Unexpected error, consider reporting this problem as you may have found a bug in simdjson" } + }; // error_messages[] +} // namespace simdjson::internal + +namespace simdjson { + +inline const char *error_message(error_code error) noexcept { + // If you're using error_code, we're trusting you got it from the enum. + return internal::error_codes[int(error)].message.c_str(); +} + +inline const std::string &error_message(int error) noexcept { + if (error < 0 || error >= error_code::NUM_ERROR_CODES) { + return internal::error_codes[UNEXPECTED_ERROR].message; + } + return internal::error_codes[error].message; +} + +inline std::ostream& operator<<(std::ostream& out, error_code error) noexcept { + return out << error_message(error); +} + +} // namespace simdjson + +#endif // SIMDJSON_INLINE_ERROR_H +/* end file include/simdjson/inline/error.h */ +/* begin file include/simdjson/inline/jsonstream.h */ +// TODO Remove this -- deprecated API and files + +#ifndef SIMDJSON_INLINE_JSONSTREAM_H +#define SIMDJSON_INLINE_JSONSTREAM_H + + +namespace simdjson { + +template +inline JsonStream::JsonStream(const string_container &s, size_t _batch_size) noexcept + : str(s), batch_size(_batch_size) { +} + +template +inline JsonStream::~JsonStream() noexcept { + if (stream) { delete stream; } +} + +template +inline int JsonStream::json_parse(document::parser &parser) noexcept { + if (unlikely(stream == nullptr)) { + stream = new document::stream(parser, reinterpret_cast(str.data()), str.length(), batch_size); + } else { + if (&parser != &stream->parser) { return stream->error = TAPE_ERROR; } + stream->error = stream->json_parse(); + } + return stream->error; +} + +} // namespace simdjson + #endif // SIMDJSON_INLINE_JSONSTREAM_H /* end file include/simdjson/inline/jsonstream.h */ +/* begin file include/simdjson/inline/padded_string.h */ +#ifndef SIMDJSON_INLINE_PADDED_STRING_H +#define SIMDJSON_INLINE_PADDED_STRING_H + + +#include +#include +#include +#include + +namespace simdjson::internal { + +// low-level function to allocate memory with padding so we can read past the +// "length" bytes safely. if you must provide a pointer to some data, create it +// with this function: length is the max. size in bytes of the string caller is +// responsible to free the memory (free(...)) +inline char *allocate_padded_buffer(size_t length) noexcept { + // we could do a simple malloc + // return (char *) malloc(length + SIMDJSON_PADDING); + // However, we might as well align to cache lines... + size_t totalpaddedlength = length + SIMDJSON_PADDING; + char *padded_buffer = aligned_malloc_char(64, totalpaddedlength); +#ifndef NDEBUG + if (padded_buffer == nullptr) { + return nullptr; + } +#endif // NDEBUG + memset(padded_buffer + length, 0, totalpaddedlength - length); + return padded_buffer; +} // allocate_padded_buffer() + +} // namespace simdjson::internal + +namespace simdjson { + +inline padded_string::padded_string() noexcept : viable_size(0), data_ptr(nullptr) {} +inline padded_string::padded_string(size_t length) noexcept + : viable_size(length), data_ptr(internal::allocate_padded_buffer(length)) { + if (data_ptr != nullptr) + data_ptr[length] = '\0'; // easier when you need a c_str +} +inline padded_string::padded_string(const char *data, size_t length) noexcept + : viable_size(length), data_ptr(internal::allocate_padded_buffer(length)) { + if ((data != nullptr) and (data_ptr != nullptr)) { + memcpy(data_ptr, data, length); + data_ptr[length] = '\0'; // easier when you need a c_str + } +} +// note: do not pass std::string arguments by value +inline padded_string::padded_string(const std::string & str_ ) noexcept + : viable_size(str_.size()), data_ptr(internal::allocate_padded_buffer(str_.size())) { + if (data_ptr != nullptr) { + memcpy(data_ptr, str_.data(), str_.size()); + data_ptr[str_.size()] = '\0'; // easier when you need a c_str + } +} +// note: do pass std::string_view arguments by value +inline padded_string::padded_string(std::string_view sv_) noexcept + : viable_size(sv_.size()), data_ptr(internal::allocate_padded_buffer(sv_.size())) { + if (data_ptr != nullptr) { + memcpy(data_ptr, sv_.data(), sv_.size()); + data_ptr[sv_.size()] = '\0'; // easier when you need a c_str + } +} +inline padded_string::padded_string(padded_string &&o) noexcept + : viable_size(o.viable_size), data_ptr(o.data_ptr) { + o.data_ptr = nullptr; // we take ownership +} + +inline padded_string &padded_string::operator=(padded_string &&o) noexcept { + aligned_free_char(data_ptr); + data_ptr = o.data_ptr; + viable_size = o.viable_size; + o.data_ptr = nullptr; // we take ownership + o.viable_size = 0; + return *this; +} + +inline void padded_string::swap(padded_string &o) noexcept { + size_t tmp_viable_size = viable_size; + char *tmp_data_ptr = data_ptr; + viable_size = o.viable_size; + data_ptr = o.data_ptr; + o.data_ptr = tmp_data_ptr; + o.viable_size = tmp_viable_size; +} + +inline padded_string::~padded_string() noexcept { + aligned_free_char(data_ptr); +} + +inline size_t padded_string::size() const noexcept { return viable_size; } + +inline size_t padded_string::length() const noexcept { return viable_size; } + +inline const char *padded_string::data() const noexcept { return data_ptr; } + +inline char *padded_string::data() noexcept { return data_ptr; } + +inline simdjson_move_result padded_string::load(const std::string &filename) noexcept { + // Open the file + std::FILE *fp = std::fopen(filename.c_str(), "rb"); + if (fp == nullptr) { + return IO_ERROR; + } + + // Get the file size + if(std::fseek(fp, 0, SEEK_END) < 0) { + std::fclose(fp); + return IO_ERROR; + } + long llen = std::ftell(fp); + if((llen < 0) || (llen == LONG_MAX)) { + std::fclose(fp); + return IO_ERROR; + } + + // Allocate the padded_string + size_t len = (size_t) llen; + padded_string s(len); + if (s.data() == nullptr) { + std::fclose(fp); + return MEMALLOC; + } + + // Read the padded_string + std::rewind(fp); + size_t bytes_read = std::fread(s.data(), 1, len, fp); + if (std::fclose(fp) != 0 || bytes_read != len) { + return IO_ERROR; + } + + return std::move(s); +} + +} // namespace simdjson + +#endif // SIMDJSON_INLINE_PADDED_STRING_H +/* end file include/simdjson/inline/padded_string.h */ #endif // SIMDJSON_H -/* end file include/simdjson/inline/jsonstream.h */ +/* end file include/simdjson/inline/padded_string.h */