diff --git a/amalgamation.sh b/amalgamation.sh index bb8afb6a4..0484dde3c 100755 --- a/amalgamation.sh +++ b/amalgamation.sh @@ -27,19 +27,7 @@ stage2_build_tape.cpp # order matters ALLCHEADERS=" -simdjson/simdjson_version.h -simdjson/portability.h -simdjson/isadetection.h -simdjson/jsonformatutils.h -simdjson/simdjson.h -simdjson/common_defs.h -simdjson/padded_string.h -simdjson/jsonioutil.h -simdjson/jsonminifier.h -simdjson/document.h -simdjson/parsedjson.h -simdjson/jsonparser.h -simdjson/jsonstream.h +simdjson.h " found_includes=() diff --git a/benchmark/bench_parse_call.cpp b/benchmark/bench_parse_call.cpp index cf9b7040f..4cb442eb2 100644 --- a/benchmark/bench_parse_call.cpp +++ b/benchmark/bench_parse_call.cpp @@ -1,6 +1,5 @@ #include -#include "simdjson/document.h" -#include "simdjson/jsonparser.h" +#include "simdjson.h" using namespace simdjson; using namespace benchmark; using namespace std; diff --git a/benchmark/benchfeatures.cpp b/benchmark/benchfeatures.cpp index 8a2ba88b3..c99a601fa 100644 --- a/benchmark/benchfeatures.cpp +++ b/benchmark/benchfeatures.cpp @@ -29,11 +29,8 @@ #ifdef __linux__ #include #endif -//#define DEBUG -#include "simdjson/common_defs.h" -#include "simdjson/isadetection.h" -#include "simdjson/jsonioutil.h" -#include "simdjson/document.h" + +#include "simdjson.h" #include diff --git a/benchmark/benchmarker.h b/benchmark/benchmarker.h index 4b2eb87d9..60700c333 100644 --- a/benchmark/benchmarker.h +++ b/benchmark/benchmarker.h @@ -32,11 +32,7 @@ #include #endif //#define DEBUG -#include "simdjson/common_defs.h" -#include "simdjson/isadetection.h" -#include "simdjson/jsonioutil.h" -#include "simdjson/jsonparser.h" -#include "simdjson/document.h" +#include "simdjson.h" #include diff --git a/benchmark/distinctuseridcompetition.cpp b/benchmark/distinctuseridcompetition.cpp index 7064415a9..9a003c0da 100644 --- a/benchmark/distinctuseridcompetition.cpp +++ b/benchmark/distinctuseridcompetition.cpp @@ -1,5 +1,4 @@ -#include "simdjson/jsonioutil.h" -#include "simdjson/document.h" +#include "simdjson.h" #include #include #include diff --git a/benchmark/event_counter.h b/benchmark/event_counter.h index a1d427bfc..3baa6b3a6 100644 --- a/benchmark/event_counter.h +++ b/benchmark/event_counter.h @@ -29,9 +29,8 @@ #ifdef __linux__ #include #endif -//#define DEBUG -#include "simdjson/common_defs.h" -#include "simdjson/isadetection.h" + +#include "simdjson.h" using std::string; using std::vector; diff --git a/benchmark/get_corpus_benchmark.cpp b/benchmark/get_corpus_benchmark.cpp index c62a6a621..26ea2b450 100644 --- a/benchmark/get_corpus_benchmark.cpp +++ b/benchmark/get_corpus_benchmark.cpp @@ -1,7 +1,5 @@ -#include "simdjson/common_defs.h" -#include "simdjson/jsonioutil.h" -#include "simdjson/jsonparser.h" +#include "simdjson.h" #include #include #include diff --git a/benchmark/minifiercompetition.cpp b/benchmark/minifiercompetition.cpp index bb3d159be..8700564e1 100644 --- a/benchmark/minifiercompetition.cpp +++ b/benchmark/minifiercompetition.cpp @@ -2,9 +2,7 @@ #include #include "benchmark.h" -#include "simdjson/jsonioutil.h" -#include "simdjson/jsonminifier.h" -#include "simdjson/jsonparser.h" +#include "simdjson.h" // #define RAPIDJSON_SSE2 // bad // #define RAPIDJSON_SSE42 // bad diff --git a/benchmark/parse.cpp b/benchmark/parse.cpp index 6508bb874..8982e7706 100644 --- a/benchmark/parse.cpp +++ b/benchmark/parse.cpp @@ -28,11 +28,8 @@ #ifdef __linux__ #include #endif -//#define DEBUG -#include "simdjson/common_defs.h" -#include "simdjson/isadetection.h" -#include "simdjson/jsonioutil.h" -#include "simdjson/document.h" + +#include "simdjson.h" #include diff --git a/benchmark/parse_stream.cpp b/benchmark/parse_stream.cpp index 2b4fa7099..7c969cb92 100755 --- a/benchmark/parse_stream.cpp +++ b/benchmark/parse_stream.cpp @@ -2,10 +2,8 @@ #include #include #include -#include "simdjson/jsonstream.h" +#include "simdjson.h" #include -#include "simdjson/jsonparser.h" -#include "simdjson/parsedjson.h" #define NB_ITERATION 5 #define MIN_BATCH_SIZE 200000 diff --git a/benchmark/parseandstatcompetition.cpp b/benchmark/parseandstatcompetition.cpp index 9a31486de..faf708cb9 100644 --- a/benchmark/parseandstatcompetition.cpp +++ b/benchmark/parseandstatcompetition.cpp @@ -1,4 +1,4 @@ -#include "simdjson/jsonparser.h" +#include "simdjson.h" #include #include "benchmark.h" diff --git a/benchmark/parsingcompetition.cpp b/benchmark/parsingcompetition.cpp index 55ba0bdd7..e12c2c0f7 100644 --- a/benchmark/parsingcompetition.cpp +++ b/benchmark/parsingcompetition.cpp @@ -1,4 +1,4 @@ -#include "simdjson/jsonparser.h" +#include "simdjson.h" #ifndef _MSC_VER #include "linux-perf-events.h" #include @@ -48,7 +48,7 @@ using namespace rapidjson; #ifdef ALLPARSER // fastjson has a tricky interface -void on_json_error(void *, const fastjson::ErrorContext &ec) { +void on_json_error(void *, UNUSED const fastjson::ErrorContext &ec) { // std::cerr<<"ERROR: "< #endif -#include "simdjson/jsonioutil.h" -#include "simdjson/jsonparser.h" +#include "simdjson.h" #ifdef __linux__ #include "linux-perf-events.h" #endif diff --git a/fuzz/fuzz_dump.cpp b/fuzz/fuzz_dump.cpp index 831a15c6d..c90fc5214 100644 --- a/fuzz/fuzz_dump.cpp +++ b/fuzz/fuzz_dump.cpp @@ -1,4 +1,4 @@ -#include "simdjson/jsonparser.h" +#include "simdjson.h" #include #include #include diff --git a/fuzz/fuzz_dump_raw_tape.cpp b/fuzz/fuzz_dump_raw_tape.cpp index 125e56c15..5c146b8c9 100644 --- a/fuzz/fuzz_dump_raw_tape.cpp +++ b/fuzz/fuzz_dump_raw_tape.cpp @@ -1,4 +1,4 @@ -#include "simdjson/jsonparser.h" +#include "simdjson.h" #include #include #include diff --git a/fuzz/fuzz_minify.cpp b/fuzz/fuzz_minify.cpp index ab6bc1650..63380b480 100644 --- a/fuzz/fuzz_minify.cpp +++ b/fuzz/fuzz_minify.cpp @@ -1,5 +1,5 @@ -#include "simdjson/jsonminifier.h" -#include "simdjson/jsonparser.h" +#include "simdjson.h" +#include "simdjson.h" #include #include #include diff --git a/fuzz/fuzz_parser.cpp b/fuzz/fuzz_parser.cpp index c1b11b7e0..dd3a5912d 100644 --- a/fuzz/fuzz_parser.cpp +++ b/fuzz/fuzz_parser.cpp @@ -1,4 +1,4 @@ -#include "simdjson/jsonparser.h" +#include "simdjson.h" #include #include #include diff --git a/fuzz/fuzz_print_json.cpp b/fuzz/fuzz_print_json.cpp index 4961e0844..57a2a220f 100644 --- a/fuzz/fuzz_print_json.cpp +++ b/fuzz/fuzz_print_json.cpp @@ -1,4 +1,4 @@ -#include "simdjson/jsonparser.h" +#include "simdjson.h" #include #include #include diff --git a/include/simdjson.h b/include/simdjson.h new file mode 100644 index 000000000..6764b21ae --- /dev/null +++ b/include/simdjson.h @@ -0,0 +1,25 @@ +#ifndef SIMDJSON_H +#define SIMDJSON_H + +#include "simdjson/compiler_check.h" + +// Public API +#include "simdjson/simdjson_version.h" +#include "simdjson/error.h" +#include "simdjson/padded_string.h" +#include "simdjson/implementation.h" +#include "simdjson/document.h" +#include "simdjson/jsonstream.h" +#include "simdjson/jsonminifier.h" + +// Deprecated API +#include "simdjson/parsedjsoniterator.h" +#include "simdjson/jsonparser.h" +#include "simdjson/parsedjson.h" +#include "simdjson/document_iterator.h" + +// Inline functions +#include "simdjson/inline/document.h" +#include "simdjson/inline/document_iterator.h" + +#endif // SIMDJSON_H diff --git a/include/simdjson/compiler_check.h b/include/simdjson/compiler_check.h new file mode 100644 index 000000000..533a196ac --- /dev/null +++ b/include/simdjson/compiler_check.h @@ -0,0 +1,20 @@ +#ifndef SIMDJSON_COMPILER_CHECK_H +#define SIMDJSON_COMPILER_CHECK_H + +#ifndef __cplusplus +#error simdjson requires a C++ compiler +#endif + +#ifndef SIMDJSON_CPLUSPLUS +#if defined(_MSVC_LANG) && !defined(__clang__) +#define SIMDJSON_CPLUSPLUS (_MSC_VER == 1900 ? 201103L : _MSVC_LANG) +#else +#define SIMDJSON_CPLUSPLUS __cplusplus +#endif +#endif + +#if (SIMDJSON_CPLUSPLUS < 201703L) +#error simdjson requires a compiler compliant with the C++17 standard +#endif + +#endif // SIMDJSON_COMPILER_CHECK_H diff --git a/include/simdjson/document.h b/include/simdjson/document.h index 82eb19f48..62601df8d 100644 --- a/include/simdjson/document.h +++ b/include/simdjson/document.h @@ -1084,7 +1084,4 @@ private: } // namespace simdjson -#include "simdjson/inline/document.h" -#include "simdjson/document_iterator.h" - #endif // SIMDJSON_DOCUMENT_H \ No newline at end of file diff --git a/include/simdjson/document_iterator.h b/include/simdjson/document_iterator.h index f5c77a21a..18b697524 100644 --- a/include/simdjson/document_iterator.h +++ b/include/simdjson/document_iterator.h @@ -261,6 +261,4 @@ public: } // namespace simdjson -#include "simdjson/inline/document_iterator.h" - #endif // SIMDJSON_DOCUMENT_ITERATOR_H diff --git a/include/simdjson/implementation.h b/include/simdjson/implementation.h index 2bf8a013f..14b8e89fc 100644 --- a/include/simdjson/implementation.h +++ b/include/simdjson/implementation.h @@ -1,6 +1,3 @@ -// Declaration order requires we get to document.h before implementation.h no matter what -#include "simdjson/document.h" - #ifndef SIMDJSON_IMPLEMENTATION_H #define SIMDJSON_IMPLEMENTATION_H @@ -8,6 +5,7 @@ #include #include #include +#include "simdjson/document.h" namespace simdjson { diff --git a/include/simdjson/inline/document.h b/include/simdjson/inline/document.h index f47d57762..9f7956286 100644 --- a/include/simdjson/inline/document.h +++ b/include/simdjson/inline/document.h @@ -1,9 +1,7 @@ #ifndef SIMDJSON_INLINE_DOCUMENT_H #define SIMDJSON_INLINE_DOCUMENT_H -#ifndef SIMDJSON_DOCUMENT_H -#error This is an internal file only. Include document.h instead. -#endif +#include "simdjson/document.h" // Inline implementations go in here if they aren't small enough to go in the class itself or if // there are complex header file dependencies that need to be broken by externalizing the diff --git a/include/simdjson/inline/document_iterator.h b/include/simdjson/inline/document_iterator.h index 9cf63bed8..9d86ec4f5 100644 --- a/include/simdjson/inline/document_iterator.h +++ b/include/simdjson/inline/document_iterator.h @@ -1,9 +1,7 @@ #ifndef SIMDJSON_INLINE_DOCUMENT_ITERATOR_H #define SIMDJSON_INLINE_DOCUMENT_ITERATOR_H -#ifndef SIMDJSON_DOCUMENT_ITERATOR_H -#error This is an internal file only. Include document.h instead. -#endif +#include "simdjson/document_iterator.h" namespace simdjson { diff --git a/include/simdjson/parsedjsoniterator.h b/include/simdjson/parsedjsoniterator.h index a1ad37139..d5e95548e 100644 --- a/include/simdjson/parsedjsoniterator.h +++ b/include/simdjson/parsedjsoniterator.h @@ -3,6 +3,6 @@ #ifndef SIMDJSON_PARSEDJSONITERATOR_H #define SIMDJSON_PARSEDJSONITERATOR_H -#include "document_iterator.h" +#include "simdjson/document_iterator.h" #endif diff --git a/include/simdjson/simdjson.h b/include/simdjson/simdjson.h index 1d7367715..1c5cc55a3 100644 --- a/include/simdjson/simdjson.h +++ b/include/simdjson/simdjson.h @@ -1,22 +1,11 @@ +/** + * @file + * @deprecated We'll be removing this file so it isn't confused with the top level simdjson.h + */ #ifndef SIMDJSON_SIMDJSON_H #define SIMDJSON_SIMDJSON_H -#ifndef __cplusplus -#error simdjson requires a C++ compiler -#endif - -#ifndef SIMDJSON_CPLUSPLUS -#if defined(_MSVC_LANG) && !defined(__clang__) -#define SIMDJSON_CPLUSPLUS (_MSC_VER == 1900 ? 201103L : _MSVC_LANG) -#else -#define SIMDJSON_CPLUSPLUS __cplusplus -#endif -#endif - -#if (SIMDJSON_CPLUSPLUS < 201703L) -#error simdjson requires a compiler compliant with the C++17 standard -#endif - +#include "simdjson/compiler_check.h" #include "simdjson/error.h" #endif // SIMDJSON_H diff --git a/singleheader/amalgamation_demo.cpp b/singleheader/amalgamation_demo.cpp index d3f189387..83f9088f8 100755 --- a/singleheader/amalgamation_demo.cpp +++ b/singleheader/amalgamation_demo.cpp @@ -1,4 +1,4 @@ -/* auto-generated on Sat Feb 22 10:41:58 PST 2020. Do not edit! */ +/* auto-generated on Mon Mar 2 14:10:52 PST 2020. Do not edit! */ #include #include "simdjson.h" @@ -11,7 +11,9 @@ int main(int argc, char *argv[]) { simdjson::padded_string p = simdjson::get_corpus(filename); auto [doc, error] = simdjson::document::parse(p); // do the parsing if (error) { - std::cout << "document::parse not valid" << std::endl; + std::cout << "document::parse failed" << std::endl; + std::cout << "error code: " << error << std::endl; + std::cout << error_message(error) << std::endl; } else { std::cout << "document::parse valid" << std::endl; } diff --git a/singleheader/simdjson.cpp b/singleheader/simdjson.cpp index faa55d97a..353d58716 100755 --- a/singleheader/simdjson.cpp +++ b/singleheader/simdjson.cpp @@ -1,4 +1,4 @@ -/* auto-generated on Sat Feb 22 10:41:58 PST 2020. Do not edit! */ +/* auto-generated on Mon Mar 2 14:10:52 PST 2020. Do not edit! */ #include "simdjson.h" /* used for http://dmalloc.com/ Dmalloc - Debug Malloc Library */ diff --git a/singleheader/simdjson.h b/singleheader/simdjson.h index 54b9cf7f1..79e1d22fc 100755 --- a/singleheader/simdjson.h +++ b/singleheader/simdjson.h @@ -1,4 +1,32 @@ -/* auto-generated on Sat Feb 22 10:41:58 PST 2020. Do not edit! */ +/* auto-generated on Mon Mar 2 14:10:52 PST 2020. Do not edit! */ +/* begin file include/simdjson.h */ +#ifndef SIMDJSON_H +#define SIMDJSON_H + +/* begin file include/simdjson/compiler_check.h */ +#ifndef SIMDJSON_COMPILER_CHECK_H +#define SIMDJSON_COMPILER_CHECK_H + +#ifndef __cplusplus +#error simdjson requires a C++ compiler +#endif + +#ifndef SIMDJSON_CPLUSPLUS +#if defined(_MSVC_LANG) && !defined(__clang__) +#define SIMDJSON_CPLUSPLUS (_MSC_VER == 1900 ? 201103L : _MSVC_LANG) +#else +#define SIMDJSON_CPLUSPLUS __cplusplus +#endif +#endif + +#if (SIMDJSON_CPLUSPLUS < 201703L) +#error simdjson requires a compiler compliant with the C++17 standard +#endif + +#endif // SIMDJSON_COMPILER_CHECK_H +/* end file include/simdjson/compiler_check.h */ + +// Public API /* begin file include/simdjson/simdjson_version.h */ // /include/simdjson/simdjson_version.h automatically generated by release.py, // do not change by hand @@ -14,6 +42,55 @@ enum { } #endif // SIMDJSON_SIMDJSON_VERSION_H /* end file include/simdjson/simdjson_version.h */ +/* begin file include/simdjson/error.h */ +#ifndef SIMDJSON_ERROR_H +#define SIMDJSON_ERROR_H + +#include + +namespace simdjson { + +enum error_code { + SUCCESS = 0, + SUCCESS_AND_HAS_MORE, //No errors and buffer still has more data + CAPACITY, // This parser can't support a document that big + MEMALLOC, // Error allocating memory, most likely out of memory + TAPE_ERROR, // Something went wrong while writing to the tape (stage 2), this + // is a generic error + DEPTH_ERROR, // Your document exceeds the user-specified depth limitation + STRING_ERROR, // Problem while parsing a string + T_ATOM_ERROR, // Problem while parsing an atom starting with the letter 't' + F_ATOM_ERROR, // Problem while parsing an atom starting with the letter 'f' + N_ATOM_ERROR, // Problem while parsing an atom starting with the letter 'n' + NUMBER_ERROR, // Problem while parsing a number + UTF8_ERROR, // the input is not valid UTF-8 + UNINITIALIZED, // unknown error, or uninitialized document + EMPTY, // no structural element found + UNESCAPED_CHARS, // found unescaped characters in a string. + UNCLOSED_STRING, // missing quote at the end + UNSUPPORTED_ARCHITECTURE, // unsupported architecture + UNEXPECTED_ERROR // indicative of a bug in simdjson +}; + +const std::string &error_message(error_code error) noexcept; + +struct invalid_json : public std::exception { + invalid_json(error_code _error) : error{_error} {} + const char *what() const noexcept { return error_message(error).c_str(); } + error_code error; +}; + +// TODO these are deprecated, remove +using ErrorValues = error_code; +inline const std::string &error_message(int error) noexcept { return error_message(error_code(error)); } + +} // namespace simdjson + +#endif // SIMDJSON_ERROR_H +/* end file include/simdjson/error.h */ +/* begin file include/simdjson/padded_string.h */ +#ifndef SIMDJSON_PADDING_STRING_H +#define SIMDJSON_PADDING_STRING_H /* begin file include/simdjson/portability.h */ #ifndef SIMDJSON_PORTABILITY_H #define SIMDJSON_PORTABILITY_H @@ -135,347 +212,6 @@ static inline void aligned_free_char(char *mem_block) { } // namespace simdjson #endif // SIMDJSON_PORTABILITY_H /* end file include/simdjson/portability.h */ -/* begin file include/simdjson/isadetection.h */ -/* From -https://github.com/endorno/pytorch/blob/master/torch/lib/TH/generic/simd/simd.h -Highly modified. - -Copyright (c) 2016- Facebook, Inc (Adam Paszke) -Copyright (c) 2014- Facebook, Inc (Soumith Chintala) -Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert) -Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu) -Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu) -Copyright (c) 2011-2013 NYU (Clement Farabet) -Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, -Iain Melvin, Jason Weston) Copyright (c) 2006 Idiap Research Institute -(Samy Bengio) Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, -Samy Bengio, Johnny Mariethoz) - -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - -3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories -America and IDIAP Research Institute nor the names of its contributors may be - used to endorse or promote products derived from this software without - specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. -*/ - -#ifndef SIMDJSON_ISADETECTION_H -#define SIMDJSON_ISADETECTION_H - -#include -#include -#if defined(_MSC_VER) -#include -#elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID) -#include -#endif - -namespace simdjson { -// Can be found on Intel ISA Reference for CPUID -constexpr uint32_t cpuid_avx2_bit = 1 << 5; // Bit 5 of EBX for EAX=0x7 -constexpr uint32_t cpuid_bmi1_bit = 1 << 3; // bit 3 of EBX for EAX=0x7 -constexpr uint32_t cpuid_bmi2_bit = 1 << 8; // bit 8 of EBX for EAX=0x7 -constexpr uint32_t cpuid_sse42_bit = 1 << 20; // bit 20 of ECX for EAX=0x1 -constexpr uint32_t cpuid_pclmulqdq_bit = 1 << 1; // bit 1 of ECX for EAX=0x1 - -enum instruction_set { - DEFAULT = 0x0, - NEON = 0x1, - AVX2 = 0x4, - SSE42 = 0x8, - PCLMULQDQ = 0x10, - BMI1 = 0x20, - BMI2 = 0x40 -}; - -#if defined(__arm__) || defined(__aarch64__) // incl. armel, armhf, arm64 - -#if defined(__ARM_NEON) - -static inline uint32_t detect_supported_architectures() { - return instruction_set::NEON; -} - -#else // ARM without NEON - -static inline uint32_t detect_supported_architectures() { - return instruction_set::DEFAULT; -} - -#endif - -#else // x86 -static inline void cpuid(uint32_t *eax, uint32_t *ebx, uint32_t *ecx, - uint32_t *edx) { -#if defined(_MSC_VER) - int cpu_info[4]; - __cpuid(cpu_info, *eax); - *eax = cpu_info[0]; - *ebx = cpu_info[1]; - *ecx = cpu_info[2]; - *edx = cpu_info[3]; -#elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID) - uint32_t level = *eax; - __get_cpuid(level, eax, ebx, ecx, edx); -#else - uint32_t a = *eax, b, c = *ecx, d; - asm volatile("cpuid\n\t" : "+a"(a), "=b"(b), "+c"(c), "=d"(d)); - *eax = a; - *ebx = b; - *ecx = c; - *edx = d; -#endif -} - -static inline uint32_t detect_supported_architectures() { - uint32_t eax, ebx, ecx, edx; - uint32_t host_isa = 0x0; - - // ECX for EAX=0x7 - eax = 0x7; - ecx = 0x0; - cpuid(&eax, &ebx, &ecx, &edx); -#ifndef SIMDJSON_DISABLE_AVX2_DETECTION - if (ebx & cpuid_avx2_bit) { - host_isa |= instruction_set::AVX2; - } -#endif - if (ebx & cpuid_bmi1_bit) { - host_isa |= instruction_set::BMI1; - } - - if (ebx & cpuid_bmi2_bit) { - host_isa |= instruction_set::BMI2; - } - - // EBX for EAX=0x1 - eax = 0x1; - cpuid(&eax, &ebx, &ecx, &edx); - - if (ecx & cpuid_sse42_bit) { - host_isa |= instruction_set::SSE42; - } - - if (ecx & cpuid_pclmulqdq_bit) { - host_isa |= instruction_set::PCLMULQDQ; - } - - return host_isa; -} - -#endif // end SIMD extension detection code -} // namespace simdjson -#endif -/* end file include/simdjson/isadetection.h */ -/* begin file include/simdjson/jsonformatutils.h */ -#ifndef SIMDJSON_JSONFORMATUTILS_H -#define SIMDJSON_JSONFORMATUTILS_H - -#include -#include - -namespace simdjson { - - -// ends with zero char -static inline void print_with_escapes(const unsigned char *src, - std::ostream &os) { - while (*src) { - switch (*src) { - case '\b': - os << '\\'; - os << 'b'; - break; - case '\f': - os << '\\'; - os << 'f'; - break; - case '\n': - os << '\\'; - os << 'n'; - break; - case '\r': - os << '\\'; - os << 'r'; - break; - case '\"': - os << '\\'; - os << '"'; - break; - case '\t': - os << '\\'; - os << 't'; - break; - case '\\': - os << '\\'; - os << '\\'; - break; - default: - if (*src <= 0x1F) { - std::ios::fmtflags f(os.flags()); - os << std::hex << std::setw(4) << std::setfill('0') - << static_cast(*src); - os.flags(f); - } else { - os << *src; - } - } - src++; - } -} - - -// print len chars -static inline void print_with_escapes(const unsigned char *src, - std::ostream &os, size_t len) { - const unsigned char *finalsrc = src + len; - while (src < finalsrc) { - switch (*src) { - case '\b': - os << '\\'; - os << 'b'; - break; - case '\f': - os << '\\'; - os << 'f'; - break; - case '\n': - os << '\\'; - os << 'n'; - break; - case '\r': - os << '\\'; - os << 'r'; - break; - case '\"': - os << '\\'; - os << '"'; - break; - case '\t': - os << '\\'; - os << 't'; - break; - case '\\': - os << '\\'; - os << '\\'; - break; - default: - if (*src <= 0x1F) { - std::ios::fmtflags f(os.flags()); - os << std::hex << std::setw(4) << std::setfill('0') - << static_cast(*src); - os.flags(f); - } else { - os << *src; - } - } - src++; - } -} - -static inline void print_with_escapes(const char *src, std::ostream &os) { - print_with_escapes(reinterpret_cast(src), os); -} - -static inline void print_with_escapes(const char *src, std::ostream &os, - size_t len) { - print_with_escapes(reinterpret_cast(src), os, len); -} -} // namespace simdjson - -#endif -/* end file include/simdjson/jsonformatutils.h */ -/* begin file include/simdjson/simdjson.h */ -#ifndef SIMDJSON_SIMDJSON_H -#define SIMDJSON_SIMDJSON_H - -#ifndef __cplusplus -#error simdjson requires a C++ compiler -#endif - -#ifndef SIMDJSON_CPLUSPLUS -#if defined(_MSVC_LANG) && !defined(__clang__) -#define SIMDJSON_CPLUSPLUS (_MSC_VER == 1900 ? 201103L : _MSVC_LANG) -#else -#define SIMDJSON_CPLUSPLUS __cplusplus -#endif -#endif - -#if (SIMDJSON_CPLUSPLUS < 201703L) -#error simdjson requires a compiler compliant with the C++17 standard -#endif - -/* begin file include/simdjson/error.h */ -#ifndef SIMDJSON_ERROR_H -#define SIMDJSON_ERROR_H - -#include - -namespace simdjson { - -enum error_code { - SUCCESS = 0, - SUCCESS_AND_HAS_MORE, //No errors and buffer still has more data - CAPACITY, // This parser can't support a document that big - MEMALLOC, // Error allocating memory, most likely out of memory - TAPE_ERROR, // Something went wrong while writing to the tape (stage 2), this - // is a generic error - DEPTH_ERROR, // Your document exceeds the user-specified depth limitation - STRING_ERROR, // Problem while parsing a string - T_ATOM_ERROR, // Problem while parsing an atom starting with the letter 't' - F_ATOM_ERROR, // Problem while parsing an atom starting with the letter 'f' - N_ATOM_ERROR, // Problem while parsing an atom starting with the letter 'n' - NUMBER_ERROR, // Problem while parsing a number - UTF8_ERROR, // the input is not valid UTF-8 - UNINITIALIZED, // unknown error, or uninitialized document - EMPTY, // no structural element found - UNESCAPED_CHARS, // found unescaped characters in a string. - UNCLOSED_STRING, // missing quote at the end - UNSUPPORTED_ARCHITECTURE, // unsupported architecture - UNEXPECTED_ERROR // indicative of a bug in simdjson -}; - -const std::string &error_message(error_code error) noexcept; - -struct invalid_json : public std::exception { - invalid_json(error_code _error) : error{_error} {} - const char *what() const noexcept { return error_message(error).c_str(); } - error_code error; -}; - -// TODO these are deprecated, remove -using ErrorValues = error_code; -inline const std::string &error_message(int error) noexcept { return error_message(error_code(error)); } - -} // namespace simdjson - -#endif // SIMDJSON_ERROR_H -/* end file include/simdjson/error.h */ - -#endif // SIMDJSON_H -/* end file include/simdjson/error.h */ /* begin file include/simdjson/common_defs.h */ #ifndef SIMDJSON_COMMON_DEFS_H #define SIMDJSON_COMMON_DEFS_H @@ -549,9 +285,6 @@ inline const std::string &error_message(int error) noexcept { return error_messa #endif // SIMDJSON_COMMON_DEFS_H /* end file include/simdjson/common_defs.h */ -/* begin file include/simdjson/padded_string.h */ -#ifndef SIMDJSON_PADDING_STRING_H -#define SIMDJSON_PADDING_STRING_H #include #include @@ -662,71 +395,9 @@ private: } // namespace simdjson #endif -/* end file include/simdjson/padded_string.h */ -/* begin file include/simdjson/jsonioutil.h */ -#ifndef SIMDJSON_JSONIOUTIL_H -#define SIMDJSON_JSONIOUTIL_H - -#include -#include -#include -#include -#include -#include - - -namespace simdjson { - -// load a file in memory... -// get a corpus; pad out to cache line so we can always use SIMD -// throws exceptions in case of failure -// first element of the pair is a string (null terminated) -// whereas the second element is the length. -// caller is responsible to free (aligned_free((void*)result.data()))) -// -// throws an exception if the file cannot be opened, use try/catch -// try { -// p = get_corpus(filename); -// } catch (const std::exception& e) { -// aligned_free((void*)p.data()); -// std::cout << "Could not load the file " << filename << std::endl; -// } -padded_string get_corpus(const std::string &filename); -} // namespace simdjson - -#endif -/* end file include/simdjson/jsonioutil.h */ -/* begin file include/simdjson/jsonminifier.h */ -#ifndef SIMDJSON_JSONMINIFIER_H -#define SIMDJSON_JSONMINIFIER_H - -#include -#include -#include - -namespace simdjson { - -// Take input from buf and remove useless whitespace, write it to out; buf and -// out can be the same pointer. Result is null terminated, -// return the string length (minus the null termination). -// The accelerated version of this function only runs on AVX2 hardware. -size_t json_minify(const uint8_t *buf, size_t len, uint8_t *out); - -static inline size_t json_minify(const char *buf, size_t len, char *out) { - return json_minify(reinterpret_cast(buf), len, - reinterpret_cast(out)); -} - -static inline size_t json_minify(const std::string_view &p, char *out) { - return json_minify(p.data(), p.size(), out); -} - -static inline size_t json_minify(const padded_string &p, char *out) { - return json_minify(p.data(), p.size(), out); -} -} // namespace simdjson -#endif -/* end file include/simdjson/jsonminifier.h */ +/* end file include/simdjson/common_defs.h */ +/* begin file include/simdjson/implementation.h */ +// Declaration order requires we get to document.h before implementation.h no matter what /* begin file include/simdjson/document.h */ #ifndef SIMDJSON_DOCUMENT_H #define SIMDJSON_DOCUMENT_H @@ -734,6 +405,17 @@ static inline size_t json_minify(const padded_string &p, char *out) { #include #include #include +/* begin file include/simdjson/simdjson.h */ +/** + * @file + * @deprecated We'll be removing this file so it isn't confused with the top level simdjson.h + */ +#ifndef SIMDJSON_SIMDJSON_H +#define SIMDJSON_SIMDJSON_H + + +#endif // SIMDJSON_H +/* end file include/simdjson/simdjson.h */ #define JSON_VALUE_MASK 0xFFFFFFFFFFFFFF #define DEFAULT_MAX_DEPTH 1024 // a JSON document with a depth exceeding 1024 is probably de facto invalid @@ -1049,20 +731,8 @@ private: } // namespace simdjson -/* begin file include/simdjson/inline/document.h */ -#ifndef SIMDJSON_INLINE_DOCUMENT_H -#define SIMDJSON_INLINE_DOCUMENT_H - -#ifndef SIMDJSON_DOCUMENT_H -#error This is an internal file only. Include document.h instead. -#endif - -// Inline implementations go in here if they aren't small enough to go in the class itself or if -// there are complex header file dependencies that need to be broken by externalizing the -// implementation. - -/* begin file include/simdjson/implementation.h */ -// Declaration order requires we get to document.h before implementation.h no matter what +#endif // SIMDJSON_DOCUMENT_H +/* end file include/simdjson/simdjson.h */ #ifndef SIMDJSON_IMPLEMENTATION_H #define SIMDJSON_IMPLEMENTATION_H @@ -1299,188 +969,919 @@ inline internal::atomic_ptr active_implementation = &inter } // namespace simdjson #endif // SIMDJSON_IMPLEMENTATION_H -/* end file include/simdjson/implementation.h */ +/* end file include/simdjson/simdjson.h */ +/* begin file include/simdjson/jsonstream.h */ +#ifndef SIMDJSON_JSONSTREAM_H +#define SIMDJSON_JSONSTREAM_H + +#include +#include +#include +#include +/* begin file include/simdjson/isadetection.h */ +/* From +https://github.com/endorno/pytorch/blob/master/torch/lib/TH/generic/simd/simd.h +Highly modified. + +Copyright (c) 2016- Facebook, Inc (Adam Paszke) +Copyright (c) 2014- Facebook, Inc (Soumith Chintala) +Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert) +Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu) +Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu) +Copyright (c) 2011-2013 NYU (Clement Farabet) +Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, +Iain Melvin, Jason Weston) Copyright (c) 2006 Idiap Research Institute +(Samy Bengio) Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, +Samy Bengio, Johnny Mariethoz) + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories +America and IDIAP Research Institute nor the names of its contributors may be + used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef SIMDJSON_ISADETECTION_H +#define SIMDJSON_ISADETECTION_H + +#include +#include +#if defined(_MSC_VER) +#include +#elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID) +#include +#endif + +namespace simdjson { +// Can be found on Intel ISA Reference for CPUID +constexpr uint32_t cpuid_avx2_bit = 1 << 5; // Bit 5 of EBX for EAX=0x7 +constexpr uint32_t cpuid_bmi1_bit = 1 << 3; // bit 3 of EBX for EAX=0x7 +constexpr uint32_t cpuid_bmi2_bit = 1 << 8; // bit 8 of EBX for EAX=0x7 +constexpr uint32_t cpuid_sse42_bit = 1 << 20; // bit 20 of ECX for EAX=0x1 +constexpr uint32_t cpuid_pclmulqdq_bit = 1 << 1; // bit 1 of ECX for EAX=0x1 + +enum instruction_set { + DEFAULT = 0x0, + NEON = 0x1, + AVX2 = 0x4, + SSE42 = 0x8, + PCLMULQDQ = 0x10, + BMI1 = 0x20, + BMI2 = 0x40 +}; + +#if defined(__arm__) || defined(__aarch64__) // incl. armel, armhf, arm64 + +#if defined(__ARM_NEON) + +static inline uint32_t detect_supported_architectures() { + return instruction_set::NEON; +} + +#else // ARM without NEON + +static inline uint32_t detect_supported_architectures() { + return instruction_set::DEFAULT; +} + +#endif + +#else // x86 +static inline void cpuid(uint32_t *eax, uint32_t *ebx, uint32_t *ecx, + uint32_t *edx) { +#if defined(_MSC_VER) + int cpu_info[4]; + __cpuid(cpu_info, *eax); + *eax = cpu_info[0]; + *ebx = cpu_info[1]; + *ecx = cpu_info[2]; + *edx = cpu_info[3]; +#elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID) + uint32_t level = *eax; + __get_cpuid(level, eax, ebx, ecx, edx); +#else + uint32_t a = *eax, b, c = *ecx, d; + asm volatile("cpuid\n\t" : "+a"(a), "=b"(b), "+c"(c), "=d"(d)); + *eax = a; + *ebx = b; + *ecx = c; + *edx = d; +#endif +} + +static inline uint32_t detect_supported_architectures() { + uint32_t eax, ebx, ecx, edx; + uint32_t host_isa = 0x0; + + // ECX for EAX=0x7 + eax = 0x7; + ecx = 0x0; + cpuid(&eax, &ebx, &ecx, &edx); +#ifndef SIMDJSON_DISABLE_AVX2_DETECTION + if (ebx & cpuid_avx2_bit) { + host_isa |= instruction_set::AVX2; + } +#endif + if (ebx & cpuid_bmi1_bit) { + host_isa |= instruction_set::BMI1; + } + + if (ebx & cpuid_bmi2_bit) { + host_isa |= instruction_set::BMI2; + } + + // EBX for EAX=0x1 + eax = 0x1; + cpuid(&eax, &ebx, &ecx, &edx); + + if (ecx & cpuid_sse42_bit) { + host_isa |= instruction_set::SSE42; + } + + if (ecx & cpuid_pclmulqdq_bit) { + host_isa |= instruction_set::PCLMULQDQ; + } + + return host_isa; +} + +#endif // end SIMD extension detection code +} // namespace simdjson +#endif +/* end file include/simdjson/isadetection.h */ +/* begin file src/jsoncharutils.h */ +#ifndef SIMDJSON_JSONCHARUTILS_H +#define SIMDJSON_JSONCHARUTILS_H + +/* begin file include/simdjson/parsedjson.h */ +// TODO Remove this -- deprecated API and files + +#ifndef SIMDJSON_PARSEDJSON_H +#define SIMDJSON_PARSEDJSON_H + namespace simdjson { -// TODO inline? -document::doc_ref_result document::parser::parse(const uint8_t *buf, size_t len, bool realloc_if_needed) noexcept { - error_code code = init_parse(len); - if (code) { return document::doc_ref_result(doc, code); } - - if (realloc_if_needed) { - const uint8_t *tmp_buf = buf; - buf = (uint8_t *)allocate_padded_buffer(len); - if (buf == nullptr) - return document::doc_ref_result(doc, MEMALLOC); - memcpy((void *)buf, tmp_buf, len); - } - - code = simdjson::active_implementation->parse(buf, len, *this); - - // We're indicating validity via the doc_ref_result, so set the parse state back to invalid - valid = false; - error = UNINITIALIZED; - if (realloc_if_needed) { - aligned_free((void *)buf); // must free before we exit - } - return document::doc_ref_result(doc, code); -} -really_inline document::doc_ref_result document::parser::parse(const char *buf, size_t len, bool realloc_if_needed) noexcept { - return parse((const uint8_t *)buf, len, realloc_if_needed); -} -really_inline document::doc_ref_result document::parser::parse(const std::string &s, bool realloc_if_needed) noexcept { - return parse(s.data(), s.length(), realloc_if_needed); -} -really_inline document::doc_ref_result document::parser::parse(const padded_string &s) noexcept { - return parse(s.data(), s.length(), false); -} - -// TODO really_inline? -inline document::doc_result document::parse(const uint8_t *buf, size_t len, bool realloc_if_needed) noexcept { - document::parser parser; - if (!parser.allocate_capacity(len)) { - return MEMALLOC; - } - auto [doc, error] = parser.parse(buf, len, realloc_if_needed); - return document::doc_result((document &&)doc, error); -} -really_inline document::doc_result document::parse(const char *buf, size_t len, bool realloc_if_needed) noexcept { - return parse((const uint8_t *)buf, len, realloc_if_needed); -} -really_inline document::doc_result document::parse(const std::string &s, bool realloc_if_needed) noexcept { - return parse(s.data(), s.length(), realloc_if_needed); -} -really_inline document::doc_result document::parse(const padded_string &s) noexcept { - return parse(s.data(), s.length(), false); -} - -// -// Parser callbacks -// - -WARN_UNUSED -inline error_code document::parser::init_parse(size_t len) { - if (len > capacity()) { - return error = CAPACITY; - } - // If the last doc was taken, we need to allocate a new one - if (!doc.tape) { - if (!doc.set_capacity(len)) { - return error = MEMALLOC; - } - } - return SUCCESS; -} - -inline void document::parser::init_stage2() { - current_string_buf_loc = doc.string_buf.get(); - current_loc = 0; - valid = false; - error = UNINITIALIZED; -} - -really_inline error_code document::parser::on_error(error_code new_error_code) { - error = new_error_code; - return new_error_code; -} -really_inline error_code document::parser::on_success(error_code success_code) { - error = success_code; - valid = true; - return success_code; -} -really_inline bool document::parser::on_start_document(uint32_t depth) { - containing_scope_offset[depth] = current_loc; - write_tape(0, 'r'); - return true; -} -really_inline bool document::parser::on_start_object(uint32_t depth) { - containing_scope_offset[depth] = current_loc; - write_tape(0, '{'); - return true; -} -really_inline bool document::parser::on_start_array(uint32_t depth) { - containing_scope_offset[depth] = current_loc; - write_tape(0, '['); - return true; -} -// TODO we're not checking this bool -really_inline bool document::parser::on_end_document(uint32_t depth) { - // write our doc.tape location to the header scope - // The root scope gets written *at* the previous location. - annotate_previous_loc(containing_scope_offset[depth], current_loc); - write_tape(containing_scope_offset[depth], 'r'); - return true; -} -really_inline bool document::parser::on_end_object(uint32_t depth) { - // write our doc.tape location to the header scope - write_tape(containing_scope_offset[depth], '}'); - annotate_previous_loc(containing_scope_offset[depth], current_loc); - return true; -} -really_inline bool document::parser::on_end_array(uint32_t depth) { - // write our doc.tape location to the header scope - write_tape(containing_scope_offset[depth], ']'); - annotate_previous_loc(containing_scope_offset[depth], current_loc); - return true; -} - -really_inline bool document::parser::on_true_atom() { - write_tape(0, 't'); - return true; -} -really_inline bool document::parser::on_false_atom() { - write_tape(0, 'f'); - return true; -} -really_inline bool document::parser::on_null_atom() { - write_tape(0, 'n'); - return true; -} - -really_inline uint8_t *document::parser::on_start_string() { - /* we advance the point, accounting for the fact that we have a NULL - * termination */ - write_tape(current_string_buf_loc - doc.string_buf.get(), '"'); - return current_string_buf_loc + sizeof(uint32_t); -} - -really_inline bool document::parser::on_end_string(uint8_t *dst) { - uint32_t str_length = dst - (current_string_buf_loc + sizeof(uint32_t)); - // TODO check for overflow in case someone has a crazy string (>=4GB?) - // But only add the overflow check when the document itself exceeds 4GB - // Currently unneeded because we refuse to parse docs larger or equal to 4GB. - memcpy(current_string_buf_loc, &str_length, sizeof(uint32_t)); - // NULL termination is still handy if you expect all your strings to - // be NULL terminated? It comes at a small cost - *dst = 0; - current_string_buf_loc = dst + 1; - return true; -} - -really_inline bool document::parser::on_number_s64(int64_t value) { - write_tape(0, 'l'); - std::memcpy(&doc.tape[current_loc], &value, sizeof(value)); - ++current_loc; - return true; -} -really_inline bool document::parser::on_number_u64(uint64_t value) { - write_tape(0, 'u'); - doc.tape[current_loc++] = value; - return true; -} -really_inline bool document::parser::on_number_double(double value) { - write_tape(0, 'd'); - static_assert(sizeof(value) == sizeof(doc.tape[current_loc]), "mismatch size"); - memcpy(&doc.tape[current_loc++], &value, sizeof(double)); - // doc.tape[doc.current_loc++] = *((uint64_t *)&d); - return true; -} +using ParsedJson = document::parser; } // namespace simdjson +#endif +/* end file include/simdjson/parsedjson.h */ + +namespace simdjson { +// structural chars here are +// they are { 0x7b } 0x7d : 0x3a [ 0x5b ] 0x5d , 0x2c (and NULL) +// we are also interested in the four whitespace characters +// space 0x20, linefeed 0x0a, horizontal tab 0x09 and carriage return 0x0d + +// these are the chars that can follow a true/false/null or number atom +// and nothing else +const uint32_t structural_or_whitespace_or_null_negated[256] = { + 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, + + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, + + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; + +// return non-zero if not a structural or whitespace char +// zero otherwise +really_inline uint32_t is_not_structural_or_whitespace_or_null(uint8_t c) { + return structural_or_whitespace_or_null_negated[c]; +} + +const uint32_t structural_or_whitespace_negated[256] = { + 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, + + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, + + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; + +// return non-zero if not a structural or whitespace char +// zero otherwise +really_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { + return structural_or_whitespace_negated[c]; +} + +const uint32_t structural_or_whitespace_or_null[256] = { + 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + +really_inline uint32_t is_structural_or_whitespace_or_null(uint8_t c) { + return structural_or_whitespace_or_null[c]; +} + +const uint32_t structural_or_whitespace[256] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + +really_inline uint32_t is_structural_or_whitespace(uint8_t c) { + return structural_or_whitespace[c]; +} + +const uint32_t digit_to_val32[886] = { + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, + 0x6, 0x7, 0x8, 0x9, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xa, + 0xb, 0xc, 0xd, 0xe, 0xf, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xa, 0xb, 0xc, 0xd, 0xe, + 0xf, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0x0, 0x10, 0x20, 0x30, 0x40, 0x50, + 0x60, 0x70, 0x80, 0x90, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xa0, + 0xb0, 0xc0, 0xd0, 0xe0, 0xf0, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, + 0xf0, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0x0, 0x100, 0x200, 0x300, 0x400, 0x500, + 0x600, 0x700, 0x800, 0x900, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xa00, + 0xb00, 0xc00, 0xd00, 0xe00, 0xf00, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xa00, 0xb00, 0xc00, 0xd00, 0xe00, + 0xf00, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0x0, 0x1000, 0x2000, 0x3000, 0x4000, 0x5000, + 0x6000, 0x7000, 0x8000, 0x9000, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xa000, + 0xb000, 0xc000, 0xd000, 0xe000, 0xf000, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xa000, 0xb000, 0xc000, 0xd000, 0xe000, + 0xf000, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}; +// returns a value with the high 16 bits set if not valid +// otherwise returns the conversion of the 4 hex digits at src into the bottom +// 16 bits of the 32-bit return register +// +// see +// https://lemire.me/blog/2019/04/17/parsing-short-hexadecimal-strings-efficiently/ +static inline uint32_t hex_to_u32_nocheck( + const uint8_t *src) { // strictly speaking, static inline is a C-ism + uint32_t v1 = digit_to_val32[630 + src[0]]; + uint32_t v2 = digit_to_val32[420 + src[1]]; + uint32_t v3 = digit_to_val32[210 + src[2]]; + uint32_t v4 = digit_to_val32[0 + src[3]]; + return v1 | v2 | v3 | v4; +} + +// returns true if the provided byte value is a +// "continuing" UTF-8 value, that is, if it starts with +// 0b10... +static inline bool is_utf8_continuing(char c) { + // in 2 complement's notation, values start at 0b10000 (-128)... and + // go up to 0b11111 (-1)... so we want all values from -128 to -65 (which is 0b10111111) + return ((signed char)c) <= -65; +} +// returns true if the provided byte value is an ASCII character +static inline bool is_ascii(char c) { + return ((unsigned char)c) <= 127; +} + +// if the string ends with UTF-8 values, backtrack +// up to the first ASCII character. May return 0. +static inline size_t trimmed_length_safe_utf8(const char * c, size_t len) { + while ((len > 0) and (not is_ascii(c[len - 1]))) { + len--; + } + return len; +} + + + +// given a code point cp, writes to c +// the utf-8 code, outputting the length in +// bytes, if the length is zero, the code point +// is invalid +// +// This can possibly be made faster using pdep +// and clz and table lookups, but JSON documents +// have few escaped code points, and the following +// function looks cheap. +// +// Note: we assume that surrogates are treated separately +// +inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { + if (cp <= 0x7F) { + c[0] = cp; + return 1; // ascii + } + if (cp <= 0x7FF) { + c[0] = (cp >> 6) + 192; + c[1] = (cp & 63) + 128; + return 2; // universal plane + // Surrogates are treated elsewhere... + //} //else if (0xd800 <= cp && cp <= 0xdfff) { + // return 0; // surrogates // could put assert here + } else if (cp <= 0xFFFF) { + c[0] = (cp >> 12) + 224; + c[1] = ((cp >> 6) & 63) + 128; + c[2] = (cp & 63) + 128; + return 3; + } else if (cp <= 0x10FFFF) { // if you know you have a valid code point, this + // is not needed + c[0] = (cp >> 18) + 240; + c[1] = ((cp >> 12) & 63) + 128; + c[2] = ((cp >> 6) & 63) + 128; + c[3] = (cp & 63) + 128; + return 4; + } + // will return 0 when the code point was too large. + return 0; // bad r +} +} // namespace simdjson + +#endif +/* end file include/simdjson/parsedjson.h */ + + +namespace simdjson { +/************************************************************************************* + * The main motivation for this piece of software is to achieve maximum speed + *and offer + * good quality of life while parsing files containing multiple JSON documents. + * + * Since we want to offer flexibility and not restrict ourselves to a specific + *file + * format, we support any file that contains any valid JSON documents separated + *by one + * or more character that is considered a whitespace by the JSON spec. + * Namely: space, nothing, linefeed, carriage return, horizontal tab. + * Anything that is not whitespace will be parsed as a JSON document and could + *lead + * to failure. + * + * To offer maximum parsing speed, our implementation processes the data inside + *the + * buffer by batches and their size is defined by the parameter "batch_size". + * By loading data in batches, we can optimize the time spent allocating data in + *the + * parser and can also open the possibility of multi-threading. + * The batch_size must be at least as large as the biggest document in the file, + *but + * not too large in order to submerge the chached memory. We found that 1MB is + * somewhat a sweet spot for now. Eventually, this batch_size could be fully + * automated and be optimal at all times. + ************************************************************************************/ +/** +* The template parameter (string_container) must +* support the data() and size() methods, returning a pointer +* to a char* and to the number of bytes respectively. +* The simdjson parser may read up to SIMDJSON_PADDING bytes beyond the end +* of the string, so if you do not use a padded_string container, +* you have the responsability to overallocated. If you fail to +* do so, your software may crash if you cross a page boundary, +* and you should expect memory checkers to object. +* Most users should use a simdjson::padded_string. +*/ +template class JsonStream { +public: + /* Create a JsonStream object that can be used to parse sequentially the valid + * JSON documents found in the buffer "buf". + * + * The batch_size must be at least as large as the biggest document in the + * file, but + * not too large to submerge the cached memory. We found that 1MB is + * somewhat a sweet spot for now. + * + * The user is expected to call the following json_parse method to parse the + * next + * valid JSON document found in the buffer. This method can and is expected + * to be + * called in a loop. + * + * Various methods are offered to keep track of the status, like + * get_current_buffer_loc, + * get_n_parsed_docs, get_n_bytes_parsed, etc. + * + * */ + JsonStream(const string_container &s, size_t batch_size = 1000000); + + ~JsonStream(); + + /* Parse the next document found in the buffer previously given to JsonStream. + + * The content should be a valid JSON document encoded as UTF-8. If there is a + * UTF-8 BOM, the caller is responsible for omitting it, UTF-8 BOM are + * discouraged. + * + * You do NOT need to pre-allocate a parser. This function takes care of + * pre-allocating a capacity defined by the batch_size defined when creating + the + * JsonStream object. + * + * The function returns simdjson::SUCCESS_AND_HAS_MORE (an integer = 1) in + case + * of success and indicates that the buffer still contains more data to be + parsed, + * meaning this function can be called again to return the next JSON document + * after this one. + * + * The function returns simdjson::SUCCESS (as integer = 0) in case of success + * and indicates that the buffer has successfully been parsed to the end. + * Every document it contained has been parsed without error. + * + * The function returns an error code from simdjson/simdjson.h in case of + failure + * such as simdjson::CAPACITY, simdjson::MEMALLOC, simdjson::DEPTH_ERROR and + so forth; + * the simdjson::error_message function converts these error codes into a + * string). + * + * You can also check validity by calling parser.is_valid(). The same parser + can + * and should be reused for the other documents in the buffer. */ + int json_parse(document::parser &parser); + + /* Returns the location (index) of where the next document should be in the + * buffer. + * Can be used for debugging, it tells the user the position of the end of the + * last + * valid JSON document parsed*/ + inline size_t get_current_buffer_loc() const { return current_buffer_loc; } + + /* Returns the total amount of complete documents parsed by the JsonStream, + * in the current buffer, at the given time.*/ + inline size_t get_n_parsed_docs() const { return n_parsed_docs; } + + /* Returns the total amount of data (in bytes) parsed by the JsonStream, + * in the current buffer, at the given time.*/ + inline size_t get_n_bytes_parsed() const { return n_bytes_parsed; } + +private: + inline const uint8_t *buf() const { return reinterpret_cast(str.data()) + str_start; } + + inline void advance(size_t offset) { str_start += offset; } + + inline size_t remaining() const { return str.size() - str_start; } + + const string_container &str; + size_t _batch_size; // this is actually variable! + size_t str_start{0}; + size_t next_json{0}; + bool load_next_batch{true}; + size_t current_buffer_loc{0}; +#ifdef SIMDJSON_THREADS_ENABLED + size_t last_json_buffer_loc{0}; +#endif + size_t n_parsed_docs{0}; + size_t n_bytes_parsed{0}; + simdjson::implementation *stage_parser; +#ifdef SIMDJSON_THREADS_ENABLED + error_code stage1_is_ok_thread{SUCCESS}; + std::thread stage_1_thread; + document::parser parser_thread; +#endif +}; // end of class JsonStream + +/* This algorithm is used to quickly identify the buffer position of + * the last JSON document inside the current batch. + * + * It does its work by finding the last pair of structural characters + * that represent the end followed by the start of a document. + * + * Simply put, we iterate over the structural characters, starting from + * the end. We consider that we found the end of a JSON document when the + * first element of the pair is NOT one of these characters: '{' '[' ';' ',' + * and when the second element is NOT one of these characters: '}' '}' ';' ','. + * + * This simple comparison works most of the time, but it does not cover cases + * where the batch's structural indexes contain a perfect amount of documents. + * In such a case, we do not have access to the structural index which follows + * the last document, therefore, we do not have access to the second element in + * the pair, and means that we cannot identify the last document. To fix this + * issue, we keep a count of the open and closed curly/square braces we found + * while searching for the pair. When we find a pair AND the count of open and + * closed curly/square braces is the same, we know that we just passed a + * complete + * document, therefore the last json buffer location is the end of the batch + * */ +inline size_t find_last_json_buf_idx(const uint8_t *buf, size_t size, + const document::parser &parser) { + // this function can be generally useful + if (parser.n_structural_indexes == 0) + return 0; + auto last_i = parser.n_structural_indexes - 1; + if (parser.structural_indexes[last_i] == size) { + if (last_i == 0) + return 0; + last_i = parser.n_structural_indexes - 2; + } + auto arr_cnt = 0; + auto obj_cnt = 0; + for (auto i = last_i; i > 0; i--) { + auto idxb = parser.structural_indexes[i]; + switch (buf[idxb]) { + case ':': + case ',': + continue; + case '}': + obj_cnt--; + continue; + case ']': + arr_cnt--; + continue; + case '{': + obj_cnt++; + break; + case '[': + arr_cnt++; + break; + } + auto idxa = parser.structural_indexes[i - 1]; + switch (buf[idxa]) { + case '{': + case '[': + case ':': + case ',': + continue; + } + if (!arr_cnt && !obj_cnt) { + return last_i + 1; + } + return i; + } + return 0; +} + +template +JsonStream::JsonStream(const string_container &s, + size_t batchSize) + : str(s), _batch_size(batchSize) { +} + +template JsonStream::~JsonStream() { +#ifdef SIMDJSON_THREADS_ENABLED + if (stage_1_thread.joinable()) { + stage_1_thread.join(); + } +#endif +} + +#ifdef SIMDJSON_THREADS_ENABLED + +// threaded version of json_parse +// todo: simplify this code further +template +int JsonStream::json_parse(document::parser &parser) { + if (unlikely(parser.capacity() == 0)) { + const bool allocok = parser.allocate_capacity(_batch_size); + if (!allocok) { + return parser.error = simdjson::MEMALLOC; + } + } else if (unlikely(parser.capacity() < _batch_size)) { + return parser.error = simdjson::CAPACITY; + } + if (unlikely(parser_thread.capacity() < _batch_size)) { + const bool allocok_thread = parser_thread.allocate_capacity(_batch_size); + if (!allocok_thread) { + return parser.error = simdjson::MEMALLOC; + } + } + if (unlikely(load_next_batch)) { + // First time loading + if (!stage_1_thread.joinable()) { + _batch_size = (std::min)(_batch_size, remaining()); + _batch_size = trimmed_length_safe_utf8((const char *)buf(), _batch_size); + if (_batch_size == 0) { + return parser.error = simdjson::UTF8_ERROR; + } + auto stage1_is_ok = error_code(simdjson::active_implementation->stage1(buf(), _batch_size, parser, true)); + if (stage1_is_ok != simdjson::SUCCESS) { + return parser.error = stage1_is_ok; + } + size_t last_index = find_last_json_buf_idx(buf(), _batch_size, parser); + if (last_index == 0) { + if (parser.n_structural_indexes == 0) { + return parser.error = simdjson::EMPTY; + } + } else { + parser.n_structural_indexes = last_index + 1; + } + } + // the second thread is running or done. + else { + stage_1_thread.join(); + if (stage1_is_ok_thread != simdjson::SUCCESS) { + return parser.error = stage1_is_ok_thread; + } + std::swap(parser.structural_indexes, parser_thread.structural_indexes); + parser.n_structural_indexes = parser_thread.n_structural_indexes; + advance(last_json_buffer_loc); + n_bytes_parsed += last_json_buffer_loc; + } + // let us decide whether we will start a new thread + if (remaining() - _batch_size > 0) { + last_json_buffer_loc = + parser.structural_indexes[find_last_json_buf_idx(buf(), _batch_size, parser)]; + _batch_size = (std::min)(_batch_size, remaining() - last_json_buffer_loc); + if (_batch_size > 0) { + _batch_size = trimmed_length_safe_utf8( + (const char *)(buf() + last_json_buffer_loc), _batch_size); + if (_batch_size == 0) { + return parser.error = simdjson::UTF8_ERROR; + } + // let us capture read-only variables + const uint8_t *const b = buf() + last_json_buffer_loc; + const size_t bs = _batch_size; + // we call the thread on a lambda that will update + // this->stage1_is_ok_thread + // there is only one thread that may write to this value + stage_1_thread = std::thread([this, b, bs] { + this->stage1_is_ok_thread = error_code(simdjson::active_implementation->stage1(b, bs, this->parser_thread, true)); + }); + } + } + next_json = 0; + load_next_batch = false; + } // load_next_batch + int res = simdjson::active_implementation->stage2(buf(), remaining(), parser, next_json); + if (res == simdjson::SUCCESS_AND_HAS_MORE) { + n_parsed_docs++; + current_buffer_loc = parser.structural_indexes[next_json]; + load_next_batch = (current_buffer_loc == last_json_buffer_loc); + } else if (res == simdjson::SUCCESS) { + n_parsed_docs++; + if (remaining() > _batch_size) { + current_buffer_loc = parser.structural_indexes[next_json - 1]; + load_next_batch = true; + res = simdjson::SUCCESS_AND_HAS_MORE; + } + } + return res; +} + +#else // SIMDJSON_THREADS_ENABLED + +// single-threaded version of json_parse +template +int JsonStream::json_parse(document::parser &parser) { + if (unlikely(parser.capacity() == 0)) { + const bool allocok = parser.allocate_capacity(_batch_size); + if (!allocok) { + return parser.on_error(MEMALLOC); + } + } else if (unlikely(parser.capacity() < _batch_size)) { + return parser.on_error(CAPACITY); + } + if (unlikely(load_next_batch)) { + advance(current_buffer_loc); + n_bytes_parsed += current_buffer_loc; + _batch_size = (std::min)(_batch_size, remaining()); + _batch_size = trimmed_length_safe_utf8((const char *)buf(), _batch_size); + auto stage1_is_ok = (error_code)simdjson::active_implementation->stage1(buf(), _batch_size, parser, true); + if (stage1_is_ok != simdjson::SUCCESS) { + return parser.on_error(stage1_is_ok); + } + size_t last_index = find_last_json_buf_idx(buf(), _batch_size, parser); + if (last_index == 0) { + if (parser.n_structural_indexes == 0) { + return parser.on_error(EMPTY); + } + } else { + parser.n_structural_indexes = last_index + 1; + } + load_next_batch = false; + } // load_next_batch + int res = simdjson::active_implementation->stage2(buf(), remaining(), parser, next_json); + if (likely(res == simdjson::SUCCESS_AND_HAS_MORE)) { + n_parsed_docs++; + current_buffer_loc = parser.structural_indexes[next_json]; + } else if (res == simdjson::SUCCESS) { + n_parsed_docs++; + if (remaining() > _batch_size) { + current_buffer_loc = parser.structural_indexes[next_json - 1]; + next_json = 1; + load_next_batch = true; + res = simdjson::SUCCESS_AND_HAS_MORE; + } + } else { + printf("E\n"); + } + return res; +} +#endif // SIMDJSON_THREADS_ENABLED + +} // end of namespace simdjson +#endif // SIMDJSON_JSONSTREAM_H +/* end file include/simdjson/parsedjson.h */ +/* begin file include/simdjson/jsonminifier.h */ +#ifndef SIMDJSON_JSONMINIFIER_H +#define SIMDJSON_JSONMINIFIER_H + +#include +#include +#include + +namespace simdjson { + +// Take input from buf and remove useless whitespace, write it to out; buf and +// out can be the same pointer. Result is null terminated, +// return the string length (minus the null termination). +// The accelerated version of this function only runs on AVX2 hardware. +size_t json_minify(const uint8_t *buf, size_t len, uint8_t *out); + +static inline size_t json_minify(const char *buf, size_t len, char *out) { + return json_minify(reinterpret_cast(buf), len, + reinterpret_cast(out)); +} + +static inline size_t json_minify(const std::string_view &p, char *out) { + return json_minify(p.data(), p.size(), out); +} + +static inline size_t json_minify(const padded_string &p, char *out) { + return json_minify(p.data(), p.size(), out); +} +} // namespace simdjson +#endif +/* end file include/simdjson/jsonminifier.h */ + +// Deprecated API +/* begin file include/simdjson/parsedjsoniterator.h */ +// TODO Remove this -- deprecated API and files + +#ifndef SIMDJSON_PARSEDJSONITERATOR_H +#define SIMDJSON_PARSEDJSONITERATOR_H -#endif // SIMDJSON_INLINE_DOCUMENT_H -/* end file include/simdjson/implementation.h */ /* begin file include/simdjson/document_iterator.h */ #ifndef SIMDJSON_DOCUMENT_ITERATOR_H #define SIMDJSON_DOCUMENT_ITERATOR_H @@ -1492,6 +1893,124 @@ really_inline bool document::parser::on_number_double(double value) { #include #include +/* begin file include/simdjson/jsonformatutils.h */ +#ifndef SIMDJSON_JSONFORMATUTILS_H +#define SIMDJSON_JSONFORMATUTILS_H + +#include +#include + +namespace simdjson { + + +// ends with zero char +static inline void print_with_escapes(const unsigned char *src, + std::ostream &os) { + while (*src) { + switch (*src) { + case '\b': + os << '\\'; + os << 'b'; + break; + case '\f': + os << '\\'; + os << 'f'; + break; + case '\n': + os << '\\'; + os << 'n'; + break; + case '\r': + os << '\\'; + os << 'r'; + break; + case '\"': + os << '\\'; + os << '"'; + break; + case '\t': + os << '\\'; + os << 't'; + break; + case '\\': + os << '\\'; + os << '\\'; + break; + default: + if (*src <= 0x1F) { + std::ios::fmtflags f(os.flags()); + os << std::hex << std::setw(4) << std::setfill('0') + << static_cast(*src); + os.flags(f); + } else { + os << *src; + } + } + src++; + } +} + + +// print len chars +static inline void print_with_escapes(const unsigned char *src, + std::ostream &os, size_t len) { + const unsigned char *finalsrc = src + len; + while (src < finalsrc) { + switch (*src) { + case '\b': + os << '\\'; + os << 'b'; + break; + case '\f': + os << '\\'; + os << 'f'; + break; + case '\n': + os << '\\'; + os << 'n'; + break; + case '\r': + os << '\\'; + os << 'r'; + break; + case '\"': + os << '\\'; + os << '"'; + break; + case '\t': + os << '\\'; + os << 't'; + break; + case '\\': + os << '\\'; + os << '\\'; + break; + default: + if (*src <= 0x1F) { + std::ios::fmtflags f(os.flags()); + os << std::hex << std::setw(4) << std::setfill('0') + << static_cast(*src); + os.flags(f); + } else { + os << *src; + } + } + src++; + } +} + +static inline void print_with_escapes(const char *src, std::ostream &os) { + print_with_escapes(reinterpret_cast(src), os); +} + +static inline void print_with_escapes(const char *src, std::ostream &os, + size_t len) { + print_with_escapes(reinterpret_cast(src), os, len); +} +} // namespace simdjson + +#endif +/* end file include/simdjson/jsonformatutils.h */ namespace simdjson { @@ -1743,6 +2262,300 @@ public: } // namespace simdjson +#endif // SIMDJSON_DOCUMENT_ITERATOR_H +/* end file include/simdjson/jsonformatutils.h */ + +#endif +/* end file include/simdjson/jsonformatutils.h */ +/* begin file include/simdjson/jsonparser.h */ +// TODO Remove this -- deprecated API and files + +#ifndef SIMDJSON_JSONPARSER_H +#define SIMDJSON_JSONPARSER_H + +/* begin file include/simdjson/jsonioutil.h */ +#ifndef SIMDJSON_JSONIOUTIL_H +#define SIMDJSON_JSONIOUTIL_H + +#include +#include +#include +#include +#include +#include + + +namespace simdjson { + +// load a file in memory... +// get a corpus; pad out to cache line so we can always use SIMD +// throws exceptions in case of failure +// first element of the pair is a string (null terminated) +// whereas the second element is the length. +// caller is responsible to free (aligned_free((void*)result.data()))) +// +// throws an exception if the file cannot be opened, use try/catch +// try { +// p = get_corpus(filename); +// } catch (const std::exception& e) { +// aligned_free((void*)p.data()); +// std::cout << "Could not load the file " << filename << std::endl; +// } +padded_string get_corpus(const std::string &filename); +} // namespace simdjson + +#endif +/* end file include/simdjson/jsonioutil.h */ + +namespace simdjson { + +// +// C API (json_parse and build_parsed_json) declarations +// + +inline int json_parse(const uint8_t *buf, size_t len, document::parser &parser, bool realloc_if_needed = true) noexcept { + error_code code = parser.parse(buf, len, realloc_if_needed).error; + // The deprecated json_parse API is a signal that the user plans to *use* the error code / valid + // bits in the parser instead of heeding the result code. The normal parser unsets those in + // anticipation of making the error code ephemeral. + // Here we put the code back into the parser, until we've removed this method. + parser.valid = code == SUCCESS; + parser.error = code; + return code; +} +inline int json_parse(const char *buf, size_t len, document::parser &parser, bool realloc_if_needed = true) noexcept { + return json_parse(reinterpret_cast(buf), len, parser, realloc_if_needed); +} +inline int json_parse(const std::string &s, document::parser &parser, bool realloc_if_needed = true) noexcept { + return json_parse(s.data(), s.length(), parser, realloc_if_needed); +} +inline int json_parse(const padded_string &s, document::parser &parser) noexcept { + return json_parse(s.data(), s.length(), parser, false); +} + +WARN_UNUSED static document::parser build_parsed_json(const uint8_t *buf, size_t len, bool realloc_if_needed = true) noexcept { + document::parser parser; + if (!parser.allocate_capacity(len)) { + parser.valid = false; + parser.error = MEMALLOC; + return parser; + } + json_parse(buf, len, parser, realloc_if_needed); + return parser; +} +WARN_UNUSED inline document::parser build_parsed_json(const char *buf, size_t len, bool realloc_if_needed = true) noexcept { + return build_parsed_json(reinterpret_cast(buf), len, realloc_if_needed); +} +WARN_UNUSED inline document::parser build_parsed_json(const std::string &s, bool realloc_if_needed = true) noexcept { + return build_parsed_json(s.data(), s.length(), realloc_if_needed); +} +WARN_UNUSED inline document::parser build_parsed_json(const padded_string &s) noexcept { + return build_parsed_json(s.data(), s.length(), false); +} + +// We do not want to allow implicit conversion from C string to std::string. +int json_parse(const char *buf, document::parser &parser) noexcept = delete; +document::parser build_parsed_json(const char *buf) noexcept = delete; + +} // namespace simdjson + +#endif +/* end file include/simdjson/jsonioutil.h */ + +// Inline functions +/* begin file include/simdjson/inline/document.h */ +#ifndef SIMDJSON_INLINE_DOCUMENT_H +#define SIMDJSON_INLINE_DOCUMENT_H + +#ifndef SIMDJSON_DOCUMENT_H +#error This is an internal file only. Include document.h instead. +#endif + +// Inline implementations go in here if they aren't small enough to go in the class itself or if +// there are complex header file dependencies that need to be broken by externalizing the +// implementation. + + +namespace simdjson { + +// TODO inline? +document::doc_ref_result document::parser::parse(const uint8_t *buf, size_t len, bool realloc_if_needed) noexcept { + error_code code = init_parse(len); + if (code) { return document::doc_ref_result(doc, code); } + + if (realloc_if_needed) { + const uint8_t *tmp_buf = buf; + buf = (uint8_t *)allocate_padded_buffer(len); + if (buf == nullptr) + return document::doc_ref_result(doc, MEMALLOC); + memcpy((void *)buf, tmp_buf, len); + } + + code = simdjson::active_implementation->parse(buf, len, *this); + + // We're indicating validity via the doc_ref_result, so set the parse state back to invalid + valid = false; + error = UNINITIALIZED; + if (realloc_if_needed) { + aligned_free((void *)buf); // must free before we exit + } + return document::doc_ref_result(doc, code); +} +really_inline document::doc_ref_result document::parser::parse(const char *buf, size_t len, bool realloc_if_needed) noexcept { + return parse((const uint8_t *)buf, len, realloc_if_needed); +} +really_inline document::doc_ref_result document::parser::parse(const std::string &s, bool realloc_if_needed) noexcept { + return parse(s.data(), s.length(), realloc_if_needed); +} +really_inline document::doc_ref_result document::parser::parse(const padded_string &s) noexcept { + return parse(s.data(), s.length(), false); +} + +// TODO really_inline? +inline document::doc_result document::parse(const uint8_t *buf, size_t len, bool realloc_if_needed) noexcept { + document::parser parser; + if (!parser.allocate_capacity(len)) { + return MEMALLOC; + } + auto [doc, error] = parser.parse(buf, len, realloc_if_needed); + return document::doc_result((document &&)doc, error); +} +really_inline document::doc_result document::parse(const char *buf, size_t len, bool realloc_if_needed) noexcept { + return parse((const uint8_t *)buf, len, realloc_if_needed); +} +really_inline document::doc_result document::parse(const std::string &s, bool realloc_if_needed) noexcept { + return parse(s.data(), s.length(), realloc_if_needed); +} +really_inline document::doc_result document::parse(const padded_string &s) noexcept { + return parse(s.data(), s.length(), false); +} + +// +// Parser callbacks +// + +WARN_UNUSED +inline error_code document::parser::init_parse(size_t len) { + if (len > capacity()) { + return error = CAPACITY; + } + // If the last doc was taken, we need to allocate a new one + if (!doc.tape) { + if (!doc.set_capacity(len)) { + return error = MEMALLOC; + } + } + return SUCCESS; +} + +inline void document::parser::init_stage2() { + current_string_buf_loc = doc.string_buf.get(); + current_loc = 0; + valid = false; + error = UNINITIALIZED; +} + +really_inline error_code document::parser::on_error(error_code new_error_code) { + error = new_error_code; + return new_error_code; +} +really_inline error_code document::parser::on_success(error_code success_code) { + error = success_code; + valid = true; + return success_code; +} +really_inline bool document::parser::on_start_document(uint32_t depth) { + containing_scope_offset[depth] = current_loc; + write_tape(0, 'r'); + return true; +} +really_inline bool document::parser::on_start_object(uint32_t depth) { + containing_scope_offset[depth] = current_loc; + write_tape(0, '{'); + return true; +} +really_inline bool document::parser::on_start_array(uint32_t depth) { + containing_scope_offset[depth] = current_loc; + write_tape(0, '['); + return true; +} +// TODO we're not checking this bool +really_inline bool document::parser::on_end_document(uint32_t depth) { + // write our doc.tape location to the header scope + // The root scope gets written *at* the previous location. + annotate_previous_loc(containing_scope_offset[depth], current_loc); + write_tape(containing_scope_offset[depth], 'r'); + return true; +} +really_inline bool document::parser::on_end_object(uint32_t depth) { + // write our doc.tape location to the header scope + write_tape(containing_scope_offset[depth], '}'); + annotate_previous_loc(containing_scope_offset[depth], current_loc); + return true; +} +really_inline bool document::parser::on_end_array(uint32_t depth) { + // write our doc.tape location to the header scope + write_tape(containing_scope_offset[depth], ']'); + annotate_previous_loc(containing_scope_offset[depth], current_loc); + return true; +} + +really_inline bool document::parser::on_true_atom() { + write_tape(0, 't'); + return true; +} +really_inline bool document::parser::on_false_atom() { + write_tape(0, 'f'); + return true; +} +really_inline bool document::parser::on_null_atom() { + write_tape(0, 'n'); + return true; +} + +really_inline uint8_t *document::parser::on_start_string() { + /* we advance the point, accounting for the fact that we have a NULL + * termination */ + write_tape(current_string_buf_loc - doc.string_buf.get(), '"'); + return current_string_buf_loc + sizeof(uint32_t); +} + +really_inline bool document::parser::on_end_string(uint8_t *dst) { + uint32_t str_length = dst - (current_string_buf_loc + sizeof(uint32_t)); + // TODO check for overflow in case someone has a crazy string (>=4GB?) + // But only add the overflow check when the document itself exceeds 4GB + // Currently unneeded because we refuse to parse docs larger or equal to 4GB. + memcpy(current_string_buf_loc, &str_length, sizeof(uint32_t)); + // NULL termination is still handy if you expect all your strings to + // be NULL terminated? It comes at a small cost + *dst = 0; + current_string_buf_loc = dst + 1; + return true; +} + +really_inline bool document::parser::on_number_s64(int64_t value) { + write_tape(0, 'l'); + std::memcpy(&doc.tape[current_loc], &value, sizeof(value)); + ++current_loc; + return true; +} +really_inline bool document::parser::on_number_u64(uint64_t value) { + write_tape(0, 'u'); + doc.tape[current_loc++] = value; + return true; +} +really_inline bool document::parser::on_number_double(double value) { + write_tape(0, 'd'); + static_assert(sizeof(value) == sizeof(doc.tape[current_loc]), "mismatch size"); + memcpy(&doc.tape[current_loc++], &value, sizeof(double)); + // doc.tape[doc.current_loc++] = *((uint64_t *)&d); + return true; +} + +} // namespace simdjson + +#endif // SIMDJSON_INLINE_DOCUMENT_H +/* end file include/simdjson/inline/document.h */ /* begin file include/simdjson/inline/document_iterator.h */ #ifndef SIMDJSON_INLINE_DOCUMENT_ITERATOR_H #define SIMDJSON_INLINE_DOCUMENT_ITERATOR_H @@ -2239,788 +3052,5 @@ bool document_iterator::relative_move_to(const char *pointer, #endif // SIMDJSON_INLINE_DOCUMENT_ITERATOR_H /* end file include/simdjson/inline/document_iterator.h */ -#endif // SIMDJSON_DOCUMENT_ITERATOR_H +#endif // SIMDJSON_H /* end file include/simdjson/inline/document_iterator.h */ - -#endif // SIMDJSON_DOCUMENT_H -/* end file include/simdjson/inline/document_iterator.h */ -/* begin file include/simdjson/parsedjson.h */ -// TODO Remove this -- deprecated API and files - -#ifndef SIMDJSON_PARSEDJSON_H -#define SIMDJSON_PARSEDJSON_H - - -namespace simdjson { - -using ParsedJson = document::parser; - -} // namespace simdjson -#endif -/* end file include/simdjson/parsedjson.h */ -/* begin file include/simdjson/jsonparser.h */ -// TODO Remove this -- deprecated API and files - -#ifndef SIMDJSON_JSONPARSER_H -#define SIMDJSON_JSONPARSER_H - - -namespace simdjson { - -// -// C API (json_parse and build_parsed_json) declarations -// - -inline int json_parse(const uint8_t *buf, size_t len, document::parser &parser, bool realloc_if_needed = true) noexcept { - error_code code = parser.parse(buf, len, realloc_if_needed).error; - // The deprecated json_parse API is a signal that the user plans to *use* the error code / valid - // bits in the parser instead of heeding the result code. The normal parser unsets those in - // anticipation of making the error code ephemeral. - // Here we put the code back into the parser, until we've removed this method. - parser.valid = code == SUCCESS; - parser.error = code; - return code; -} -inline int json_parse(const char *buf, size_t len, document::parser &parser, bool realloc_if_needed = true) noexcept { - return json_parse(reinterpret_cast(buf), len, parser, realloc_if_needed); -} -inline int json_parse(const std::string &s, document::parser &parser, bool realloc_if_needed = true) noexcept { - return json_parse(s.data(), s.length(), parser, realloc_if_needed); -} -inline int json_parse(const padded_string &s, document::parser &parser) noexcept { - return json_parse(s.data(), s.length(), parser, false); -} - -WARN_UNUSED static document::parser build_parsed_json(const uint8_t *buf, size_t len, bool realloc_if_needed = true) noexcept { - document::parser parser; - if (!parser.allocate_capacity(len)) { - parser.valid = false; - parser.error = MEMALLOC; - return parser; - } - json_parse(buf, len, parser, realloc_if_needed); - return parser; -} -WARN_UNUSED inline document::parser build_parsed_json(const char *buf, size_t len, bool realloc_if_needed = true) noexcept { - return build_parsed_json(reinterpret_cast(buf), len, realloc_if_needed); -} -WARN_UNUSED inline document::parser build_parsed_json(const std::string &s, bool realloc_if_needed = true) noexcept { - return build_parsed_json(s.data(), s.length(), realloc_if_needed); -} -WARN_UNUSED inline document::parser build_parsed_json(const padded_string &s) noexcept { - return build_parsed_json(s.data(), s.length(), false); -} - -// We do not want to allow implicit conversion from C string to std::string. -int json_parse(const char *buf, document::parser &parser) noexcept = delete; -document::parser build_parsed_json(const char *buf) noexcept = delete; - -} // namespace simdjson - -#endif -/* end file include/simdjson/jsonparser.h */ -/* begin file include/simdjson/jsonstream.h */ -#ifndef SIMDJSON_JSONSTREAM_H -#define SIMDJSON_JSONSTREAM_H - -#include -#include -#include -#include -/* begin file src/jsoncharutils.h */ -#ifndef SIMDJSON_JSONCHARUTILS_H -#define SIMDJSON_JSONCHARUTILS_H - - -namespace simdjson { -// structural chars here are -// they are { 0x7b } 0x7d : 0x3a [ 0x5b ] 0x5d , 0x2c (and NULL) -// we are also interested in the four whitespace characters -// space 0x20, linefeed 0x0a, horizontal tab 0x09 and carriage return 0x0d - -// these are the chars that can follow a true/false/null or number atom -// and nothing else -const uint32_t structural_or_whitespace_or_null_negated[256] = { - 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, - - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, - - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; - -// return non-zero if not a structural or whitespace char -// zero otherwise -really_inline uint32_t is_not_structural_or_whitespace_or_null(uint8_t c) { - return structural_or_whitespace_or_null_negated[c]; -} - -const uint32_t structural_or_whitespace_negated[256] = { - 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, - - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, - - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; - -// return non-zero if not a structural or whitespace char -// zero otherwise -really_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { - return structural_or_whitespace_negated[c]; -} - -const uint32_t structural_or_whitespace_or_null[256] = { - 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; - -really_inline uint32_t is_structural_or_whitespace_or_null(uint8_t c) { - return structural_or_whitespace_or_null[c]; -} - -const uint32_t structural_or_whitespace[256] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; - -really_inline uint32_t is_structural_or_whitespace(uint8_t c) { - return structural_or_whitespace[c]; -} - -const uint32_t digit_to_val32[886] = { - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, - 0x6, 0x7, 0x8, 0x9, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xa, - 0xb, 0xc, 0xd, 0xe, 0xf, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xa, 0xb, 0xc, 0xd, 0xe, - 0xf, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0x0, 0x10, 0x20, 0x30, 0x40, 0x50, - 0x60, 0x70, 0x80, 0x90, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xa0, - 0xb0, 0xc0, 0xd0, 0xe0, 0xf0, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, - 0xf0, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0x0, 0x100, 0x200, 0x300, 0x400, 0x500, - 0x600, 0x700, 0x800, 0x900, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xa00, - 0xb00, 0xc00, 0xd00, 0xe00, 0xf00, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xa00, 0xb00, 0xc00, 0xd00, 0xe00, - 0xf00, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0x0, 0x1000, 0x2000, 0x3000, 0x4000, 0x5000, - 0x6000, 0x7000, 0x8000, 0x9000, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xa000, - 0xb000, 0xc000, 0xd000, 0xe000, 0xf000, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xa000, 0xb000, 0xc000, 0xd000, 0xe000, - 0xf000, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}; -// returns a value with the high 16 bits set if not valid -// otherwise returns the conversion of the 4 hex digits at src into the bottom -// 16 bits of the 32-bit return register -// -// see -// https://lemire.me/blog/2019/04/17/parsing-short-hexadecimal-strings-efficiently/ -static inline uint32_t hex_to_u32_nocheck( - const uint8_t *src) { // strictly speaking, static inline is a C-ism - uint32_t v1 = digit_to_val32[630 + src[0]]; - uint32_t v2 = digit_to_val32[420 + src[1]]; - uint32_t v3 = digit_to_val32[210 + src[2]]; - uint32_t v4 = digit_to_val32[0 + src[3]]; - return v1 | v2 | v3 | v4; -} - -// returns true if the provided byte value is a -// "continuing" UTF-8 value, that is, if it starts with -// 0b10... -static inline bool is_utf8_continuing(char c) { - // in 2 complement's notation, values start at 0b10000 (-128)... and - // go up to 0b11111 (-1)... so we want all values from -128 to -65 (which is 0b10111111) - return ((signed char)c) <= -65; -} -// returns true if the provided byte value is an ASCII character -static inline bool is_ascii(char c) { - return ((unsigned char)c) <= 127; -} - -// if the string ends with UTF-8 values, backtrack -// up to the first ASCII character. May return 0. -static inline size_t trimmed_length_safe_utf8(const char * c, size_t len) { - while ((len > 0) and (not is_ascii(c[len - 1]))) { - len--; - } - return len; -} - - - -// given a code point cp, writes to c -// the utf-8 code, outputting the length in -// bytes, if the length is zero, the code point -// is invalid -// -// This can possibly be made faster using pdep -// and clz and table lookups, but JSON documents -// have few escaped code points, and the following -// function looks cheap. -// -// Note: we assume that surrogates are treated separately -// -inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { - if (cp <= 0x7F) { - c[0] = cp; - return 1; // ascii - } - if (cp <= 0x7FF) { - c[0] = (cp >> 6) + 192; - c[1] = (cp & 63) + 128; - return 2; // universal plane - // Surrogates are treated elsewhere... - //} //else if (0xd800 <= cp && cp <= 0xdfff) { - // return 0; // surrogates // could put assert here - } else if (cp <= 0xFFFF) { - c[0] = (cp >> 12) + 224; - c[1] = ((cp >> 6) & 63) + 128; - c[2] = (cp & 63) + 128; - return 3; - } else if (cp <= 0x10FFFF) { // if you know you have a valid code point, this - // is not needed - c[0] = (cp >> 18) + 240; - c[1] = ((cp >> 12) & 63) + 128; - c[2] = ((cp >> 6) & 63) + 128; - c[3] = (cp & 63) + 128; - return 4; - } - // will return 0 when the code point was too large. - return 0; // bad r -} -} // namespace simdjson - -#endif -/* end file src/jsoncharutils.h */ - - -namespace simdjson { -/************************************************************************************* - * The main motivation for this piece of software is to achieve maximum speed - *and offer - * good quality of life while parsing files containing multiple JSON documents. - * - * Since we want to offer flexibility and not restrict ourselves to a specific - *file - * format, we support any file that contains any valid JSON documents separated - *by one - * or more character that is considered a whitespace by the JSON spec. - * Namely: space, nothing, linefeed, carriage return, horizontal tab. - * Anything that is not whitespace will be parsed as a JSON document and could - *lead - * to failure. - * - * To offer maximum parsing speed, our implementation processes the data inside - *the - * buffer by batches and their size is defined by the parameter "batch_size". - * By loading data in batches, we can optimize the time spent allocating data in - *the - * parser and can also open the possibility of multi-threading. - * The batch_size must be at least as large as the biggest document in the file, - *but - * not too large in order to submerge the chached memory. We found that 1MB is - * somewhat a sweet spot for now. Eventually, this batch_size could be fully - * automated and be optimal at all times. - ************************************************************************************/ -/** -* The template parameter (string_container) must -* support the data() and size() methods, returning a pointer -* to a char* and to the number of bytes respectively. -* The simdjson parser may read up to SIMDJSON_PADDING bytes beyond the end -* of the string, so if you do not use a padded_string container, -* you have the responsability to overallocated. If you fail to -* do so, your software may crash if you cross a page boundary, -* and you should expect memory checkers to object. -* Most users should use a simdjson::padded_string. -*/ -template class JsonStream { -public: - /* Create a JsonStream object that can be used to parse sequentially the valid - * JSON documents found in the buffer "buf". - * - * The batch_size must be at least as large as the biggest document in the - * file, but - * not too large to submerge the cached memory. We found that 1MB is - * somewhat a sweet spot for now. - * - * The user is expected to call the following json_parse method to parse the - * next - * valid JSON document found in the buffer. This method can and is expected - * to be - * called in a loop. - * - * Various methods are offered to keep track of the status, like - * get_current_buffer_loc, - * get_n_parsed_docs, get_n_bytes_parsed, etc. - * - * */ - JsonStream(const string_container &s, size_t batch_size = 1000000); - - ~JsonStream(); - - /* Parse the next document found in the buffer previously given to JsonStream. - - * The content should be a valid JSON document encoded as UTF-8. If there is a - * UTF-8 BOM, the caller is responsible for omitting it, UTF-8 BOM are - * discouraged. - * - * You do NOT need to pre-allocate a parser. This function takes care of - * pre-allocating a capacity defined by the batch_size defined when creating - the - * JsonStream object. - * - * The function returns simdjson::SUCCESS_AND_HAS_MORE (an integer = 1) in - case - * of success and indicates that the buffer still contains more data to be - parsed, - * meaning this function can be called again to return the next JSON document - * after this one. - * - * The function returns simdjson::SUCCESS (as integer = 0) in case of success - * and indicates that the buffer has successfully been parsed to the end. - * Every document it contained has been parsed without error. - * - * The function returns an error code from simdjson/simdjson.h in case of - failure - * such as simdjson::CAPACITY, simdjson::MEMALLOC, simdjson::DEPTH_ERROR and - so forth; - * the simdjson::error_message function converts these error codes into a - * string). - * - * You can also check validity by calling parser.is_valid(). The same parser - can - * and should be reused for the other documents in the buffer. */ - int json_parse(document::parser &parser); - - /* Returns the location (index) of where the next document should be in the - * buffer. - * Can be used for debugging, it tells the user the position of the end of the - * last - * valid JSON document parsed*/ - inline size_t get_current_buffer_loc() const { return current_buffer_loc; } - - /* Returns the total amount of complete documents parsed by the JsonStream, - * in the current buffer, at the given time.*/ - inline size_t get_n_parsed_docs() const { return n_parsed_docs; } - - /* Returns the total amount of data (in bytes) parsed by the JsonStream, - * in the current buffer, at the given time.*/ - inline size_t get_n_bytes_parsed() const { return n_bytes_parsed; } - -private: - inline const uint8_t *buf() const { return reinterpret_cast(str.data()) + str_start; } - - inline void advance(size_t offset) { str_start += offset; } - - inline size_t remaining() const { return str.size() - str_start; } - - const string_container &str; - size_t _batch_size; // this is actually variable! - size_t str_start{0}; - size_t next_json{0}; - bool load_next_batch{true}; - size_t current_buffer_loc{0}; -#ifdef SIMDJSON_THREADS_ENABLED - size_t last_json_buffer_loc{0}; -#endif - size_t n_parsed_docs{0}; - size_t n_bytes_parsed{0}; - simdjson::implementation *stage_parser; -#ifdef SIMDJSON_THREADS_ENABLED - error_code stage1_is_ok_thread{SUCCESS}; - std::thread stage_1_thread; - document::parser parser_thread; -#endif -}; // end of class JsonStream - -/* This algorithm is used to quickly identify the buffer position of - * the last JSON document inside the current batch. - * - * It does its work by finding the last pair of structural characters - * that represent the end followed by the start of a document. - * - * Simply put, we iterate over the structural characters, starting from - * the end. We consider that we found the end of a JSON document when the - * first element of the pair is NOT one of these characters: '{' '[' ';' ',' - * and when the second element is NOT one of these characters: '}' '}' ';' ','. - * - * This simple comparison works most of the time, but it does not cover cases - * where the batch's structural indexes contain a perfect amount of documents. - * In such a case, we do not have access to the structural index which follows - * the last document, therefore, we do not have access to the second element in - * the pair, and means that we cannot identify the last document. To fix this - * issue, we keep a count of the open and closed curly/square braces we found - * while searching for the pair. When we find a pair AND the count of open and - * closed curly/square braces is the same, we know that we just passed a - * complete - * document, therefore the last json buffer location is the end of the batch - * */ -inline size_t find_last_json_buf_idx(const uint8_t *buf, size_t size, - const document::parser &parser) { - // this function can be generally useful - if (parser.n_structural_indexes == 0) - return 0; - auto last_i = parser.n_structural_indexes - 1; - if (parser.structural_indexes[last_i] == size) { - if (last_i == 0) - return 0; - last_i = parser.n_structural_indexes - 2; - } - auto arr_cnt = 0; - auto obj_cnt = 0; - for (auto i = last_i; i > 0; i--) { - auto idxb = parser.structural_indexes[i]; - switch (buf[idxb]) { - case ':': - case ',': - continue; - case '}': - obj_cnt--; - continue; - case ']': - arr_cnt--; - continue; - case '{': - obj_cnt++; - break; - case '[': - arr_cnt++; - break; - } - auto idxa = parser.structural_indexes[i - 1]; - switch (buf[idxa]) { - case '{': - case '[': - case ':': - case ',': - continue; - } - if (!arr_cnt && !obj_cnt) { - return last_i + 1; - } - return i; - } - return 0; -} - -template -JsonStream::JsonStream(const string_container &s, - size_t batchSize) - : str(s), _batch_size(batchSize) { -} - -template JsonStream::~JsonStream() { -#ifdef SIMDJSON_THREADS_ENABLED - if (stage_1_thread.joinable()) { - stage_1_thread.join(); - } -#endif -} - -#ifdef SIMDJSON_THREADS_ENABLED - -// threaded version of json_parse -// todo: simplify this code further -template -int JsonStream::json_parse(document::parser &parser) { - if (unlikely(parser.capacity() == 0)) { - const bool allocok = parser.allocate_capacity(_batch_size); - if (!allocok) { - return parser.error = simdjson::MEMALLOC; - } - } else if (unlikely(parser.capacity() < _batch_size)) { - return parser.error = simdjson::CAPACITY; - } - if (unlikely(parser_thread.capacity() < _batch_size)) { - const bool allocok_thread = parser_thread.allocate_capacity(_batch_size); - if (!allocok_thread) { - return parser.error = simdjson::MEMALLOC; - } - } - if (unlikely(load_next_batch)) { - // First time loading - if (!stage_1_thread.joinable()) { - _batch_size = (std::min)(_batch_size, remaining()); - _batch_size = trimmed_length_safe_utf8((const char *)buf(), _batch_size); - if (_batch_size == 0) { - return parser.error = simdjson::UTF8_ERROR; - } - auto stage1_is_ok = error_code(simdjson::active_implementation->stage1(buf(), _batch_size, parser, true)); - if (stage1_is_ok != simdjson::SUCCESS) { - return parser.error = stage1_is_ok; - } - size_t last_index = find_last_json_buf_idx(buf(), _batch_size, parser); - if (last_index == 0) { - if (parser.n_structural_indexes == 0) { - return parser.error = simdjson::EMPTY; - } - } else { - parser.n_structural_indexes = last_index + 1; - } - } - // the second thread is running or done. - else { - stage_1_thread.join(); - if (stage1_is_ok_thread != simdjson::SUCCESS) { - return parser.error = stage1_is_ok_thread; - } - std::swap(parser.structural_indexes, parser_thread.structural_indexes); - parser.n_structural_indexes = parser_thread.n_structural_indexes; - advance(last_json_buffer_loc); - n_bytes_parsed += last_json_buffer_loc; - } - // let us decide whether we will start a new thread - if (remaining() - _batch_size > 0) { - last_json_buffer_loc = - parser.structural_indexes[find_last_json_buf_idx(buf(), _batch_size, parser)]; - _batch_size = (std::min)(_batch_size, remaining() - last_json_buffer_loc); - if (_batch_size > 0) { - _batch_size = trimmed_length_safe_utf8( - (const char *)(buf() + last_json_buffer_loc), _batch_size); - if (_batch_size == 0) { - return parser.error = simdjson::UTF8_ERROR; - } - // let us capture read-only variables - const uint8_t *const b = buf() + last_json_buffer_loc; - const size_t bs = _batch_size; - // we call the thread on a lambda that will update - // this->stage1_is_ok_thread - // there is only one thread that may write to this value - stage_1_thread = std::thread([this, b, bs] { - this->stage1_is_ok_thread = error_code(simdjson::active_implementation->stage1(b, bs, this->parser_thread, true)); - }); - } - } - next_json = 0; - load_next_batch = false; - } // load_next_batch - int res = simdjson::active_implementation->stage2(buf(), remaining(), parser, next_json); - if (res == simdjson::SUCCESS_AND_HAS_MORE) { - n_parsed_docs++; - current_buffer_loc = parser.structural_indexes[next_json]; - load_next_batch = (current_buffer_loc == last_json_buffer_loc); - } else if (res == simdjson::SUCCESS) { - n_parsed_docs++; - if (remaining() > _batch_size) { - current_buffer_loc = parser.structural_indexes[next_json - 1]; - load_next_batch = true; - res = simdjson::SUCCESS_AND_HAS_MORE; - } - } - return res; -} - -#else // SIMDJSON_THREADS_ENABLED - -// single-threaded version of json_parse -template -int JsonStream::json_parse(document::parser &parser) { - if (unlikely(parser.capacity() == 0)) { - const bool allocok = parser.allocate_capacity(_batch_size); - if (!allocok) { - return parser.on_error(MEMALLOC); - } - } else if (unlikely(parser.capacity() < _batch_size)) { - return parser.on_error(CAPACITY); - } - if (unlikely(load_next_batch)) { - advance(current_buffer_loc); - n_bytes_parsed += current_buffer_loc; - _batch_size = (std::min)(_batch_size, remaining()); - _batch_size = trimmed_length_safe_utf8((const char *)buf(), _batch_size); - auto stage1_is_ok = (error_code)simdjson::active_implementation->stage1(buf(), _batch_size, parser, true); - if (stage1_is_ok != simdjson::SUCCESS) { - return parser.on_error(stage1_is_ok); - } - size_t last_index = find_last_json_buf_idx(buf(), _batch_size, parser); - if (last_index == 0) { - if (parser.n_structural_indexes == 0) { - return parser.on_error(EMPTY); - } - } else { - parser.n_structural_indexes = last_index + 1; - } - load_next_batch = false; - } // load_next_batch - int res = simdjson::active_implementation->stage2(buf(), remaining(), parser, next_json); - if (likely(res == simdjson::SUCCESS_AND_HAS_MORE)) { - n_parsed_docs++; - current_buffer_loc = parser.structural_indexes[next_json]; - } else if (res == simdjson::SUCCESS) { - n_parsed_docs++; - if (remaining() > _batch_size) { - current_buffer_loc = parser.structural_indexes[next_json - 1]; - next_json = 1; - load_next_batch = true; - res = simdjson::SUCCESS_AND_HAS_MORE; - } - } else { - printf("E\n"); - } - return res; -} -#endif // SIMDJSON_THREADS_ENABLED - -} // end of namespace simdjson -#endif // SIMDJSON_JSONSTREAM_H -/* end file src/jsoncharutils.h */ diff --git a/src/arm64/numberparsing.h b/src/arm64/numberparsing.h index 4cb47c03c..b3b092e7d 100644 --- a/src/arm64/numberparsing.h +++ b/src/arm64/numberparsing.h @@ -7,7 +7,7 @@ #include "simdjson/portability.h" #include "arm64/intrinsics.h" #include "arm64/bitmanipulation.h" -#include "simdjson/parsedjson.h" +#include "simdjson/inline/document.h" #include "jsoncharutils.h" #include #include diff --git a/src/arm64/stringparsing.h b/src/arm64/stringparsing.h index b2fc39cc1..ae07c5ee3 100644 --- a/src/arm64/stringparsing.h +++ b/src/arm64/stringparsing.h @@ -7,7 +7,7 @@ #include "arm64/simd.h" #include "simdjson/common_defs.h" -#include "simdjson/parsedjson.h" +#include "simdjson/inline/document.h" #include "jsoncharutils.h" #include "arm64/intrinsics.h" #include "arm64/bitmanipulation.h" diff --git a/src/document.cpp b/src/document.cpp index 19a2411b2..5a1880082 100644 --- a/src/document.cpp +++ b/src/document.cpp @@ -1,6 +1,6 @@ #include "simdjson/document.h" +#include "simdjson/inline/document.h" #include "simdjson/jsonformatutils.h" -#include "simdjson/document.h" namespace simdjson { diff --git a/src/generic/numberparsing.h b/src/generic/numberparsing.h index 82a2aa4b0..b8d481360 100644 --- a/src/generic/numberparsing.h +++ b/src/generic/numberparsing.h @@ -374,9 +374,9 @@ never_inline bool parse_large_integer(const uint8_t *const buf, // content and append a space before calling this function. // // Our objective is accurate parsing (ULP of 0 or 1) at high speed. -really_inline bool parse_number(const uint8_t *const buf, - const uint32_t offset, - bool found_minus, +really_inline bool parse_number(UNUSED const uint8_t *const buf, + UNUSED const uint32_t offset, + UNUSED bool found_minus, document::parser &parser) { #ifdef SIMDJSON_SKIPNUMBERPARSING // for performance analysis, it is sometimes // useful to skip parsing diff --git a/src/haswell/numberparsing.h b/src/haswell/numberparsing.h index f1820077b..88f1e0b66 100644 --- a/src/haswell/numberparsing.h +++ b/src/haswell/numberparsing.h @@ -7,7 +7,7 @@ #include "simdjson/portability.h" #include "haswell/intrinsics.h" #include "haswell/bitmanipulation.h" -#include "simdjson/parsedjson.h" +#include "simdjson/inline/document.h" #include "jsoncharutils.h" #include #include diff --git a/src/haswell/stringparsing.h b/src/haswell/stringparsing.h index 587f8e16c..5a4ea5cf3 100644 --- a/src/haswell/stringparsing.h +++ b/src/haswell/stringparsing.h @@ -7,7 +7,7 @@ #include "haswell/simd.h" #include "simdjson/common_defs.h" -#include "simdjson/parsedjson.h" +#include "simdjson/inline/document.h" #include "jsoncharutils.h" #include "haswell/intrinsics.h" #include "haswell/bitmanipulation.h" diff --git a/src/westmere/numberparsing.h b/src/westmere/numberparsing.h index f2f93bd82..babfebc0f 100644 --- a/src/westmere/numberparsing.h +++ b/src/westmere/numberparsing.h @@ -10,7 +10,7 @@ #include "simdjson/portability.h" #include "westmere/intrinsics.h" #include "westmere/bitmanipulation.h" -#include "simdjson/parsedjson.h" +#include "simdjson/inline/document.h" #include "jsoncharutils.h" #include #include diff --git a/src/westmere/stringparsing.h b/src/westmere/stringparsing.h index f8acfd642..b6acd9508 100644 --- a/src/westmere/stringparsing.h +++ b/src/westmere/stringparsing.h @@ -7,7 +7,7 @@ #include "westmere/simd.h" #include "simdjson/common_defs.h" -#include "simdjson/parsedjson.h" +#include "simdjson/inline/document.h" #include "jsoncharutils.h" #include "westmere/intrinsics.h" #include "westmere/bitmanipulation.h" diff --git a/tests/allparserscheckfile.cpp b/tests/allparserscheckfile.cpp index 4da586792..e3940a4be 100644 --- a/tests/allparserscheckfile.cpp +++ b/tests/allparserscheckfile.cpp @@ -1,6 +1,6 @@ #include -#include "simdjson/jsonparser.h" +#include "simdjson.h" // #define RAPIDJSON_SSE2 // bad // #define RAPIDJSON_SSE42 // bad diff --git a/tests/basictests.cpp b/tests/basictests.cpp index baa2d94c6..d2154884e 100644 --- a/tests/basictests.cpp +++ b/tests/basictests.cpp @@ -10,9 +10,7 @@ #include #include -#include "simdjson/jsonparser.h" -#include "simdjson/jsonstream.h" -#include "simdjson/document.h" +#include "simdjson.h" #ifndef JSON_TEST_PATH #define JSON_TEST_PATH "jsonexamples/twitter.json" diff --git a/tests/integer_tests.cpp b/tests/integer_tests.cpp index be99c1443..ca329fa4f 100644 --- a/tests/integer_tests.cpp +++ b/tests/integer_tests.cpp @@ -3,8 +3,7 @@ #include #include -#include "simdjson/jsonparser.h" -#include "simdjson/document.h" +#include "simdjson.h" using namespace simdjson; diff --git a/tests/jsoncheck.cpp b/tests/jsoncheck.cpp index d7256d142..c17135b28 100644 --- a/tests/jsoncheck.cpp +++ b/tests/jsoncheck.cpp @@ -12,7 +12,7 @@ #include #include -#include "simdjson/jsonparser.h" +#include "simdjson.h" /** * Does the file filename ends with the given extension. diff --git a/tests/jsonstream_test.cpp b/tests/jsonstream_test.cpp index b3a489119..71c4e82bb 100644 --- a/tests/jsonstream_test.cpp +++ b/tests/jsonstream_test.cpp @@ -11,9 +11,8 @@ #include #include -#include -#include "simdjson/jsonparser.h" +#include "simdjson.h" /** * Does the file filename ends with the given extension. diff --git a/tests/numberparsingcheck.cpp b/tests/numberparsingcheck.cpp index 782f62dd6..92fd07eee 100644 --- a/tests/numberparsingcheck.cpp +++ b/tests/numberparsingcheck.cpp @@ -11,7 +11,7 @@ #define JSON_TEST_NUMBERS #endif -#include "simdjson/common_defs.h" +#include "simdjson.h" // ulp distance // Marc B. Reynolds, 2016-2019 @@ -132,7 +132,7 @@ void found_float(double result, const uint8_t *buf) { } } -#include "simdjson/jsonparser.h" +#include "simdjson.h" #include "src/stage2_build_tape.cpp" /** diff --git a/tests/pointercheck.cpp b/tests/pointercheck.cpp index 764fecd9d..6a6affbb8 100644 --- a/tests/pointercheck.cpp +++ b/tests/pointercheck.cpp @@ -1,7 +1,6 @@ #include -#include "simdjson/jsonparser.h" -#include "simdjson/parsedjson.h" +#include "simdjson.h" int main() { // {"/~01abc": [0, {"\\\" 0": ["value0", "value1"]}]}" diff --git a/tests/readme_examples.cpp b/tests/readme_examples.cpp index 422022b14..31ac32b34 100644 --- a/tests/readme_examples.cpp +++ b/tests/readme_examples.cpp @@ -1,6 +1,5 @@ #include -#include "simdjson/document.h" -#include "simdjson/jsonioutil.h" +#include "simdjson.h" using namespace std; using namespace simdjson; @@ -10,7 +9,7 @@ void document_parse_error_code() { string json("[ 1, 2, 3 ]"); auto [doc, error] = document::parse(json); if (error) { cerr << "Error: " << error_message(error) << endl; exit(1); } - doc.print_json(cout); + UNUSED doc.print_json(cout); cout << endl; } @@ -19,7 +18,7 @@ void document_parse_exception() { string json("[ 1, 2, 3 ]"); document doc = document::parse(json); - doc.print_json(cout); + UNUSED doc.print_json(cout); cout << endl; } @@ -28,7 +27,7 @@ void document_parse_padded_string() { padded_string json(string("[ 1, 2, 3 ]")); document doc = document::parse(json); - doc.print_json(cout); + UNUSED doc.print_json(cout); cout << endl; } @@ -37,7 +36,7 @@ void document_parse_get_corpus() { padded_string json(get_corpus("jsonexamples/small/demo.json")); document doc = document::parse(json); - doc.print_json(cout); + UNUSED doc.print_json(cout); cout << endl; } @@ -53,7 +52,7 @@ void parser_parse() { cout << "Parsing " << json.data() << " ..." << endl; auto [doc, error] = parser.parse(json); if (error) { cerr << "Error: " << error_message(error) << endl; exit(1); } - doc.print_json(cout); + UNUSED doc.print_json(cout); cout << endl; } } diff --git a/tests/stringparsingcheck.cpp b/tests/stringparsingcheck.cpp index 9424c4c27..7bf45ec8f 100644 --- a/tests/stringparsingcheck.cpp +++ b/tests/stringparsingcheck.cpp @@ -13,7 +13,7 @@ #define JSON_TEST_STRINGS #endif -#include "simdjson/common_defs.h" +#include "simdjson.h" char *fullpath; @@ -289,7 +289,7 @@ void found_string(const uint8_t *buf, const uint8_t *parsed_begin, } } -#include "simdjson/jsonparser.h" +#include "simdjson.h" #include "src/stage2_build_tape.cpp" /** diff --git a/tools/json2json.cpp b/tools/json2json.cpp index f2415575a..7a110b749 100644 --- a/tools/json2json.cpp +++ b/tools/json2json.cpp @@ -2,8 +2,7 @@ #ifndef _MSC_VER #include #endif -#include "simdjson/jsonioutil.h" -#include "simdjson/jsonparser.h" +#include "simdjson.h" void compute_dump(simdjson::ParsedJson::Iterator &pjh) { if (pjh.is_object()) { diff --git a/tools/jsonpointer.cpp b/tools/jsonpointer.cpp index e3a1eec0a..38910c847 100644 --- a/tools/jsonpointer.cpp +++ b/tools/jsonpointer.cpp @@ -1,5 +1,4 @@ -#include "simdjson/jsonioutil.h" -#include "simdjson/jsonparser.h" +#include "simdjson.h" #include void compute_dump(simdjson::ParsedJson::Iterator &pjh) { diff --git a/tools/jsonstats.cpp b/tools/jsonstats.cpp index 7ef739daf..93242dcc2 100644 --- a/tools/jsonstats.cpp +++ b/tools/jsonstats.cpp @@ -1,7 +1,6 @@ #include -#include "simdjson/jsonioutil.h" -#include "simdjson/jsonparser.h" +#include "simdjson.h" size_t count_nonasciibytes(const uint8_t *input, size_t length) { size_t count = 0; diff --git a/tools/minify.cpp b/tools/minify.cpp index a2d70ec7b..3cff2abc0 100644 --- a/tools/minify.cpp +++ b/tools/minify.cpp @@ -1,7 +1,6 @@ #include -#include "simdjson/jsonioutil.h" -#include "simdjson/jsonminifier.h" +#include "simdjson.h" int main(int argc, char *argv[]) { if (argc != 2) {