/* auto-generated on Thu Mar 5 10:30:07 PST 2020. Do not edit! */ /* begin file include/simdjson.h */ #ifndef SIMDJSON_H #define SIMDJSON_H /* begin file include/simdjson/compiler_check.h */ #ifndef SIMDJSON_COMPILER_CHECK_H #define SIMDJSON_COMPILER_CHECK_H #ifndef __cplusplus #error simdjson requires a C++ compiler #endif #ifndef SIMDJSON_CPLUSPLUS #if defined(_MSVC_LANG) && !defined(__clang__) #define SIMDJSON_CPLUSPLUS (_MSC_VER == 1900 ? 201103L : _MSVC_LANG) #else #define SIMDJSON_CPLUSPLUS __cplusplus #endif #endif #if (SIMDJSON_CPLUSPLUS < 201703L) #error simdjson requires a compiler compliant with the C++17 standard #endif #endif // SIMDJSON_COMPILER_CHECK_H /* end file include/simdjson/compiler_check.h */ // Public API /* begin file include/simdjson/simdjson_version.h */ // /include/simdjson/simdjson_version.h automatically generated by release.py, // do not change by hand #ifndef SIMDJSON_SIMDJSON_VERSION_H #define SIMDJSON_SIMDJSON_VERSION_H #define SIMDJSON_VERSION 0.2.1 namespace simdjson { enum { SIMDJSON_VERSION_MAJOR = 0, SIMDJSON_VERSION_MINOR = 2, SIMDJSON_VERSION_REVISION = 1 }; } #endif // SIMDJSON_SIMDJSON_VERSION_H /* end file include/simdjson/simdjson_version.h */ /* begin file include/simdjson/error.h */ #ifndef SIMDJSON_ERROR_H #define SIMDJSON_ERROR_H #include namespace simdjson { enum error_code { SUCCESS = 0, SUCCESS_AND_HAS_MORE, //No errors and buffer still has more data CAPACITY, // This parser can't support a document that big MEMALLOC, // Error allocating memory, most likely out of memory TAPE_ERROR, // Something went wrong while writing to the tape (stage 2), this // is a generic error DEPTH_ERROR, // Your document exceeds the user-specified depth limitation STRING_ERROR, // Problem while parsing a string T_ATOM_ERROR, // Problem while parsing an atom starting with the letter 't' F_ATOM_ERROR, // Problem while parsing an atom starting with the letter 'f' N_ATOM_ERROR, // Problem while parsing an atom starting with the letter 'n' NUMBER_ERROR, // Problem while parsing a number UTF8_ERROR, // the input is not valid UTF-8 UNINITIALIZED, // unknown error, or uninitialized document EMPTY, // no structural element found UNESCAPED_CHARS, // found unescaped characters in a string. UNCLOSED_STRING, // missing quote at the end UNSUPPORTED_ARCHITECTURE, // unsupported architecture INCORRECT_TYPE, // JSON element has a different type than user expected NUMBER_OUT_OF_RANGE, // JSON number does not fit in 64 bits NO_SUCH_FIELD, // JSON field not found in object UNEXPECTED_ERROR // indicative of a bug in simdjson }; const std::string &error_message(error_code error) noexcept; struct invalid_json : public std::exception { invalid_json(error_code _error) : error{_error} { } const char *what() const noexcept { return error_message(error).c_str(); } error_code error; }; // TODO these are deprecated, remove using ErrorValues = error_code; inline const std::string &error_message(int error) noexcept { return error_message(error_code(error)); } } // namespace simdjson #endif // SIMDJSON_ERROR_H /* end file include/simdjson/error.h */ /* begin file include/simdjson/padded_string.h */ #ifndef SIMDJSON_PADDING_STRING_H #define SIMDJSON_PADDING_STRING_H /* begin file include/simdjson/portability.h */ #ifndef SIMDJSON_PORTABILITY_H #define SIMDJSON_PORTABILITY_H #include #include #include #ifdef _MSC_VER #include #endif #if defined(__x86_64__) || defined(_M_AMD64) #define IS_X86_64 1 #endif #if defined(__aarch64__) || defined(_M_ARM64) #define IS_ARM64 1 #endif // this is almost standard? #undef STRINGIFY_IMPLEMENTATION_ #undef STRINGIFY #define STRINGIFY_IMPLEMENTATION_(a) #a #define STRINGIFY(a) STRINGIFY_IMPLEMENTATION_(a) // we are going to use runtime dispatch #ifdef IS_X86_64 #ifdef __clang__ // clang does not have GCC push pop // warning: clang attribute push can't be used within a namespace in clang up // til 8.0 so TARGET_REGION and UNTARGET_REGION must be *outside* of a // namespace. #define TARGET_REGION(T) \ _Pragma(STRINGIFY( \ clang attribute push(__attribute__((target(T))), apply_to = function))) #define UNTARGET_REGION _Pragma("clang attribute pop") #elif defined(__GNUC__) // GCC is easier #define TARGET_REGION(T) \ _Pragma("GCC push_options") _Pragma(STRINGIFY(GCC target(T))) #define UNTARGET_REGION _Pragma("GCC pop_options") #endif // clang then gcc #endif // x86 // Default target region macros don't do anything. #ifndef TARGET_REGION #define TARGET_REGION(T) #define UNTARGET_REGION #endif // under GCC and CLANG, we use these two macros #define TARGET_HASWELL TARGET_REGION("avx2,bmi,pclmul,lzcnt") #define TARGET_WESTMERE TARGET_REGION("sse4.2,pclmul") #define TARGET_ARM64 // Threading is disabled #undef SIMDJSON_THREADS_ENABLED // Is threading enabled? #if defined(BOOST_HAS_THREADS) || defined(_REENTRANT) || defined(_MT) #define SIMDJSON_THREADS_ENABLED #endif #if defined(__clang__) #define NO_SANITIZE_UNDEFINED __attribute__((no_sanitize("undefined"))) #elif defined(__GNUC__) #define NO_SANITIZE_UNDEFINED __attribute__((no_sanitize_undefined)) #else #define NO_SANITIZE_UNDEFINED #endif #ifdef _MSC_VER #include // visual studio #endif #ifdef _MSC_VER #define simdjson_strcasecmp _stricmp #else #define simdjson_strcasecmp strcasecmp #endif namespace simdjson { // portable version of posix_memalign static inline void *aligned_malloc(size_t alignment, size_t size) { void *p; #ifdef _MSC_VER p = _aligned_malloc(size, alignment); #elif defined(__MINGW32__) || defined(__MINGW64__) p = __mingw_aligned_malloc(size, alignment); #else // somehow, if this is used before including "x86intrin.h", it creates an // implicit defined warning. if (posix_memalign(&p, alignment, size) != 0) { return nullptr; } #endif return p; } static inline char *aligned_malloc_char(size_t alignment, size_t size) { return (char *)aligned_malloc(alignment, size); } static inline void aligned_free(void *mem_block) { if (mem_block == nullptr) { return; } #ifdef _MSC_VER _aligned_free(mem_block); #elif defined(__MINGW32__) || defined(__MINGW64__) __mingw_aligned_free(mem_block); #else free(mem_block); #endif } static inline void aligned_free_char(char *mem_block) { aligned_free((void *)mem_block); } } // namespace simdjson #endif // SIMDJSON_PORTABILITY_H /* end file include/simdjson/portability.h */ /* begin file include/simdjson/common_defs.h */ #ifndef SIMDJSON_COMMON_DEFS_H #define SIMDJSON_COMMON_DEFS_H #include // we support documents up to 4GB #define SIMDJSON_MAXSIZE_BYTES 0xFFFFFFFF // the input buf should be readable up to buf + SIMDJSON_PADDING // this is a stopgap; there should be a better description of the // main loop and its behavior that abstracts over this // See https://github.com/lemire/simdjson/issues/174 #define SIMDJSON_PADDING 32 #if defined(__GNUC__) // Marks a block with a name so that MCA analysis can see it. #define BEGIN_DEBUG_BLOCK(name) __asm volatile("# LLVM-MCA-BEGIN " #name); #define END_DEBUG_BLOCK(name) __asm volatile("# LLVM-MCA-END " #name); #define DEBUG_BLOCK(name, block) BEGIN_DEBUG_BLOCK(name); block; END_DEBUG_BLOCK(name); #else #define BEGIN_DEBUG_BLOCK(name) #define END_DEBUG_BLOCK(name) #define DEBUG_BLOCK(name, block) #endif #if !defined(_MSC_VER) && !defined(SIMDJSON_NO_COMPUTED_GOTO) // Implemented using Labels as Values which works in GCC and CLANG (and maybe // also in Intel's compiler), but won't work in MSVC. #define SIMDJSON_USE_COMPUTED_GOTO #endif // Align to N-byte boundary #define ROUNDUP_N(a, n) (((a) + ((n)-1)) & ~((n)-1)) #define ROUNDDOWN_N(a, n) ((a) & ~((n)-1)) #define ISALIGNED_N(ptr, n) (((uintptr_t)(ptr) & ((n)-1)) == 0) #ifdef _MSC_VER #define really_inline __forceinline #define never_inline __declspec(noinline) #define UNUSED #define WARN_UNUSED #ifndef likely #define likely(x) x #endif #ifndef unlikely #define unlikely(x) x #endif #else #define really_inline inline __attribute__((always_inline, unused)) #define never_inline inline __attribute__((noinline, unused)) #define UNUSED __attribute__((unused)) #define WARN_UNUSED __attribute__((warn_unused_result)) #ifndef likely #define likely(x) __builtin_expect(!!(x), 1) #endif #ifndef unlikely #define unlikely(x) __builtin_expect(!!(x), 0) #endif #endif // MSC_VER #endif // SIMDJSON_COMMON_DEFS_H /* end file include/simdjson/common_defs.h */ #include #include #include namespace simdjson { // low-level function to allocate memory with padding so we can read past the // "length" bytes safely. if you must provide a pointer to some data, create it // with this function: length is the max. size in bytes of the string caller is // responsible to free the memory (free(...)) inline char *allocate_padded_buffer(size_t length) noexcept { // we could do a simple malloc // return (char *) malloc(length + SIMDJSON_PADDING); // However, we might as well align to cache lines... size_t totalpaddedlength = length + SIMDJSON_PADDING; char *padded_buffer = aligned_malloc_char(64, totalpaddedlength); #ifndef NDEBUG if (padded_buffer == nullptr) { return nullptr; } #endif // NDEBUG memset(padded_buffer + length, 0, totalpaddedlength - length); return padded_buffer; } // allocate_padded_buffer // Simple string with padded allocation. // We deliberately forbid copies, users should rely on swap or move // constructors. struct padded_string final { explicit padded_string() noexcept : viable_size(0), data_ptr(nullptr) {} explicit padded_string(size_t length) noexcept : viable_size(length), data_ptr(allocate_padded_buffer(length)) { if (data_ptr != nullptr) data_ptr[length] = '\0'; // easier when you need a c_str } explicit padded_string(const char *data, size_t length) noexcept : viable_size(length), data_ptr(allocate_padded_buffer(length)) { if ((data != nullptr) and (data_ptr != nullptr)) { memcpy(data_ptr, data, length); data_ptr[length] = '\0'; // easier when you need a c_str } } // note: do not pass std::string arguments by value padded_string(const std::string & str_ ) noexcept : viable_size(str_.size()), data_ptr(allocate_padded_buffer(str_.size())) { if (data_ptr != nullptr) { memcpy(data_ptr, str_.data(), str_.size()); data_ptr[str_.size()] = '\0'; // easier when you need a c_str } } // note: do pass std::string_view arguments by value padded_string(std::string_view sv_) noexcept : viable_size(sv_.size()), data_ptr(allocate_padded_buffer(sv_.size())) { if (data_ptr != nullptr) { memcpy(data_ptr, sv_.data(), sv_.size()); data_ptr[sv_.size()] = '\0'; // easier when you need a c_str } } padded_string(padded_string &&o) noexcept : viable_size(o.viable_size), data_ptr(o.data_ptr) { o.data_ptr = nullptr; // we take ownership } padded_string &operator=(padded_string &&o) { aligned_free_char(data_ptr); data_ptr = o.data_ptr; viable_size = o.viable_size; o.data_ptr = nullptr; // we take ownership o.viable_size = 0; return *this; } void swap(padded_string &o) { size_t tmp_viable_size = viable_size; char *tmp_data_ptr = data_ptr; viable_size = o.viable_size; data_ptr = o.data_ptr; o.data_ptr = tmp_data_ptr; o.viable_size = tmp_viable_size; } ~padded_string() { aligned_free_char(data_ptr); } size_t size() const { return viable_size; } size_t length() const { return viable_size; } char *data() const { return data_ptr; } private: padded_string &operator=(const padded_string &o) = delete; padded_string(const padded_string &o) = delete; size_t viable_size; char *data_ptr{nullptr}; }; // padded_string } // namespace simdjson #endif /* end file include/simdjson/common_defs.h */ /* begin file include/simdjson/implementation.h */ #ifndef SIMDJSON_IMPLEMENTATION_H #define SIMDJSON_IMPLEMENTATION_H #include #include #include #include /* begin file include/simdjson/document.h */ #ifndef SIMDJSON_DOCUMENT_H #define SIMDJSON_DOCUMENT_H #include #include #include #include /* begin file include/simdjson/simdjson.h */ /** * @file * @deprecated We'll be removing this file so it isn't confused with the top level simdjson.h */ #ifndef SIMDJSON_SIMDJSON_H #define SIMDJSON_SIMDJSON_H #endif // SIMDJSON_H /* end file include/simdjson/simdjson.h */ #define JSON_VALUE_MASK 0x00FFFFFFFFFFFFFF #define DEFAULT_MAX_DEPTH 1024 // a JSON document with a depth exceeding 1024 is probably de facto invalid namespace simdjson { template class document_iterator; /** * A parsed JSON document. * * This class cannot be copied, only moved, to avoid unintended allocations. */ class document { public: /** * Create a document container with zero capacity. * * The parser will allocate capacity as needed. */ document() noexcept=default; ~document() noexcept=default; /** * Take another document's buffers. * * @param other The document to take. Its capacity is zeroed and it is invalidated. */ document(document &&other) noexcept = default; document(const document &) = delete; // Disallow copying /** * Take another document's buffers. * * @param other The document to take. Its capacity is zeroed. */ document &operator=(document &&other) noexcept = default; document &operator=(const document &) = delete; // Disallow copying // Nested classes class element; class array; class object; class key_value_pair; class parser; template class element_result; class doc_result; class doc_ref_result; // Nested classes. See definitions later in file. using iterator = document_iterator; /** * Get the root element of this document as a JSON array. */ element root() const noexcept; /** * Get the root element of this document as a JSON array. */ element_result as_array() const noexcept; /** * Get the root element of this document as a JSON object. */ element_result as_object() const noexcept; /** * Get the root element of this document. */ operator element() const noexcept; /** * Read the root element of this document as a JSON array. * * @return The JSON array. * @exception invalid_json(UNEXPECTED_TYPE) if the JSON element is not an array */ operator array() const noexcept(false); /** * Read this element as a JSON object (key/value pairs). * * @return The JSON object. * @exception invalid_json(UNEXPECTED_TYPE) if the JSON element is not an object */ operator object() const noexcept(false); /** * Get the value associated with the given key. * * The key will be matched against **unescaped** JSON: * * document::parse(R"({ "a\n": 1 })")["a\n"].as_uint64_t().value == 1 * document::parse(R"({ "a\n": 1 })")["a\\n"].as_uint64_t().error == NO_SUCH_FIELD * * @return The value associated with the given key, or: * - NO_SUCH_FIELD if the field does not exist in the object * - UNEXPECTED_TYPE if the document is not an object */ element_result operator[](const std::string_view &s) const noexcept; /** * Get the value associated with the given key. * * The key will be matched against **unescaped** JSON: * * document::parse(R"({ "a\n": 1 })")["a\n"].as_uint64_t().value == 1 * document::parse(R"({ "a\n": 1 })")["a\\n"].as_uint64_t().error == NO_SUCH_FIELD * * @return The value associated with this field, or: * - NO_SUCH_FIELD if the field does not exist in the object * - UNEXPECTED_TYPE if the document is not an object */ element_result operator[](const char *s) const noexcept; /** * Print this JSON to a std::ostream. * * @param os the stream to output to. * @param max_depth the maximum JSON depth to output. * @return false if the tape is likely wrong (e.g., you did not parse a valid JSON). */ bool print_json(std::ostream &os, size_t max_depth=DEFAULT_MAX_DEPTH) const noexcept; /** * Dump the raw tape for debugging. * * @param os the stream to output to. * @return false if the tape is likely wrong (e.g., you did not parse a valid JSON). */ bool dump_raw_tape(std::ostream &os) const noexcept; /** * Parse a JSON document and return a reference to it. * * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what * those bytes are initialized to, as long as they are allocated. If realloc_if_needed is true, * it is assumed that the buffer does *not* have enough padding, and it is reallocated, enlarged * and copied before parsing. * * @param buf The JSON to parse. Must have at least len + SIMDJSON_PADDING allocated bytes, unless * realloc_if_needed is true. * @param len The length of the JSON. * @param realloc_if_needed Whether to reallocate and enlarge the JSON buffer to add padding. * @return the document, or an error if the JSON is invalid. */ inline static doc_result parse(const uint8_t *buf, size_t len, bool realloc_if_needed = true) noexcept; /** * Parse a JSON document. * * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what * those bytes are initialized to, as long as they are allocated. If realloc_if_needed is true, * it is assumed that the buffer does *not* have enough padding, and it is reallocated, enlarged * and copied before parsing. * * @param buf The JSON to parse. Must have at least len + SIMDJSON_PADDING allocated bytes, unless * realloc_if_needed is true. * @param len The length of the JSON. * @param realloc_if_needed Whether to reallocate and enlarge the JSON buffer to add padding. * @return the document, or an error if the JSON is invalid. */ really_inline static doc_result parse(const char *buf, size_t len, bool realloc_if_needed = true) noexcept; /** * Parse a JSON document. * * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what * those bytes are initialized to, as long as they are allocated. If `str.capacity() - str.size() * < SIMDJSON_PADDING`, the string will be copied to a string with larger capacity before parsing. * * @param s The JSON to parse. Must have at least len + SIMDJSON_PADDING allocated bytes, or * a new string will be created with the extra padding. * @return the document, or an error if the JSON is invalid. */ really_inline static doc_result parse(const std::string &s) noexcept; /** * Parse a JSON document. * * @param s The JSON to parse. * @return the document, or an error if the JSON is invalid. */ really_inline static doc_result parse(const padded_string &s) noexcept; // We do not want to allow implicit conversion from C string to std::string. doc_ref_result parse(const char *buf, bool realloc_if_needed = true) noexcept = delete; std::unique_ptr tape; std::unique_ptr string_buf;// should be at least byte_capacity private: class tape_ref; enum class tape_type; bool set_capacity(size_t len); }; // class document /** * A parsed, *owned* document, or an error if the parse failed. * * document &doc = document::parse(json); * * Returns an owned `document`. When the doc_result (or the document retrieved from it) goes out of * scope, the document's memory is deallocated. * * ## Error Codes vs. Exceptions * * This result type allows the user to pick whether to use exceptions or not. * * Use like this to avoid exceptions: * * auto [doc, error] = document::parse(json); * if (error) { exit(1); } * * Use like this if you'd prefer to use exceptions: * * document doc = document::parse(json); * */ class document::doc_result { public: /** * The parsed document. This is *invalid* if there is an error. */ document doc; /** * The error code, or SUCCESS (0) if there is no error. */ error_code error; /** * Return the document, or throw an exception if it is invalid. * * @return the document. * @exception invalid_json if the document is invalid or there was an error parsing it. */ operator document() noexcept(false); /** * Get the error message for the error. */ const std::string &get_error_message() const noexcept; ~doc_result() noexcept=default; private: doc_result(document &&_doc, error_code _error) noexcept; doc_result(document &&_doc) noexcept; doc_result(error_code _error) noexcept; friend class document; }; // class document::doc_result /** * A parsed document reference, or an error if the parse failed. * * document &doc = document::parse(json); * * ## Document Ownership * * The `document &` refers to an internal document the parser reuses on each `parse()` call. It will * become invalidated on the next `parse()`. * * This is more efficient for common cases where documents are parsed and used one at a time. If you * need to keep the document around longer, you may *take* it from the parser by casting it: * * document doc = parser.parse(); // take ownership * * If you do this, the parser will automatically allocate a new document on the next `parse()` call. * * ## Error Codes vs. Exceptions * * This result type allows the user to pick whether to use exceptions or not. * * Use like this to avoid exceptions: * * auto [doc, error] = parser.parse(json); * if (error) { exit(1); } * * Use like this if you'd prefer to use exceptions: * * document &doc = document::parse(json); * */ class document::doc_ref_result { public: /** * The parsed document. This is *invalid* if there is an error. */ document &doc; /** * The error code, or SUCCESS (0) if there is no error. */ error_code error; /** * A reference to the document, or throw an exception if it is invalid. * * @return the document. * @exception invalid_json if the document is invalid or there was an error parsing it. */ operator document&() noexcept(false); /** * Get the error message for the error. */ const std::string &get_error_message() const noexcept; ~doc_ref_result()=default; private: doc_ref_result(document &_doc, error_code _error) noexcept; friend class document::parser; }; // class document::doc_ref_result /** * The possible types in the tape. Internal only. */ enum class document::tape_type { ROOT = 'r', START_ARRAY = '[', START_OBJECT = '{', END_ARRAY = ']', END_OBJECT = '}', STRING = '"', INT64 = 'l', UINT64 = 'u', DOUBLE = 'd', TRUE_VALUE = 't', FALSE_VALUE = 'f', NULL_VALUE = 'n' }; /** * A reference to an element on the tape. Internal only. */ class document::tape_ref { protected: really_inline tape_ref() noexcept; really_inline tape_ref(const document *_doc, size_t _json_index) noexcept; inline size_t after_element() const noexcept; really_inline tape_type type() const noexcept; really_inline uint64_t tape_value() const noexcept; template really_inline T next_tape_value() const noexcept; /** The document this element references. */ const document *doc; /** The index of this element on `doc.tape[]` */ size_t json_index; friend class document::key_value_pair; }; /** * A JSON element. * * References an element in a JSON document, representing a JSON null, boolean, string, number, * array or object. */ class document::element : protected document::tape_ref { public: /** Whether this element is a json `null`. */ really_inline bool is_null() const noexcept; /** Whether this is a JSON `true` or `false` */ really_inline bool is_bool() const noexcept; /** Whether this is a JSON number (e.g. 1, 1.0 or 1e2) */ really_inline bool is_number() const noexcept; /** Whether this is a JSON integer (e.g. 1 or -1, but *not* 1.0 or 1e2) */ really_inline bool is_integer() const noexcept; /** Whether this is a JSON string (e.g. "abc") */ really_inline bool is_string() const noexcept; /** Whether this is a JSON array (e.g. []) */ really_inline bool is_array() const noexcept; /** Whether this is a JSON array (e.g. []) */ really_inline bool is_object() const noexcept; /** * Read this element as a boolean (json `true` or `false`). * * @return The boolean value, or: * - UNEXPECTED_TYPE error if the JSON element is not a boolean */ inline element_result as_bool() const noexcept; /** * Read this element as a null-terminated string. * * Does *not* convert other types to a string; requires that the JSON type of the element was * an actual string. * * @return A `string_view` into the string, or: * - UNEXPECTED_TYPE error if the JSON element is not a string */ inline element_result as_c_str() const noexcept; /** * Read this element as a C++ string_view (string with length). * * Does *not* convert other types to a string; requires that the JSON type of the element was * an actual string. * * @return A `string_view` into the string, or: * - UNEXPECTED_TYPE error if the JSON element is not a string */ inline element_result as_string() const noexcept; /** * Read this element as an unsigned integer. * * @return The uninteger value, or: * - UNEXPECTED_TYPE if the JSON element is not an integer * - NUMBER_OUT_OF_RANGE if the integer doesn't fit in 64 bits or is negative */ inline element_result as_uint64_t() const noexcept; /** * Read this element as a signed integer. * * @return The integer value, or: * - UNEXPECTED_TYPE if the JSON element is not an integer * - NUMBER_OUT_OF_RANGE if the integer doesn't fit in 64 bits */ inline element_result as_int64_t() const noexcept; /** * Read this element as a floating point value. * * @return The double value, or: * - UNEXPECTED_TYPE if the JSON element is not a number */ inline element_result as_double() const noexcept; /** * Read this element as a JSON array. * * @return The array value, or: * - UNEXPECTED_TYPE if the JSON element is not an array */ inline element_result as_array() const noexcept; /** * Read this element as a JSON object (key/value pairs). * * @return The object value, or: * - UNEXPECTED_TYPE if the JSON element is not an object */ inline element_result as_object() const noexcept; /** * Read this element as a boolean. * * @return The boolean value * @exception invalid_json(UNEXPECTED_TYPE) if the JSON element is not a boolean. */ inline operator bool() const noexcept(false); /** * Read this element as a null-terminated string. * * Does *not* convert other types to a string; requires that the JSON type of the element was * an actual string. * * @return The string value. * @exception invalid_json(UNEXPECTED_TYPE) if the JSON element is not a string. */ inline explicit operator const char*() const noexcept(false); /** * Read this element as a null-terminated string. * * Does *not* convert other types to a string; requires that the JSON type of the element was * an actual string. * * @return The string value. * @exception invalid_json(UNEXPECTED_TYPE) if the JSON element is not a string. */ inline operator std::string_view() const noexcept(false); /** * Read this element as an unsigned integer. * * @return The integer value. * @exception invalid_json(UNEXPECTED_TYPE) if the JSON element is not an integer * @exception invalid_json(NUMBER_OUT_OF_RANGE) if the integer doesn't fit in 64 bits or is negative */ inline operator uint64_t() const noexcept(false); /** * Read this element as an signed integer. * * @return The integer value. * @exception invalid_json(UNEXPECTED_TYPE) if the JSON element is not an integer * @exception invalid_json(NUMBER_OUT_OF_RANGE) if the integer doesn't fit in 64 bits */ inline operator int64_t() const noexcept(false); /** * Read this element as an double. * * @return The double value. * @exception invalid_json(UNEXPECTED_TYPE) if the JSON element is not a number * @exception invalid_json(NUMBER_OUT_OF_RANGE) if the integer doesn't fit in 64 bits or is negative */ inline operator double() const noexcept(false); /** * Read this element as a JSON array. * * @return The JSON array. * @exception invalid_json(UNEXPECTED_TYPE) if the JSON element is not an array */ inline operator document::array() const noexcept(false); /** * Read this element as a JSON object (key/value pairs). * * @return The JSON object. * @exception invalid_json(UNEXPECTED_TYPE) if the JSON element is not an object */ inline operator document::object() const noexcept(false); /** * Get the value associated with the given key. * * The key will be matched against **unescaped** JSON: * * document::parse(R"({ "a\n": 1 })")["a\n"].as_uint64_t().value == 1 * document::parse(R"({ "a\n": 1 })")["a\\n"].as_uint64_t().error == NO_SUCH_FIELD * * @return The value associated with this field, or: * - NO_SUCH_FIELD if the field does not exist in the object * - UNEXPECTED_TYPE if the document is not an object */ inline element_result operator[](const std::string_view &s) const noexcept; /** * Get the value associated with the given key. * * Note: The key will be matched against **unescaped** JSON: * * document::parse(R"({ "a\n": 1 })")["a\n"].as_uint64_t().value == 1 * document::parse(R"({ "a\n": 1 })")["a\\n"].as_uint64_t().error == NO_SUCH_FIELD * * @return The value associated with this field, or: * - NO_SUCH_FIELD if the field does not exist in the object * - UNEXPECTED_TYPE if the document is not an object */ inline element_result operator[](const char *s) const noexcept; private: really_inline element() noexcept; really_inline element(const document *_doc, size_t _json_index) noexcept; friend class document; template friend class document::element_result; }; /** * Represents a JSON array. */ class document::array : protected document::tape_ref { public: class iterator : tape_ref { public: /** * Get the actual value */ inline element operator*() const noexcept; /** * Get the next value. * * Part of the std::iterator interface. */ inline void operator++() noexcept; /** * Check if these values come from the same place in the JSON. * * Part of the std::iterator interface. */ inline bool operator!=(const iterator& other) const noexcept; private: really_inline iterator(const document *_doc, size_t _json_index) noexcept; friend class array; }; /** * Return the first array element. * * Part of the std::iterable interface. */ inline iterator begin() const noexcept; /** * One past the last array element. * * Part of the std::iterable interface. */ inline iterator end() const noexcept; private: really_inline array() noexcept; really_inline array(const document *_doc, size_t _json_index) noexcept; friend class document::element; template friend class document::element_result; }; /** * Represents a JSON object. */ class document::object : protected document::tape_ref { public: class iterator : protected document::tape_ref { public: /** * Get the actual key/value pair */ inline const document::key_value_pair operator*() const noexcept; /** * Get the next key/value pair. * * Part of the std::iterator interface. */ inline void operator++() noexcept; /** * Check if these key value pairs come from the same place in the JSON. * * Part of the std::iterator interface. */ inline bool operator!=(const iterator& other) const noexcept; /** * Get the key of this key/value pair. */ inline std::string_view key() const noexcept; /** * Get the key of this key/value pair. */ inline const char *key_c_str() const noexcept; /** * Get the value of this key/value pair. */ inline element value() const noexcept; private: really_inline iterator(const document *_doc, size_t _json_index) noexcept; friend class document::object; }; /** * Return the first key/value pair. * * Part of the std::iterable interface. */ inline iterator begin() const noexcept; /** * One past the last key/value pair. * * Part of the std::iterable interface. */ inline iterator end() const noexcept; /** * Get the value associated with the given key. * * The key will be matched against **unescaped** JSON: * * document::parse(R"({ "a\n": 1 })")["a\n"].as_uint64_t().value == 1 * document::parse(R"({ "a\n": 1 })")["a\\n"].as_uint64_t().error == NO_SUCH_FIELD * * @return The value associated with this field, or: * - NO_SUCH_FIELD if the field does not exist in the object */ inline element_result operator[](const std::string_view &s) const noexcept; /** * Get the value associated with the given key. * * Note: The key will be matched against **unescaped** JSON: * * document::parse(R"({ "a\n": 1 })")["a\n"].as_uint64_t().value == 1 * document::parse(R"({ "a\n": 1 })")["a\\n"].as_uint64_t().error == NO_SUCH_FIELD * * @return The value associated with this field, or: * - NO_SUCH_FIELD if the field does not exist in the object */ inline element_result operator[](const char *s) const noexcept; private: really_inline object() noexcept; really_inline object(const document *_doc, size_t _json_index) noexcept; friend class document::element; template friend class document::element_result; }; /** * Key/value pair in an object. */ class document::key_value_pair { public: std::string_view key; document::element value; private: really_inline key_value_pair(std::string_view _key, document::element _value) noexcept; friend class document::object; }; /** * The result of a JSON navigation or conversion, or an error (if the navigation or conversion * failed). Allows the user to pick whether to use exceptions or not. * * Use like this to avoid exceptions: * * auto [str, error] = document::parse(json).root().as_string(); * if (error) { exit(1); } * cout << str; * * Use like this if you'd prefer to use exceptions: * * string str = document::parse(json).root(); * cout << str; * */ template class document::element_result { public: /** The value */ T value; /** The error code (or 0 if there is no error) */ error_code error; inline operator T() const noexcept(false); private: really_inline element_result(T value) noexcept; really_inline element_result(error_code _error) noexcept; friend class document; friend class element; }; // Add exception-throwing navigation / conversion methods to element_result template<> class document::element_result { public: /** The value */ element value; /** The error code (or 0 if there is no error) */ error_code error; /** Whether this is a JSON `null` */ inline element_result is_null() const noexcept; inline element_result as_bool() const noexcept; inline element_result as_string() const noexcept; inline element_result as_c_str() const noexcept; inline element_result as_uint64_t() const noexcept; inline element_result as_int64_t() const noexcept; inline element_result as_double() const noexcept; inline element_result as_array() const noexcept; inline element_result as_object() const noexcept; inline operator bool() const noexcept(false); inline explicit operator const char*() const noexcept(false); inline operator std::string_view() const noexcept(false); inline operator uint64_t() const noexcept(false); inline operator int64_t() const noexcept(false); inline operator double() const noexcept(false); inline operator array() const noexcept(false); inline operator object() const noexcept(false); inline element_result operator[](const std::string_view &s) const noexcept; inline element_result operator[](const char *s) const noexcept; private: really_inline element_result(element value) noexcept; really_inline element_result(error_code _error) noexcept; friend class document; friend class element; }; // Add exception-throwing navigation methods to element_result template<> class document::element_result { public: /** The value */ array value; /** The error code (or 0 if there is no error) */ error_code error; inline operator array() const noexcept(false); inline array::iterator begin() const noexcept(false); inline array::iterator end() const noexcept(false); private: really_inline element_result(array value) noexcept; really_inline element_result(error_code _error) noexcept; friend class document; friend class element; }; // Add exception-throwing navigation methods to element_result template<> class document::element_result { public: /** The value */ object value; /** The error code (or 0 if there is no error) */ error_code error; inline operator object() const noexcept(false); inline object::iterator begin() const noexcept(false); inline object::iterator end() const noexcept(false); inline element_result operator[](const std::string_view &s) const noexcept; inline element_result operator[](const char *s) const noexcept; private: really_inline element_result(object value) noexcept; really_inline element_result(error_code _error) noexcept; friend class document; friend class element; }; /** * A persistent document parser. * * Use this if you intend to parse more than one document. It holds the internal memory necessary * to do parsing, as well as memory for a single document that is overwritten on each parse. * * This class cannot be copied, only moved, to avoid unintended allocations. * * @note This is not thread safe: one parser cannot produce two documents at the same time! */ class document::parser { public: /** * Create a JSON parser with zero capacity. Call allocate_capacity() to initialize it. */ parser()=default; ~parser()=default; /** * Take another parser's buffers and state. * * @param other The parser to take. Its capacity is zeroed. */ parser(document::parser &&other) = default; parser(const document::parser &) = delete; // Disallow copying /** * Take another parser's buffers and state. * * @param other The parser to take. Its capacity is zeroed. */ parser &operator=(document::parser &&other) = default; parser &operator=(const document::parser &) = delete; // Disallow copying /** * Parse a JSON document and return a reference to it. * * The JSON document still lives in the parser: this is the most efficient way to parse JSON * documents because it reuses the same buffers, but you *must* use the document before you * destroy the parser or call parse() again. * * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what * those bytes are initialized to, as long as they are allocated. If realloc_if_needed is true, * it is assumed that the buffer does *not* have enough padding, and it is reallocated, enlarged * and copied before parsing. * * @param buf The JSON to parse. Must have at least len + SIMDJSON_PADDING allocated bytes, unless * realloc_if_needed is true. * @param len The length of the JSON. * @param realloc_if_needed Whether to reallocate and enlarge the JSON buffer to add padding. * @return the document, or an error if the JSON is invalid. */ inline doc_ref_result parse(const uint8_t *buf, size_t len, bool realloc_if_needed = true) noexcept; /** * Parse a JSON document and return a reference to it. * * The JSON document still lives in the parser: this is the most efficient way to parse JSON * documents because it reuses the same buffers, but you *must* use the document before you * destroy the parser or call parse() again. * * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what * those bytes are initialized to, as long as they are allocated. If realloc_if_needed is true, * it is assumed that the buffer does *not* have enough padding, and it is reallocated, enlarged * and copied before parsing. * * @param buf The JSON to parse. Must have at least len + SIMDJSON_PADDING allocated bytes, unless * realloc_if_needed is true. * @param len The length of the JSON. * @param realloc_if_needed Whether to reallocate and enlarge the JSON buffer to add padding. * @return the document, or an error if the JSON is invalid. */ really_inline doc_ref_result parse(const char *buf, size_t len, bool realloc_if_needed = true) noexcept; /** * Parse a JSON document and return a reference to it. * * The JSON document still lives in the parser: this is the most efficient way to parse JSON * documents because it reuses the same buffers, but you *must* use the document before you * destroy the parser or call parse() again. * * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what * those bytes are initialized to, as long as they are allocated. If `str.capacity() - str.size() * < SIMDJSON_PADDING`, the string will be copied to a string with larger capacity before parsing. * * @param s The JSON to parse. Must have at least len + SIMDJSON_PADDING allocated bytes, or * a new string will be created with the extra padding. * @return the document, or an error if the JSON is invalid. */ really_inline doc_ref_result parse(const std::string &s) noexcept; /** * Parse a JSON document and return a reference to it. * * The JSON document still lives in the parser: this is the most efficient way to parse JSON * documents because it reuses the same buffers, but you *must* use the document before you * destroy the parser or call parse() again. * * @param s The JSON to parse. * @return the document, or an error if the JSON is invalid. */ really_inline doc_ref_result parse(const padded_string &s) noexcept; // We do not want to allow implicit conversion from C string to std::string. really_inline doc_ref_result parse(const char *buf) noexcept = delete; /** * Current capacity: the largest document this parser can support without reallocating. */ really_inline size_t capacity() const noexcept; /** * The maximum level of nested object and arrays supported by this parser. */ really_inline size_t max_depth() const noexcept; /** * Ensure this parser has enough memory to process JSON documents up to `capacity` bytes in length * and `max_depth` depth. */ WARN_UNUSED inline bool allocate_capacity(size_t capacity, size_t max_depth = DEFAULT_MAX_DEPTH); // type aliases for backcompat using Iterator = document::iterator; using InvalidJSON = invalid_json; // Next location to write to in the tape uint32_t current_loc{0}; // structural indices passed from stage 1 to stage 2 uint32_t n_structural_indexes{0}; std::unique_ptr structural_indexes; // location and return address of each open { or [ std::unique_ptr containing_scope_offset; #ifdef SIMDJSON_USE_COMPUTED_GOTO std::unique_ptr ret_address; #else std::unique_ptr ret_address; #endif // Next place to write a string uint8_t *current_string_buf_loc; bool valid{false}; error_code error{UNINITIALIZED}; // Document we're writing to document doc; // // TODO these are deprecated; use the results of parse instead. // // returns true if the document parsed was valid inline bool is_valid() const noexcept; // return an error code corresponding to the last parsing attempt, see // simdjson.h will return UNITIALIZED if no parsing was attempted inline int get_error_code() const noexcept; // return the string equivalent of "get_error_code" inline std::string get_error_message() const noexcept; // print the json to std::ostream (should be valid) // return false if the tape is likely wrong (e.g., you did not parse a valid // JSON). inline bool print_json(std::ostream &os) const noexcept; inline bool dump_raw_tape(std::ostream &os) const noexcept; // // Parser callbacks: these are internal! // // TODO find a way to do this without exposing the interface or crippling performance // // this should be called when parsing (right before writing the tapes) inline void init_stage2() noexcept; really_inline error_code on_error(error_code new_error_code) noexcept; really_inline error_code on_success(error_code success_code) noexcept; really_inline bool on_start_document(uint32_t depth) noexcept; really_inline bool on_start_object(uint32_t depth) noexcept; really_inline bool on_start_array(uint32_t depth) noexcept; // TODO we're not checking this bool really_inline bool on_end_document(uint32_t depth) noexcept; really_inline bool on_end_object(uint32_t depth) noexcept; really_inline bool on_end_array(uint32_t depth) noexcept; really_inline bool on_true_atom() noexcept; really_inline bool on_false_atom() noexcept; really_inline bool on_null_atom() noexcept; really_inline uint8_t *on_start_string() noexcept; really_inline bool on_end_string(uint8_t *dst) noexcept; really_inline bool on_number_s64(int64_t value) noexcept; really_inline bool on_number_u64(uint64_t value) noexcept; really_inline bool on_number_double(double value) noexcept; // // Called before a parse is initiated. // // - Returns CAPACITY if the document is too large // - Returns MEMALLOC if we needed to allocate memory and could not // WARN_UNUSED inline error_code init_parse(size_t len) noexcept; private: // // The maximum document length this parser supports. // // Buffers are large enough to handle any document up to this length. // size_t _capacity{0}; // // The maximum depth (number of nested objects and arrays) supported by this parser. // // Defaults to DEFAULT_MAX_DEPTH. // size_t _max_depth{0}; // all nodes are stored on the doc.tape using a 64-bit word. // // strings, double and ints are stored as // a 64-bit word with a pointer to the actual value // // // // for objects or arrays, store [ or { at the beginning and } and ] at the // end. For the openings ([ or {), we annotate them with a reference to the // location on the doc.tape of the end, and for then closings (} and ]), we // annotate them with a reference to the location of the opening // // inline void write_tape(uint64_t val, tape_type t) noexcept; inline void annotate_previous_loc(uint32_t saved_loc, uint64_t val) noexcept; // // Set the current capacity: the largest document this parser can support without reallocating. // // This will allocate *or deallocate* as necessary. // // Returns false if allocation fails. // inline WARN_UNUSED bool set_capacity(size_t capacity); // // Set the maximum level of nested object and arrays supported by this parser. // // This will allocate *or deallocate* as necessary. // // Returns false if allocation fails. // inline WARN_UNUSED bool set_max_depth(size_t max_depth); // Used internally to get the document inline const document &get_document() const noexcept(false); template friend class document_iterator; }; // class parser } // namespace simdjson #endif // SIMDJSON_DOCUMENT_H /* end file include/simdjson/simdjson.h */ namespace simdjson { /** * An implementation of simdjson for a particular CPU architecture. * * Also used to maintain the currently active implementation. The active implementation is * automatically initialized on first use to the most advanced implementation supported by the host. */ class implementation { public: /** * The name of this implementation. * * const implementation *impl = simdjson::active_implementation; * cout << "simdjson is optimized for " << impl->name() << "(" << impl->description() << ")" << endl; * * @return the name of the implementation, e.g. "haswell", "westmere", "arm64" */ virtual const std::string &name() const { return _name; } /** * The description of this implementation. * * const implementation *impl = simdjson::active_implementation; * cout << "simdjson is optimized for " << impl->name() << "(" << impl->description() << ")" << endl; * * @return the name of the implementation, e.g. "haswell", "westmere", "arm64" */ virtual const std::string &description() const { return _description; } /** * The instruction sets this implementation is compiled against. * * @return a mask of all required `instruction_set` values */ virtual uint32_t required_instruction_sets() const { return _required_instruction_sets; }; /** * Run a full document parse (init_parse, stage1 and stage2). * * Overridden by each implementation. * * @param buf the json document to parse. *MUST* be allocated up to len + SIMDJSON_PADDING bytes. * @param len the length of the json document. * @param parser the parser with the buffers to use. *MUST* have allocated up to at least len capacity. * @return the error code, or SUCCESS if there was no error. */ WARN_UNUSED virtual error_code parse(const uint8_t *buf, size_t len, document::parser &parser) const noexcept = 0; /** * Stage 1 of the document parser. * * Overridden by each implementation. * * @param buf the json document to parse. *MUST* be allocated up to len + SIMDJSON_PADDING bytes. * @param len the length of the json document. * @param parser the parser with the buffers to use. *MUST* have allocated up to at least len capacity. * @param streaming whether this is being called by a JsonStream parser. * @return the error code, or SUCCESS if there was no error. */ WARN_UNUSED virtual error_code stage1(const uint8_t *buf, size_t len, document::parser &parser, bool streaming) const noexcept = 0; /** * Stage 2 of the document parser. * * Overridden by each implementation. * * @param buf the json document to parse. *MUST* be allocated up to len + SIMDJSON_PADDING bytes. * @param len the length of the json document. * @param parser the parser with the buffers to use. *MUST* have allocated up to at least len capacity. * @return the error code, or SUCCESS if there was no error. */ WARN_UNUSED virtual error_code stage2(const uint8_t *buf, size_t len, document::parser &parser) const noexcept = 0; /** * Stage 2 of the document parser for JsonStream. * * Overridden by each implementation. * * @param buf the json document to parse. *MUST* be allocated up to len + SIMDJSON_PADDING bytes. * @param len the length of the json document. * @param parser the parser with the buffers to use. *MUST* have allocated up to at least len capacity. * @param next_json the next structural index. Start this at 0 the first time, and it will be updated to the next value to pass each time. * @return the error code, SUCCESS if there was no error, or SUCCESS_AND_HAS_MORE if there was no error and stage2 can be called again. */ WARN_UNUSED virtual error_code stage2(const uint8_t *buf, size_t len, document::parser &parser, size_t &next_json) const noexcept = 0; protected: really_inline implementation( const std::string &name, const std::string &description, uint32_t required_instruction_sets ) : _name(name), _description(description), _required_instruction_sets(required_instruction_sets) { } private: /** * The name of this implementation. */ const std::string _name; /** * The description of this implementation. */ const std::string _description; /** * Instruction sets required for this implementation. */ const uint32_t _required_instruction_sets; }; namespace internal { /** * The list of available implementations compiled into simdjson. */ class available_implementation_list { public: /** Get the list of available implementations compiled into simdjson */ really_inline available_implementation_list() {} /** Number of implementations */ size_t size() const noexcept; /** STL const begin() iterator */ const implementation * const *begin() const noexcept; /** STL const end() iterator */ const implementation * const *end() const noexcept; /** * Get the implementation with the given name. * * Case sensitive. * * const implementation *impl = simdjson::available_implementations["westmere"]; * if (!impl) { exit(1); } * simdjson::active_implementation = impl; * * @param name the implementation to find, e.g. "westmere", "haswell", "arm64" * @return the implementation, or nullptr if the parse failed. */ const implementation * operator[](const std::string& name) const noexcept { for (const implementation * impl : *this) { if (impl->name() == name) { return impl; } } return nullptr; } /** * Detect the most advanced implementation supported by the current host. * * This is used to initialize the implementation on startup. * * const implementation *impl = simdjson::available_implementation::detect_best_supported(); * simdjson::active_implementation = impl; * * @return the most advanced supported implementation for the current host, or an * implementation that returns UNSUPPORTED_ARCHITECTURE if there is no supported * implementation. Will never return nullptr. */ const implementation *detect_best_supported() const noexcept; }; // Detects best supported implementation on first use, and sets it class detect_best_supported_implementation_on_first_use final : public implementation { public: const std::string& name() const noexcept final { return set_best()->name(); } const std::string& description() const noexcept final { return set_best()->description(); } uint32_t required_instruction_sets() const noexcept final { return set_best()->required_instruction_sets(); } WARN_UNUSED error_code parse(const uint8_t *buf, size_t len, document::parser &parser) const noexcept final { return set_best()->parse(buf, len, parser); } WARN_UNUSED error_code stage1(const uint8_t *buf, size_t len, document::parser &parser, bool streaming) const noexcept final { return set_best()->stage1(buf, len, parser, streaming); } WARN_UNUSED error_code stage2(const uint8_t *buf, size_t len, document::parser &parser) const noexcept final { return set_best()->stage2(buf, len, parser); } WARN_UNUSED error_code stage2(const uint8_t *buf, size_t len, document::parser &parser, size_t &next_json) const noexcept final { return set_best()->stage2(buf, len, parser, next_json); } really_inline detect_best_supported_implementation_on_first_use() noexcept : implementation("best_supported_detector", "Detects the best supported implementation and sets it", 0) {} private: const implementation *set_best() const noexcept; }; inline const detect_best_supported_implementation_on_first_use detect_best_supported_implementation_on_first_use_singleton; template class atomic_ptr { public: atomic_ptr(T *_ptr) : ptr{_ptr} {} operator const T*() const { return ptr.load(); } const T& operator*() const { return *ptr; } const T* operator->() const { return ptr.load(); } operator T*() { return ptr.load(); } T& operator*() { return *ptr; } T* operator->() { return ptr.load(); } T* operator=(T *_ptr) { return ptr = _ptr; } private: std::atomic ptr; }; } // namespace [simdjson::]internal /** * The list of available implementations compiled into simdjson. */ inline const internal::available_implementation_list available_implementations; /** * The active implementation. * * Automatically initialized on first use to the most advanced implementation supported by this hardware. */ inline internal::atomic_ptr active_implementation = &internal::detect_best_supported_implementation_on_first_use_singleton; } // namespace simdjson #endif // SIMDJSON_IMPLEMENTATION_H /* end file include/simdjson/simdjson.h */ /* begin file include/simdjson/jsonstream.h */ #ifndef SIMDJSON_JSONSTREAM_H #define SIMDJSON_JSONSTREAM_H #include namespace simdjson { /************************************************************************************* * The main motivation for this piece of software is to achieve maximum speed *and offer * good quality of life while parsing files containing multiple JSON documents. * * Since we want to offer flexibility and not restrict ourselves to a specific *file * format, we support any file that contains any valid JSON documents separated *by one * or more character that is considered a whitespace by the JSON spec. * Namely: space, nothing, linefeed, carriage return, horizontal tab. * Anything that is not whitespace will be parsed as a JSON document and could *lead * to failure. * * To offer maximum parsing speed, our implementation processes the data inside *the * buffer by batches and their size is defined by the parameter "batch_size". * By loading data in batches, we can optimize the time spent allocating data in *the * parser and can also open the possibility of multi-threading. * The batch_size must be at least as large as the biggest document in the file, *but * not too large in order to submerge the chached memory. We found that 1MB is * somewhat a sweet spot for now. Eventually, this batch_size could be fully * automated and be optimal at all times. ************************************************************************************/ /** * The template parameter (string_container) must * support the data() and size() methods, returning a pointer * to a char* and to the number of bytes respectively. * The simdjson parser may read up to SIMDJSON_PADDING bytes beyond the end * of the string, so if you do not use a padded_string container, * you have the responsability to overallocated. If you fail to * do so, your software may crash if you cross a page boundary, * and you should expect memory checkers to object. * Most users should use a simdjson::padded_string. */ template class JsonStream { public: /* Create a JsonStream object that can be used to parse sequentially the valid * JSON documents found in the buffer "buf". * * The batch_size must be at least as large as the biggest document in the * file, but * not too large to submerge the cached memory. We found that 1MB is * somewhat a sweet spot for now. * * The user is expected to call the following json_parse method to parse the * next * valid JSON document found in the buffer. This method can and is expected * to be * called in a loop. * * Various methods are offered to keep track of the status, like * get_current_buffer_loc, * get_n_parsed_docs, get_n_bytes_parsed, etc. * * */ JsonStream(const string_container &s, size_t batch_size = 1000000); ~JsonStream(); /* Parse the next document found in the buffer previously given to JsonStream. * The content should be a valid JSON document encoded as UTF-8. If there is a * UTF-8 BOM, the caller is responsible for omitting it, UTF-8 BOM are * discouraged. * * You do NOT need to pre-allocate a parser. This function takes care of * pre-allocating a capacity defined by the batch_size defined when creating the * JsonStream object. * * The function returns simdjson::SUCCESS_AND_HAS_MORE (an integer = 1) in case * of success and indicates that the buffer still contains more data to be parsed, * meaning this function can be called again to return the next JSON document * after this one. * * The function returns simdjson::SUCCESS (as integer = 0) in case of success * and indicates that the buffer has successfully been parsed to the end. * Every document it contained has been parsed without error. * * The function returns an error code from simdjson/simdjson.h in case of failure * such as simdjson::CAPACITY, simdjson::MEMALLOC, simdjson::DEPTH_ERROR and so forth; * the simdjson::error_message function converts these error codes into a * string). * * You can also check validity by calling parser.is_valid(). The same parser can * and should be reused for the other documents in the buffer. */ int json_parse(document::parser &parser); /* Returns the location (index) of where the next document should be in the * buffer. * Can be used for debugging, it tells the user the position of the end of the * last * valid JSON document parsed*/ inline size_t get_current_buffer_loc() const { return current_buffer_loc; } /* Returns the total amount of complete documents parsed by the JsonStream, * in the current buffer, at the given time.*/ inline size_t get_n_parsed_docs() const { return n_parsed_docs; } /* Returns the total amount of data (in bytes) parsed by the JsonStream, * in the current buffer, at the given time.*/ inline size_t get_n_bytes_parsed() const { return n_bytes_parsed; } private: inline const uint8_t *buf() const { return reinterpret_cast(str.data()) + str_start; } inline void advance(size_t offset) { str_start += offset; } inline size_t remaining() const { return str.size() - str_start; } const string_container &str; size_t _batch_size; // this is actually variable! size_t str_start{0}; size_t next_json{0}; bool load_next_batch{true}; size_t current_buffer_loc{0}; #ifdef SIMDJSON_THREADS_ENABLED size_t last_json_buffer_loc{0}; #endif size_t n_parsed_docs{0}; size_t n_bytes_parsed{0}; simdjson::implementation *stage_parser; #ifdef SIMDJSON_THREADS_ENABLED error_code stage1_is_ok_thread{SUCCESS}; std::thread stage_1_thread; document::parser parser_thread; #endif }; // end of class JsonStream } // end of namespace simdjson #endif // SIMDJSON_JSONSTREAM_H /* end file include/simdjson/jsonstream.h */ /* begin file include/simdjson/jsonminifier.h */ #ifndef SIMDJSON_JSONMINIFIER_H #define SIMDJSON_JSONMINIFIER_H #include #include #include namespace simdjson { // Take input from buf and remove useless whitespace, write it to out; buf and // out can be the same pointer. Result is null terminated, // return the string length (minus the null termination). // The accelerated version of this function only runs on AVX2 hardware. size_t json_minify(const uint8_t *buf, size_t len, uint8_t *out); static inline size_t json_minify(const char *buf, size_t len, char *out) { return json_minify(reinterpret_cast(buf), len, reinterpret_cast(out)); } static inline size_t json_minify(const std::string_view &p, char *out) { return json_minify(p.data(), p.size(), out); } static inline size_t json_minify(const padded_string &p, char *out) { return json_minify(p.data(), p.size(), out); } } // namespace simdjson #endif /* end file include/simdjson/jsonminifier.h */ // Deprecated API /* begin file include/simdjson/parsedjsoniterator.h */ // TODO Remove this -- deprecated API and files #ifndef SIMDJSON_PARSEDJSONITERATOR_H #define SIMDJSON_PARSEDJSONITERATOR_H /* begin file include/simdjson/document_iterator.h */ #ifndef SIMDJSON_DOCUMENT_ITERATOR_H #define SIMDJSON_DOCUMENT_ITERATOR_H #include #include #include #include #include #include /* begin file include/simdjson/internal/jsonformatutils.h */ #ifndef SIMDJSON_INTERNAL_JSONFORMATUTILS_H #define SIMDJSON_INTERNAL_JSONFORMATUTILS_H #include #include namespace simdjson::internal { // ends with zero char static inline void print_with_escapes(const unsigned char *src, std::ostream &os) { while (*src) { switch (*src) { case '\b': os << '\\'; os << 'b'; break; case '\f': os << '\\'; os << 'f'; break; case '\n': os << '\\'; os << 'n'; break; case '\r': os << '\\'; os << 'r'; break; case '\"': os << '\\'; os << '"'; break; case '\t': os << '\\'; os << 't'; break; case '\\': os << '\\'; os << '\\'; break; default: if (*src <= 0x1F) { std::ios::fmtflags f(os.flags()); os << std::hex << std::setw(4) << std::setfill('0') << static_cast(*src); os.flags(f); } else { os << *src; } } src++; } } // print len chars static inline void print_with_escapes(const unsigned char *src, std::ostream &os, size_t len) { const unsigned char *finalsrc = src + len; while (src < finalsrc) { switch (*src) { case '\b': os << '\\'; os << 'b'; break; case '\f': os << '\\'; os << 'f'; break; case '\n': os << '\\'; os << 'n'; break; case '\r': os << '\\'; os << 'r'; break; case '\"': os << '\\'; os << '"'; break; case '\t': os << '\\'; os << 't'; break; case '\\': os << '\\'; os << '\\'; break; default: if (*src <= 0x1F) { std::ios::fmtflags f(os.flags()); os << std::hex << std::setw(4) << std::setfill('0') << static_cast(*src); os.flags(f); } else { os << *src; } } src++; } } static inline void print_with_escapes(const char *src, std::ostream &os) { print_with_escapes(reinterpret_cast(src), os); } static inline void print_with_escapes(const char *src, std::ostream &os, size_t len) { print_with_escapes(reinterpret_cast(src), os, len); } } // namespace simdjson::internal #endif // SIMDJSON_INTERNAL_JSONFORMATUTILS_H /* end file include/simdjson/internal/jsonformatutils.h */ namespace simdjson { template class document_iterator { public: document_iterator(const document::parser &parser); document_iterator(const document &doc) noexcept; document_iterator(const document_iterator &o) noexcept; document_iterator &operator=(const document_iterator &o) noexcept; inline bool is_ok() const; // useful for debuging purposes inline size_t get_tape_location() const; // useful for debuging purposes inline size_t get_tape_length() const; // returns the current depth (start at 1 with 0 reserved for the fictitious // root node) inline size_t get_depth() const; // A scope is a series of nodes at the same depth, typically it is either an // object ({) or an array ([). The root node has type 'r'. inline uint8_t get_scope_type() const; // move forward in document order inline bool move_forward(); // retrieve the character code of what we're looking at: // [{"slutfn are the possibilities inline uint8_t get_type() const { return current_type; // short functions should be inlined! } // get the int64_t value at this node; valid only if get_type is "l" inline int64_t get_integer() const { if (location + 1 >= tape_length) { return 0; // default value in case of error } return static_cast(doc.tape[location + 1]); } // get the value as uint64; valid only if if get_type is "u" inline uint64_t get_unsigned_integer() const { if (location + 1 >= tape_length) { return 0; // default value in case of error } return doc.tape[location + 1]; } // get the string value at this node (NULL ended); valid only if get_type is " // note that tabs, and line endings are escaped in the returned value (see // print_with_escapes) return value is valid UTF-8, it may contain NULL chars // within the string: get_string_length determines the true string length. inline const char *get_string() const { return reinterpret_cast( doc.string_buf.get() + (current_val & JSON_VALUE_MASK) + sizeof(uint32_t)); } // return the length of the string in bytes inline uint32_t get_string_length() const { uint32_t answer; memcpy(&answer, reinterpret_cast(doc.string_buf.get() + (current_val & JSON_VALUE_MASK)), sizeof(uint32_t)); return answer; } // get the double value at this node; valid only if // get_type() is "d" inline double get_double() const { if (location + 1 >= tape_length) { return std::numeric_limits::quiet_NaN(); // default value in // case of error } double answer; memcpy(&answer, &doc.tape[location + 1], sizeof(answer)); return answer; } inline bool is_object_or_array() const { return is_object() || is_array(); } inline bool is_object() const { return get_type() == '{'; } inline bool is_array() const { return get_type() == '['; } inline bool is_string() const { return get_type() == '"'; } // Returns true if the current type of node is an signed integer. // You can get its value with `get_integer()`. inline bool is_integer() const { return get_type() == 'l'; } // Returns true if the current type of node is an unsigned integer. // You can get its value with `get_unsigned_integer()`. // // NOTE: // Only a large value, which is out of range of a 64-bit signed integer, is // represented internally as an unsigned node. On the other hand, a typical // positive integer, such as 1, 42, or 1000000, is as a signed node. // Be aware this function returns false for a signed node. inline bool is_unsigned_integer() const { return get_type() == 'u'; } inline bool is_double() const { return get_type() == 'd'; } inline bool is_number() const { return is_integer() || is_unsigned_integer() || is_double(); } inline bool is_true() const { return get_type() == 't'; } inline bool is_false() const { return get_type() == 'f'; } inline bool is_null() const { return get_type() == 'n'; } static bool is_object_or_array(uint8_t type) { return ((type == '[') || (type == '{')); } // when at {, go one level deep, looking for a given key // if successful, we are left pointing at the value, // if not, we are still pointing at the object ({) // (in case of repeated keys, this only finds the first one). // We seek the key using C's strcmp so if your JSON strings contain // NULL chars, this would trigger a false positive: if you expect that // to be the case, take extra precautions. // Furthermore, we do the comparison character-by-character // without taking into account Unicode equivalence. inline bool move_to_key(const char *key); // as above, but case insensitive lookup (strcmpi instead of strcmp) inline bool move_to_key_insensitive(const char *key); // when at {, go one level deep, looking for a given key // if successful, we are left pointing at the value, // if not, we are still pointing at the object ({) // (in case of repeated keys, this only finds the first one). // The string we search for can contain NULL values. // Furthermore, we do the comparison character-by-character // without taking into account Unicode equivalence. inline bool move_to_key(const char *key, uint32_t length); // when at a key location within an object, this moves to the accompanying // value (located next to it). This is equivalent but much faster than // calling "next()". inline void move_to_value(); // when at [, go one level deep, and advance to the given index. // if successful, we are left pointing at the value, // if not, we are still pointing at the array ([) inline bool move_to_index(uint32_t index); // Moves the iterator to the value correspoding to the json pointer. // Always search from the root of the document. // if successful, we are left pointing at the value, // if not, we are still pointing the same value we were pointing before the // call. The json pointer follows the rfc6901 standard's syntax: // https://tools.ietf.org/html/rfc6901 However, the standard says "If a // referenced member name is not unique in an object, the member that is // referenced is undefined, and evaluation fails". Here we just return the // first corresponding value. The length parameter is the length of the // jsonpointer string ('pointer'). bool move_to(const char *pointer, uint32_t length); // Moves the iterator to the value correspoding to the json pointer. // Always search from the root of the document. // if successful, we are left pointing at the value, // if not, we are still pointing the same value we were pointing before the // call. The json pointer implementation follows the rfc6901 standard's // syntax: https://tools.ietf.org/html/rfc6901 However, the standard says // "If a referenced member name is not unique in an object, the member that // is referenced is undefined, and evaluation fails". Here we just return // the first corresponding value. inline bool move_to(const std::string &pointer) { return move_to(pointer.c_str(), pointer.length()); } private: // Almost the same as move_to(), except it searchs from the current // position. The pointer's syntax is identical, though that case is not // handled by the rfc6901 standard. The '/' is still required at the // beginning. However, contrary to move_to(), the URI Fragment Identifier // Representation is not supported here. Also, in case of failure, we are // left pointing at the closest value it could reach. For these reasons it // is private. It exists because it is used by move_to(). bool relative_move_to(const char *pointer, uint32_t length); public: // throughout return true if we can do the navigation, false // otherwise // Withing a given scope (series of nodes at the same depth within either an // array or an object), we move forward. // Thus, given [true, null, {"a":1}, [1,2]], we would visit true, null, { // and [. At the object ({) or at the array ([), you can issue a "down" to // visit their content. valid if we're not at the end of a scope (returns // true). inline bool next(); // Within a given scope (series of nodes at the same depth within either an // array or an object), we move backward. // Thus, given [true, null, {"a":1}, [1,2]], we would visit ], }, null, true // when starting at the end of the scope. At the object ({) or at the array // ([), you can issue a "down" to visit their content. // Performance warning: This function is implemented by starting again // from the beginning of the scope and scanning forward. You should expect // it to be relatively slow. inline bool prev(); // Moves back to either the containing array or object (type { or [) from // within a contained scope. // Valid unless we are at the first level of the document inline bool up(); // Valid if we're at a [ or { and it starts a non-empty scope; moves us to // start of that deeper scope if it not empty. Thus, given [true, null, // {"a":1}, [1,2]], if we are at the { node, we would move to the "a" node. inline bool down(); // move us to the start of our current scope, // a scope is a series of nodes at the same level inline void to_start_scope(); inline void rewind() { while (up()) ; } // void to_end_scope(); // move us to // the start of our current scope; always succeeds // print the node we are currently pointing at bool print(std::ostream &os, bool escape_strings = true) const; typedef struct { size_t start_of_scope; uint8_t scope_type; } scopeindex_t; private: const document &doc; size_t depth; size_t location; // our current location on a tape size_t tape_length; uint8_t current_type; uint64_t current_val; scopeindex_t depth_index[max_depth]; }; } // namespace simdjson #endif // SIMDJSON_DOCUMENT_ITERATOR_H /* end file include/simdjson/internal/jsonformatutils.h */ #endif /* end file include/simdjson/internal/jsonformatutils.h */ /* begin file include/simdjson/jsonparser.h */ // TODO Remove this -- deprecated API and files #ifndef SIMDJSON_JSONPARSER_H #define SIMDJSON_JSONPARSER_H /* begin file include/simdjson/parsedjson.h */ // TODO Remove this -- deprecated API and files #ifndef SIMDJSON_PARSEDJSON_H #define SIMDJSON_PARSEDJSON_H namespace simdjson { using ParsedJson = document::parser; } // namespace simdjson #endif /* end file include/simdjson/parsedjson.h */ /* begin file include/simdjson/jsonioutil.h */ #ifndef SIMDJSON_JSONIOUTIL_H #define SIMDJSON_JSONIOUTIL_H #include #include #include #include #include #include namespace simdjson { // load a file in memory... // get a corpus; pad out to cache line so we can always use SIMD // throws exceptions in case of failure // first element of the pair is a string (null terminated) // whereas the second element is the length. // caller is responsible to free (aligned_free((void*)result.data()))) // // throws an exception if the file cannot be opened, use try/catch // try { // p = get_corpus(filename); // } catch (const std::exception& e) { // aligned_free((void*)p.data()); // std::cout << "Could not load the file " << filename << std::endl; // } padded_string get_corpus(const std::string &filename); } // namespace simdjson #endif /* end file include/simdjson/jsonioutil.h */ namespace simdjson { // // C API (json_parse and build_parsed_json) declarations // inline int json_parse(const uint8_t *buf, size_t len, document::parser &parser, bool realloc_if_needed = true) noexcept { error_code code = parser.parse(buf, len, realloc_if_needed).error; // The deprecated json_parse API is a signal that the user plans to *use* the error code / valid // bits in the parser instead of heeding the result code. The normal parser unsets those in // anticipation of making the error code ephemeral. // Here we put the code back into the parser, until we've removed this method. parser.valid = code == SUCCESS; parser.error = code; return code; } inline int json_parse(const char *buf, size_t len, document::parser &parser, bool realloc_if_needed = true) noexcept { return json_parse(reinterpret_cast(buf), len, parser, realloc_if_needed); } inline int json_parse(const std::string &s, document::parser &parser, bool realloc_if_needed = true) noexcept { return json_parse(s.data(), s.length(), parser, realloc_if_needed); } inline int json_parse(const padded_string &s, document::parser &parser) noexcept { return json_parse(s.data(), s.length(), parser, false); } WARN_UNUSED static document::parser build_parsed_json(const uint8_t *buf, size_t len, bool realloc_if_needed = true) noexcept { document::parser parser; if (!parser.allocate_capacity(len)) { parser.valid = false; parser.error = MEMALLOC; return parser; } json_parse(buf, len, parser, realloc_if_needed); return parser; } WARN_UNUSED inline document::parser build_parsed_json(const char *buf, size_t len, bool realloc_if_needed = true) noexcept { return build_parsed_json(reinterpret_cast(buf), len, realloc_if_needed); } WARN_UNUSED inline document::parser build_parsed_json(const std::string &s, bool realloc_if_needed = true) noexcept { return build_parsed_json(s.data(), s.length(), realloc_if_needed); } WARN_UNUSED inline document::parser build_parsed_json(const padded_string &s) noexcept { return build_parsed_json(s.data(), s.length(), false); } // We do not want to allow implicit conversion from C string to std::string. int json_parse(const char *buf, document::parser &parser) noexcept = delete; document::parser build_parsed_json(const char *buf) noexcept = delete; } // namespace simdjson #endif /* end file include/simdjson/jsonioutil.h */ // Inline functions /* begin file include/simdjson/inline/document.h */ #ifndef SIMDJSON_INLINE_DOCUMENT_H #define SIMDJSON_INLINE_DOCUMENT_H // Inline implementations go in here. #include namespace simdjson { // // document::element_result inline implementation // template inline document::element_result::element_result(T _value) noexcept : value(_value), error{SUCCESS} {} template inline document::element_result::element_result(error_code _error) noexcept : value(), error{_error} {} template<> inline document::element_result::operator std::string_view() const noexcept(false) { if (error) { throw invalid_json(error); } return value; } template<> inline document::element_result::operator const char *() const noexcept(false) { if (error) { throw invalid_json(error); } return value; } template<> inline document::element_result::operator bool() const noexcept(false) { if (error) { throw invalid_json(error); } return value; } template<> inline document::element_result::operator uint64_t() const noexcept(false) { if (error) { throw invalid_json(error); } return value; } template<> inline document::element_result::operator int64_t() const noexcept(false) { if (error) { throw invalid_json(error); } return value; } template<> inline document::element_result::operator double() const noexcept(false) { if (error) { throw invalid_json(error); } return value; } // // document::element_result inline implementation // inline document::element_result::element_result(document::array _value) noexcept : value(_value), error{SUCCESS} {} inline document::element_result::element_result(error_code _error) noexcept : value(), error{_error} {} inline document::element_result::operator document::array() const noexcept(false) { if (error) { throw invalid_json(error); } return value; } inline document::array::iterator document::element_result::begin() const noexcept(false) { if (error) { throw invalid_json(error); } return value.begin(); } inline document::array::iterator document::element_result::end() const noexcept(false) { if (error) { throw invalid_json(error); } return value.end(); } // // document::element_result inline implementation // inline document::element_result::element_result(document::object _value) noexcept : value(_value), error{SUCCESS} {} inline document::element_result::element_result(error_code _error) noexcept : value(), error{_error} {} inline document::element_result::operator document::object() const noexcept(false) { if (error) { throw invalid_json(error); } return value; } inline document::element_result document::element_result::operator[](const std::string_view &key) const noexcept { if (error) { return error; } return value[key]; } inline document::element_result document::element_result::operator[](const char *key) const noexcept { if (error) { return error; } return value[key]; } inline document::object::iterator document::element_result::begin() const noexcept(false) { if (error) { throw invalid_json(error); } return value.begin(); } inline document::object::iterator document::element_result::end() const noexcept(false) { if (error) { throw invalid_json(error); } return value.end(); } // // document::element_result inline implementation // inline document::element_result::element_result(document::element _value) noexcept : value(_value), error{SUCCESS} {} inline document::element_result::element_result(error_code _error) noexcept : value(), error{_error} {} inline document::element_result document::element_result::is_null() const noexcept { if (error) { return error; } return value.is_null(); } inline document::element_result document::element_result::as_bool() const noexcept { if (error) { return error; } return value.as_bool(); } inline document::element_result document::element_result::as_c_str() const noexcept { if (error) { return error; } return value.as_c_str(); } inline document::element_result document::element_result::as_string() const noexcept { if (error) { return error; } return value.as_string(); } inline document::element_result document::element_result::as_uint64_t() const noexcept { if (error) { return error; } return value.as_uint64_t(); } inline document::element_result document::element_result::as_int64_t() const noexcept { if (error) { return error; } return value.as_int64_t(); } inline document::element_result document::element_result::as_double() const noexcept { if (error) { return error; } return value.as_double(); } inline document::element_result document::element_result::as_array() const noexcept { if (error) { return error; } return value.as_array(); } inline document::element_result document::element_result::as_object() const noexcept { if (error) { return error; } return value.as_object(); } inline document::element_result::operator bool() const noexcept(false) { return as_bool(); } inline document::element_result::operator const char *() const noexcept(false) { return as_c_str(); } inline document::element_result::operator std::string_view() const noexcept(false) { return as_string(); } inline document::element_result::operator uint64_t() const noexcept(false) { return as_uint64_t(); } inline document::element_result::operator int64_t() const noexcept(false) { return as_int64_t(); } inline document::element_result::operator double() const noexcept(false) { return as_double(); } inline document::element_result::operator document::array() const noexcept(false) { return as_array(); } inline document::element_result::operator document::object() const noexcept(false) { return as_object(); } inline document::element_result document::element_result::operator[](const std::string_view &key) const noexcept { if (error) { return *this; } return value[key]; } inline document::element_result document::element_result::operator[](const char *key) const noexcept { if (error) { return *this; } return value[key]; } // // document inline implementation // inline document::element document::root() const noexcept { return document::element(this, 1); } inline document::element_result document::as_array() const noexcept { return root().as_array(); } inline document::element_result document::as_object() const noexcept { return root().as_object(); } inline document::operator document::element() const noexcept { return root(); } inline document::operator document::array() const noexcept(false) { return root(); } inline document::operator document::object() const noexcept(false) { return root(); } inline document::element_result document::operator[](const std::string_view &key) const noexcept { return root()[key]; } inline document::element_result document::operator[](const char *key) const noexcept { return root()[key]; } inline document::doc_result document::parse(const uint8_t *buf, size_t len, bool realloc_if_needed) noexcept { document::parser parser; if (!parser.allocate_capacity(len)) { return MEMALLOC; } auto [doc, error] = parser.parse(buf, len, realloc_if_needed); return document::doc_result((document &&)doc, error); } really_inline document::doc_result document::parse(const char *buf, size_t len, bool realloc_if_needed) noexcept { return parse((const uint8_t *)buf, len, realloc_if_needed); } really_inline document::doc_result document::parse(const std::string &s) noexcept { return parse(s.data(), s.length(), s.capacity() - s.length() < SIMDJSON_PADDING); } really_inline document::doc_result document::parse(const padded_string &s) noexcept { return parse(s.data(), s.length(), false); } WARN_UNUSED inline bool document::set_capacity(size_t capacity) { if (capacity == 0) { string_buf.reset(); tape.reset(); return true; } // a pathological input like "[[[[..." would generate len tape elements, so // need a capacity of at least len + 1, but it is also possible to do // worse with "[7,7,7,7,6,7,7,7,6,7,7,6,[7,7,7,7,6,7,7,7,6,7,7,6,7,7,7,7,7,7,6" //where len + 1 tape elements are // generated, see issue https://github.com/lemire/simdjson/issues/345 size_t tape_capacity = ROUNDUP_N(capacity + 2, 64); // a document with only zero-length strings... could have len/3 string // and we would need len/3 * 5 bytes on the string buffer size_t string_capacity = ROUNDUP_N(5 * capacity / 3 + 32, 64); string_buf.reset( new (std::nothrow) uint8_t[string_capacity]); tape.reset(new (std::nothrow) uint64_t[tape_capacity]); return string_buf && tape; } inline bool document::print_json(std::ostream &os, size_t max_depth) const noexcept { uint32_t string_length; size_t tape_idx = 0; uint64_t tape_val = tape[tape_idx]; uint8_t type = (tape_val >> 56); size_t how_many = 0; if (type == 'r') { how_many = tape_val & JSON_VALUE_MASK; } else { // Error: no starting root node? return false; } tape_idx++; std::unique_ptr in_object(new bool[max_depth]); std::unique_ptr in_object_idx(new size_t[max_depth]); int depth = 1; // only root at level 0 in_object_idx[depth] = 0; in_object[depth] = false; for (; tape_idx < how_many; tape_idx++) { tape_val = tape[tape_idx]; uint64_t payload = tape_val & JSON_VALUE_MASK; type = (tape_val >> 56); if (!in_object[depth]) { if ((in_object_idx[depth] > 0) && (type != ']')) { os << ","; } in_object_idx[depth]++; } else { // if (in_object) { if ((in_object_idx[depth] > 0) && ((in_object_idx[depth] & 1) == 0) && (type != '}')) { os << ","; } if (((in_object_idx[depth] & 1) == 1)) { os << ":"; } in_object_idx[depth]++; } switch (type) { case '"': // we have a string os << '"'; memcpy(&string_length, string_buf.get() + payload, sizeof(uint32_t)); internal::print_with_escapes( (const unsigned char *)(string_buf.get() + payload + sizeof(uint32_t)), os, string_length); os << '"'; break; case 'l': // we have a long int if (tape_idx + 1 >= how_many) { return false; } os << static_cast(tape[++tape_idx]); break; case 'u': if (tape_idx + 1 >= how_many) { return false; } os << tape[++tape_idx]; break; case 'd': // we have a double if (tape_idx + 1 >= how_many) { return false; } double answer; memcpy(&answer, &tape[++tape_idx], sizeof(answer)); os << answer; break; case 'n': // we have a null os << "null"; break; case 't': // we have a true os << "true"; break; case 'f': // we have a false os << "false"; break; case '{': // we have an object os << '{'; depth++; in_object[depth] = true; in_object_idx[depth] = 0; break; case '}': // we end an object depth--; os << '}'; break; case '[': // we start an array os << '['; depth++; in_object[depth] = false; in_object_idx[depth] = 0; break; case ']': // we end an array depth--; os << ']'; break; case 'r': // we start and end with the root node // should we be hitting the root node? return false; default: // bug? return false; } } return true; } inline bool document::dump_raw_tape(std::ostream &os) const noexcept { uint32_t string_length; size_t tape_idx = 0; uint64_t tape_val = tape[tape_idx]; uint8_t type = (tape_val >> 56); os << tape_idx << " : " << type; tape_idx++; size_t how_many = 0; if (type == 'r') { how_many = tape_val & JSON_VALUE_MASK; } else { // Error: no starting root node? return false; } os << "\t// pointing to " << how_many << " (right after last node)\n"; uint64_t payload; for (; tape_idx < how_many; tape_idx++) { os << tape_idx << " : "; tape_val = tape[tape_idx]; payload = tape_val & JSON_VALUE_MASK; type = (tape_val >> 56); switch (type) { case '"': // we have a string os << "string \""; memcpy(&string_length, string_buf.get() + payload, sizeof(uint32_t)); internal::print_with_escapes( (const unsigned char *)(string_buf.get() + payload + sizeof(uint32_t)), os, string_length); os << '"'; os << '\n'; break; case 'l': // we have a long int if (tape_idx + 1 >= how_many) { return false; } os << "integer " << static_cast(tape[++tape_idx]) << "\n"; break; case 'u': // we have a long uint if (tape_idx + 1 >= how_many) { return false; } os << "unsigned integer " << tape[++tape_idx] << "\n"; break; case 'd': // we have a double os << "float "; if (tape_idx + 1 >= how_many) { return false; } double answer; memcpy(&answer, &tape[++tape_idx], sizeof(answer)); os << answer << '\n'; break; case 'n': // we have a null os << "null\n"; break; case 't': // we have a true os << "true\n"; break; case 'f': // we have a false os << "false\n"; break; case '{': // we have an object os << "{\t// pointing to next tape location " << payload << " (first node after the scope) \n"; break; case '}': // we end an object os << "}\t// pointing to previous tape location " << payload << " (start of the scope) \n"; break; case '[': // we start an array os << "[\t// pointing to next tape location " << payload << " (first node after the scope) \n"; break; case ']': // we end an array os << "]\t// pointing to previous tape location " << payload << " (start of the scope) \n"; break; case 'r': // we start and end with the root node // should we be hitting the root node? return false; default: return false; } } tape_val = tape[tape_idx]; payload = tape_val & JSON_VALUE_MASK; type = (tape_val >> 56); os << tape_idx << " : " << type << "\t// pointing to " << payload << " (start root)\n"; return true; } // // document::doc_ref_result inline implementation // inline document::doc_ref_result::doc_ref_result(document &_doc, error_code _error) noexcept : doc(_doc), error(_error) { } inline document::doc_ref_result::operator document&() noexcept(false) { if (error) { throw invalid_json(error); } return doc; } inline const std::string &document::doc_ref_result::get_error_message() const noexcept { return error_message(error); } // // document::doc_result inline implementation // inline document::doc_result::doc_result(document &&_doc, error_code _error) noexcept : doc(std::move(_doc)), error(_error) { } inline document::doc_result::doc_result(document &&_doc) noexcept : doc(std::move(_doc)), error(SUCCESS) { } inline document::doc_result::doc_result(error_code _error) noexcept : doc(), error(_error) { } inline document::doc_result::operator document() noexcept(false) { if (error) { throw invalid_json(error); } return std::move(doc); } inline const std::string &document::doc_result::get_error_message() const noexcept { return error_message(error); } // // document::parser inline implementation // inline bool document::parser::is_valid() const noexcept { return valid; } inline int document::parser::get_error_code() const noexcept { return error; } inline std::string document::parser::get_error_message() const noexcept { return error_message(error); } inline bool document::parser::print_json(std::ostream &os) const noexcept { return is_valid() ? doc.print_json(os) : false; } inline bool document::parser::dump_raw_tape(std::ostream &os) const noexcept { return is_valid() ? doc.dump_raw_tape(os) : false; } inline const document &document::parser::get_document() const noexcept(false) { if (!is_valid()) { throw invalid_json(error); } return doc; } inline document::doc_ref_result document::parser::parse(const uint8_t *buf, size_t len, bool realloc_if_needed) noexcept { error_code code = init_parse(len); if (code) { return document::doc_ref_result(doc, code); } if (realloc_if_needed) { const uint8_t *tmp_buf = buf; buf = (uint8_t *)allocate_padded_buffer(len); if (buf == nullptr) return document::doc_ref_result(doc, MEMALLOC); memcpy((void *)buf, tmp_buf, len); } code = simdjson::active_implementation->parse(buf, len, *this); // We're indicating validity via the doc_ref_result, so set the parse state back to invalid valid = false; error = UNINITIALIZED; if (realloc_if_needed) { aligned_free((void *)buf); // must free before we exit } return document::doc_ref_result(doc, code); } really_inline document::doc_ref_result document::parser::parse(const char *buf, size_t len, bool realloc_if_needed) noexcept { return parse((const uint8_t *)buf, len, realloc_if_needed); } really_inline document::doc_ref_result document::parser::parse(const std::string &s) noexcept { return parse(s.data(), s.length(), s.capacity() - s.length() < SIMDJSON_PADDING); } really_inline document::doc_ref_result document::parser::parse(const padded_string &s) noexcept { return parse(s.data(), s.length(), false); } really_inline size_t document::parser::capacity() const noexcept { return _capacity; } really_inline size_t document::parser::max_depth() const noexcept { return _max_depth; } WARN_UNUSED inline bool document::parser::allocate_capacity(size_t capacity, size_t max_depth) { return set_capacity(capacity) && set_max_depth(max_depth); } WARN_UNUSED inline bool document::parser::set_capacity(size_t capacity) { if (_capacity == capacity) { return true; } // Set capacity to 0 until we finish, in case there's an error _capacity = 0; // // Reallocate the document // if (!doc.set_capacity(capacity)) { return false; } // // Don't allocate 0 bytes, just return. // if (capacity == 0) { structural_indexes.reset(); return true; } // // Initialize stage 1 output // uint32_t max_structures = ROUNDUP_N(capacity, 64) + 2 + 7; structural_indexes.reset( new (std::nothrow) uint32_t[max_structures]); // TODO realloc if (!structural_indexes) { return false; } _capacity = capacity; return true; } WARN_UNUSED inline bool document::parser::set_max_depth(size_t max_depth) { _max_depth = 0; if (max_depth == 0) { ret_address.reset(); containing_scope_offset.reset(); return true; } // // Initialize stage 2 state // containing_scope_offset.reset(new (std::nothrow) uint32_t[max_depth]); // TODO realloc #ifdef SIMDJSON_USE_COMPUTED_GOTO ret_address.reset(new (std::nothrow) void *[max_depth]); #else ret_address.reset(new (std::nothrow) char[max_depth]); #endif if (!ret_address || !containing_scope_offset) { // Could not allocate memory return false; } _max_depth = max_depth; return true; } WARN_UNUSED inline error_code document::parser::init_parse(size_t len) noexcept { if (len > capacity()) { return error = CAPACITY; } // If the last doc was taken, we need to allocate a new one if (!doc.tape) { if (!doc.set_capacity(len)) { return error = MEMALLOC; } } return SUCCESS; } // // document::tape_ref inline implementation // really_inline document::tape_ref::tape_ref() noexcept : doc{nullptr}, json_index{0} {} really_inline document::tape_ref::tape_ref(const document *_doc, size_t _json_index) noexcept : doc{_doc}, json_index{_json_index} {} inline size_t document::tape_ref::after_element() const noexcept { switch (type()) { case tape_type::START_ARRAY: case tape_type::START_OBJECT: return tape_value(); case tape_type::UINT64: case tape_type::INT64: case tape_type::DOUBLE: return json_index + 2; default: return json_index + 1; } } really_inline document::tape_type document::tape_ref::type() const noexcept { return static_cast(doc->tape[json_index] >> 56); } really_inline uint64_t document::tape_ref::tape_value() const noexcept { return doc->tape[json_index] & JSON_VALUE_MASK; } template really_inline T document::tape_ref::next_tape_value() const noexcept { static_assert(sizeof(T) == sizeof(uint64_t)); return *reinterpret_cast(&doc->tape[json_index + 1]); } // // document::array inline implementation // really_inline document::array::array() noexcept : tape_ref() {} really_inline document::array::array(const document *_doc, size_t _json_index) noexcept : tape_ref(_doc, _json_index) {} inline document::array::iterator document::array::begin() const noexcept { return iterator(doc, json_index + 1); } inline document::array::iterator document::array::end() const noexcept { return iterator(doc, after_element() - 1); } // // document::array::iterator inline implementation // really_inline document::array::iterator::iterator(const document *_doc, size_t _json_index) noexcept : tape_ref(_doc, _json_index) { } inline document::element document::array::iterator::operator*() const noexcept { return element(doc, json_index); } inline bool document::array::iterator::operator!=(const document::array::iterator& other) const noexcept { return json_index != other.json_index; } inline void document::array::iterator::operator++() noexcept { json_index = after_element(); } // // document::object inline implementation // really_inline document::object::object() noexcept : tape_ref() {} really_inline document::object::object(const document *_doc, size_t _json_index) noexcept : tape_ref(_doc, _json_index) { }; inline document::object::iterator document::object::begin() const noexcept { return iterator(doc, json_index + 1); } inline document::object::iterator document::object::end() const noexcept { return iterator(doc, after_element() - 1); } inline document::element_result document::object::operator[](const std::string_view &key) const noexcept { iterator end_field = end(); for (iterator field = begin(); field != end_field; ++field) { if (key == field.key()) { return field.value(); } } return NO_SUCH_FIELD; } inline document::element_result document::object::operator[](const char *key) const noexcept { iterator end_field = end(); for (iterator field = begin(); field != end_field; ++field) { if (!strcmp(key, field.key_c_str())) { return field.value(); } } return NO_SUCH_FIELD; } // // document::object::iterator inline implementation // really_inline document::object::iterator::iterator(const document *_doc, size_t _json_index) noexcept : tape_ref(_doc, _json_index) { } inline const document::key_value_pair document::object::iterator::operator*() const noexcept { return key_value_pair(key(), value()); } inline bool document::object::iterator::operator!=(const document::object::iterator& other) const noexcept { return json_index != other.json_index; } inline void document::object::iterator::operator++() noexcept { json_index++; json_index = after_element(); } inline std::string_view document::object::iterator::key() const noexcept { size_t string_buf_index = tape_value(); uint32_t len; memcpy(&len, &doc->string_buf[string_buf_index], sizeof(len)); return std::string_view( reinterpret_cast(&doc->string_buf[string_buf_index + sizeof(uint32_t)]), len ); } inline const char* document::object::iterator::key_c_str() const noexcept { return reinterpret_cast(&doc->string_buf[tape_value() + sizeof(uint32_t)]); } inline document::element document::object::iterator::value() const noexcept { return element(doc, json_index + 1); } // // document::key_value_pair inline implementation // inline document::key_value_pair::key_value_pair(std::string_view _key, document::element _value) noexcept : key(_key), value(_value) {} // // document::element inline implementation // really_inline document::element::element() noexcept : tape_ref() {} really_inline document::element::element(const document *_doc, size_t _json_index) noexcept : tape_ref(_doc, _json_index) { } really_inline bool document::element::is_null() const noexcept { return type() == tape_type::NULL_VALUE; } really_inline bool document::element::is_bool() const noexcept { return type() == tape_type::TRUE_VALUE || type() == tape_type::FALSE_VALUE; } really_inline bool document::element::is_number() const noexcept { return type() == tape_type::UINT64 || type() == tape_type::INT64 || type() == tape_type::DOUBLE; } really_inline bool document::element::is_integer() const noexcept { return type() == tape_type::UINT64 || type() == tape_type::INT64; } really_inline bool document::element::is_string() const noexcept { return type() == tape_type::STRING; } really_inline bool document::element::is_array() const noexcept { return type() == tape_type::START_ARRAY; } really_inline bool document::element::is_object() const noexcept { return type() == tape_type::START_OBJECT; } inline document::element::operator bool() const noexcept(false) { return as_bool(); } inline document::element::operator const char*() const noexcept(false) { return as_c_str(); } inline document::element::operator std::string_view() const noexcept(false) { return as_string(); } inline document::element::operator uint64_t() const noexcept(false) { return as_uint64_t(); } inline document::element::operator int64_t() const noexcept(false) { return as_int64_t(); } inline document::element::operator double() const noexcept(false) { return as_double(); } inline document::element::operator document::array() const noexcept(false) { return as_array(); } inline document::element::operator document::object() const noexcept(false) { return as_object(); } inline document::element_result document::element::as_bool() const noexcept { switch (type()) { case tape_type::TRUE_VALUE: return true; case tape_type::FALSE_VALUE: return false; default: return INCORRECT_TYPE; } } inline document::element_result document::element::as_c_str() const noexcept { switch (type()) { case tape_type::STRING: { size_t string_buf_index = tape_value(); return reinterpret_cast(&doc->string_buf[string_buf_index + sizeof(uint32_t)]); } default: return INCORRECT_TYPE; } } inline document::element_result document::element::as_string() const noexcept { switch (type()) { case tape_type::STRING: { size_t string_buf_index = tape_value(); uint32_t len; memcpy(&len, &doc->string_buf[string_buf_index], sizeof(len)); return std::string_view( reinterpret_cast(&doc->string_buf[string_buf_index + sizeof(uint32_t)]), len ); } default: return INCORRECT_TYPE; } } inline document::element_result document::element::as_uint64_t() const noexcept { switch (type()) { case tape_type::UINT64: return next_tape_value(); case tape_type::INT64: { int64_t result = next_tape_value(); if (result < 0) { return NUMBER_OUT_OF_RANGE; } return static_cast(result); } default: return INCORRECT_TYPE; } } inline document::element_result document::element::as_int64_t() const noexcept { switch (type()) { case tape_type::UINT64: { uint64_t result = next_tape_value(); // Wrapping max in parens to handle Windows issue: https://stackoverflow.com/questions/11544073/how-do-i-deal-with-the-max-macro-in-windows-h-colliding-with-max-in-std if (result > (std::numeric_limits::max)()) { return NUMBER_OUT_OF_RANGE; } return static_cast(result); } case tape_type::INT64: return next_tape_value(); default: std::cout << "Incorrect " << json_index << " = " << char(type()) << std::endl; return INCORRECT_TYPE; } } inline document::element_result document::element::as_double() const noexcept { switch (type()) { case tape_type::UINT64: return next_tape_value(); case tape_type::INT64: { return next_tape_value(); int64_t result = tape_value(); if (result < 0) { return NUMBER_OUT_OF_RANGE; } return result; } case tape_type::DOUBLE: return next_tape_value(); default: return INCORRECT_TYPE; } } inline document::element_result document::element::as_array() const noexcept { switch (type()) { case tape_type::START_ARRAY: return array(doc, json_index); default: return INCORRECT_TYPE; } } inline document::element_result document::element::as_object() const noexcept { switch (type()) { case tape_type::START_OBJECT: return object(doc, json_index); default: return INCORRECT_TYPE; } } inline document::element_result document::element::operator[](const std::string_view &key) const noexcept { auto [obj, error] = as_object(); if (error) { return error; } return obj[key]; } inline document::element_result document::element::operator[](const char *key) const noexcept { auto [obj, error] = as_object(); if (error) { return error; } return obj[key]; } } // namespace simdjson #endif // SIMDJSON_INLINE_DOCUMENT_H /* end file include/simdjson/inline/document.h */ /* begin file include/simdjson/inline/document_iterator.h */ #ifndef SIMDJSON_INLINE_DOCUMENT_ITERATOR_H #define SIMDJSON_INLINE_DOCUMENT_ITERATOR_H namespace simdjson { // Because of template weirdness, the actual class definition is inline in the document class template WARN_UNUSED bool document_iterator::is_ok() const { return location < tape_length; } // useful for debuging purposes template size_t document_iterator::get_tape_location() const { return location; } // useful for debuging purposes template size_t document_iterator::get_tape_length() const { return tape_length; } // returns the current depth (start at 1 with 0 reserved for the fictitious root // node) template size_t document_iterator::get_depth() const { return depth; } // A scope is a series of nodes at the same depth, typically it is either an // object ({) or an array ([). The root node has type 'r'. template uint8_t document_iterator::get_scope_type() const { return depth_index[depth].scope_type; } template bool document_iterator::move_forward() { if (location + 1 >= tape_length) { return false; // we are at the end! } if ((current_type == '[') || (current_type == '{')) { // We are entering a new scope depth++; assert(depth < max_depth); depth_index[depth].start_of_scope = location; depth_index[depth].scope_type = current_type; } else if ((current_type == ']') || (current_type == '}')) { // Leaving a scope. depth--; } else if (is_number()) { // these types use 2 locations on the tape, not just one. location += 1; } location += 1; current_val = doc.tape[location]; current_type = (current_val >> 56); return true; } template void document_iterator::move_to_value() { // assume that we are on a key, so move by 1. location += 1; current_val = doc.tape[location]; current_type = (current_val >> 56); } template bool document_iterator::move_to_key(const char *key) { if (down()) { do { const bool right_key = (strcmp(get_string(), key) == 0); move_to_value(); if (right_key) { return true; } } while (next()); up(); } return false; } template bool document_iterator::move_to_key_insensitive( const char *key) { if (down()) { do { const bool right_key = (simdjson_strcasecmp(get_string(), key) == 0); move_to_value(); if (right_key) { return true; } } while (next()); up(); } return false; } template bool document_iterator::move_to_key(const char *key, uint32_t length) { if (down()) { do { bool right_key = ((get_string_length() == length) && (memcmp(get_string(), key, length) == 0)); move_to_value(); if (right_key) { return true; } } while (next()); up(); } return false; } template bool document_iterator::move_to_index(uint32_t index) { if (down()) { uint32_t i = 0; for (; i < index; i++) { if (!next()) { break; } } if (i == index) { return true; } up(); } return false; } template bool document_iterator::prev() { size_t target_location = location; to_start_scope(); size_t npos = location; if (target_location == npos) { return false; // we were already at the start } size_t oldnpos; // we have that npos < target_location here do { oldnpos = npos; if ((current_type == '[') || (current_type == '{')) { // we need to jump npos = (current_val & JSON_VALUE_MASK); } else { npos = npos + ((current_type == 'd' || current_type == 'l') ? 2 : 1); } } while (npos < target_location); location = oldnpos; current_val = doc.tape[location]; current_type = current_val >> 56; return true; } template bool document_iterator::up() { if (depth == 1) { return false; // don't allow moving back to root } to_start_scope(); // next we just move to the previous value depth--; location -= 1; current_val = doc.tape[location]; current_type = (current_val >> 56); return true; } template bool document_iterator::down() { if (location + 1 >= tape_length) { return false; } if ((current_type == '[') || (current_type == '{')) { size_t npos = (current_val & JSON_VALUE_MASK); if (npos == location + 2) { return false; // we have an empty scope } depth++; assert(depth < max_depth); location = location + 1; depth_index[depth].start_of_scope = location; depth_index[depth].scope_type = current_type; current_val = doc.tape[location]; current_type = (current_val >> 56); return true; } return false; } template void document_iterator::to_start_scope() { location = depth_index[depth].start_of_scope; current_val = doc.tape[location]; current_type = (current_val >> 56); } template bool document_iterator::next() { size_t npos; if ((current_type == '[') || (current_type == '{')) { // we need to jump npos = (current_val & JSON_VALUE_MASK); } else { npos = location + (is_number() ? 2 : 1); } uint64_t next_val = doc.tape[npos]; uint8_t next_type = (next_val >> 56); if ((next_type == ']') || (next_type == '}')) { return false; // we reached the end of the scope } location = npos; current_val = next_val; current_type = next_type; return true; } template document_iterator::document_iterator(const document &doc_) noexcept : doc(doc_), depth(0), location(0), tape_length(0) { depth_index[0].start_of_scope = location; current_val = doc.tape[location++]; current_type = (current_val >> 56); depth_index[0].scope_type = current_type; tape_length = current_val & JSON_VALUE_MASK; if (location < tape_length) { // If we make it here, then depth_capacity must >=2, but the compiler // may not know this. current_val = doc.tape[location]; current_type = (current_val >> 56); depth++; assert(depth < max_depth); depth_index[depth].start_of_scope = location; depth_index[depth].scope_type = current_type; } } template document_iterator::document_iterator(const document::parser &parser) : document_iterator(parser.get_document()) {} template document_iterator::document_iterator( const document_iterator &o) noexcept : doc(o.doc), depth(o.depth), location(o.location), tape_length(o.tape_length), current_type(o.current_type), current_val(o.current_val) { memcpy(depth_index, o.depth_index, (depth + 1) * sizeof(depth_index[0])); } template document_iterator &document_iterator:: operator=(const document_iterator &o) noexcept { doc = o.doc; depth = o.depth; location = o.location; tape_length = o.tape_length; current_type = o.current_type; current_val = o.current_val; memcpy(depth_index, o.depth_index, (depth + 1) * sizeof(depth_index[0])); return *this; } template bool document_iterator::print(std::ostream &os, bool escape_strings) const { if (!is_ok()) { return false; } switch (current_type) { case '"': // we have a string os << '"'; if (escape_strings) { internal::print_with_escapes(get_string(), os, get_string_length()); } else { // was: os << get_string();, but given that we can include null chars, we // have to do something crazier: std::copy(get_string(), get_string() + get_string_length(), std::ostream_iterator(os)); } os << '"'; break; case 'l': // we have a long int os << get_integer(); break; case 'u': os << get_unsigned_integer(); break; case 'd': os << get_double(); break; case 'n': // we have a null os << "null"; break; case 't': // we have a true os << "true"; break; case 'f': // we have a false os << "false"; break; case '{': // we have an object case '}': // we end an object case '[': // we start an array case ']': // we end an array os << static_cast(current_type); break; default: return false; } return true; } template bool document_iterator::move_to(const char *pointer, uint32_t length) { char *new_pointer = nullptr; if (pointer[0] == '#') { // Converting fragment representation to string representation new_pointer = new char[length]; uint32_t new_length = 0; for (uint32_t i = 1; i < length; i++) { if (pointer[i] == '%' && pointer[i + 1] == 'x') { try { int fragment = std::stoi(std::string(&pointer[i + 2], 2), nullptr, 16); if (fragment == '\\' || fragment == '"' || (fragment <= 0x1F)) { // escaping the character new_pointer[new_length] = '\\'; new_length++; } new_pointer[new_length] = fragment; i += 3; } catch (std::invalid_argument &) { delete[] new_pointer; return false; // the fragment is invalid } } else { new_pointer[new_length] = pointer[i]; } new_length++; } length = new_length; pointer = new_pointer; } // saving the current state size_t depth_s = depth; size_t location_s = location; uint8_t current_type_s = current_type; uint64_t current_val_s = current_val; rewind(); // The json pointer is used from the root of the document. bool found = relative_move_to(pointer, length); delete[] new_pointer; if (!found) { // since the pointer has found nothing, we get back to the original // position. depth = depth_s; location = location_s; current_type = current_type_s; current_val = current_val_s; } return found; } template bool document_iterator::relative_move_to(const char *pointer, uint32_t length) { if (length == 0) { // returns the whole document return true; } if (pointer[0] != '/') { // '/' must be the first character return false; } // finding the key in an object or the index in an array std::string key_or_index; uint32_t offset = 1; // checking for the "-" case if (is_array() && pointer[1] == '-') { if (length != 2) { // the pointer must be exactly "/-" // there can't be anything more after '-' as an index return false; } key_or_index = '-'; offset = length; // will skip the loop coming right after } // We either transform the first reference token to a valid json key // or we make sure it is a valid index in an array. for (; offset < length; offset++) { if (pointer[offset] == '/') { // beginning of the next key or index break; } if (is_array() && (pointer[offset] < '0' || pointer[offset] > '9')) { // the index of an array must be an integer // we also make sure std::stoi won't discard whitespaces later return false; } if (pointer[offset] == '~') { // "~1" represents "/" if (pointer[offset + 1] == '1') { key_or_index += '/'; offset++; continue; } // "~0" represents "~" if (pointer[offset + 1] == '0') { key_or_index += '~'; offset++; continue; } } if (pointer[offset] == '\\') { if (pointer[offset + 1] == '\\' || pointer[offset + 1] == '"' || (pointer[offset + 1] <= 0x1F)) { key_or_index += pointer[offset + 1]; offset++; continue; } return false; // invalid escaped character } if (pointer[offset] == '\"') { // unescaped quote character. this is an invalid case. // lets do nothing and assume most pointers will be valid. // it won't find any corresponding json key anyway. // return false; } key_or_index += pointer[offset]; } bool found = false; if (is_object()) { if (move_to_key(key_or_index.c_str(), key_or_index.length())) { found = relative_move_to(pointer + offset, length - offset); } } else if (is_array()) { if (key_or_index == "-") { // handling "-" case first if (down()) { while (next()) ; // moving to the end of the array // moving to the nonexistent value right after... size_t npos; if ((current_type == '[') || (current_type == '{')) { // we need to jump npos = (current_val & JSON_VALUE_MASK); } else { npos = location + ((current_type == 'd' || current_type == 'l') ? 2 : 1); } location = npos; current_val = doc.tape[npos]; current_type = (current_val >> 56); return true; // how could it fail ? } } else { // regular numeric index // The index can't have a leading '0' if (key_or_index[0] == '0' && key_or_index.length() > 1) { return false; } // it cannot be empty if (key_or_index.length() == 0) { return false; } // we already checked the index contains only valid digits uint32_t index = std::stoi(key_or_index); if (move_to_index(index)) { found = relative_move_to(pointer + offset, length - offset); } } } return found; } } // namespace simdjson #endif // SIMDJSON_INLINE_DOCUMENT_ITERATOR_H /* end file include/simdjson/inline/document_iterator.h */ /* begin file include/simdjson/inline/jsonstream.h */ #ifndef SIMDJSON_INLINE_JSONSTREAM_H #define SIMDJSON_INLINE_JSONSTREAM_H #include #include #include #include namespace simdjson::internal { /* This algorithm is used to quickly identify the buffer position of * the last JSON document inside the current batch. * * It does its work by finding the last pair of structural characters * that represent the end followed by the start of a document. * * Simply put, we iterate over the structural characters, starting from * the end. We consider that we found the end of a JSON document when the * first element of the pair is NOT one of these characters: '{' '[' ';' ',' * and when the second element is NOT one of these characters: '}' '}' ';' ','. * * This simple comparison works most of the time, but it does not cover cases * where the batch's structural indexes contain a perfect amount of documents. * In such a case, we do not have access to the structural index which follows * the last document, therefore, we do not have access to the second element in * the pair, and means that we cannot identify the last document. To fix this * issue, we keep a count of the open and closed curly/square braces we found * while searching for the pair. When we find a pair AND the count of open and * closed curly/square braces is the same, we know that we just passed a * complete * document, therefore the last json buffer location is the end of the batch * */ inline size_t find_last_json_buf_idx(const uint8_t *buf, size_t size, const document::parser &parser) { // this function can be generally useful if (parser.n_structural_indexes == 0) return 0; auto last_i = parser.n_structural_indexes - 1; if (parser.structural_indexes[last_i] == size) { if (last_i == 0) return 0; last_i = parser.n_structural_indexes - 2; } auto arr_cnt = 0; auto obj_cnt = 0; for (auto i = last_i; i > 0; i--) { auto idxb = parser.structural_indexes[i]; switch (buf[idxb]) { case ':': case ',': continue; case '}': obj_cnt--; continue; case ']': arr_cnt--; continue; case '{': obj_cnt++; break; case '[': arr_cnt++; break; } auto idxa = parser.structural_indexes[i - 1]; switch (buf[idxa]) { case '{': case '[': case ':': case ',': continue; } if (!arr_cnt && !obj_cnt) { return last_i + 1; } return i; } return 0; } // returns true if the provided byte value is an ASCII character static inline bool is_ascii(char c) { return ((unsigned char)c) <= 127; } // if the string ends with UTF-8 values, backtrack // up to the first ASCII character. May return 0. static inline size_t trimmed_length_safe_utf8(const char * c, size_t len) { while ((len > 0) and (not is_ascii(c[len - 1]))) { len--; } return len; } } // namespace simdjson::internal namespace simdjson { template JsonStream::JsonStream(const string_container &s, size_t batchSize) : str(s), _batch_size(batchSize) { } template JsonStream::~JsonStream() { #ifdef SIMDJSON_THREADS_ENABLED if (stage_1_thread.joinable()) { stage_1_thread.join(); } #endif } #ifdef SIMDJSON_THREADS_ENABLED // threaded version of json_parse // todo: simplify this code further template int JsonStream::json_parse(document::parser &parser) { if (unlikely(parser.capacity() == 0)) { const bool allocok = parser.allocate_capacity(_batch_size); if (!allocok) { return parser.error = simdjson::MEMALLOC; } } else if (unlikely(parser.capacity() < _batch_size)) { return parser.error = simdjson::CAPACITY; } if (unlikely(parser_thread.capacity() < _batch_size)) { const bool allocok_thread = parser_thread.allocate_capacity(_batch_size); if (!allocok_thread) { return parser.error = simdjson::MEMALLOC; } } if (unlikely(load_next_batch)) { // First time loading if (!stage_1_thread.joinable()) { _batch_size = (std::min)(_batch_size, remaining()); _batch_size = internal::trimmed_length_safe_utf8((const char *)buf(), _batch_size); if (_batch_size == 0) { return parser.error = simdjson::UTF8_ERROR; } auto stage1_is_ok = error_code(simdjson::active_implementation->stage1(buf(), _batch_size, parser, true)); if (stage1_is_ok != simdjson::SUCCESS) { return parser.error = stage1_is_ok; } size_t last_index = internal::find_last_json_buf_idx(buf(), _batch_size, parser); if (last_index == 0) { if (parser.n_structural_indexes == 0) { return parser.error = simdjson::EMPTY; } } else { parser.n_structural_indexes = last_index + 1; } } // the second thread is running or done. else { stage_1_thread.join(); if (stage1_is_ok_thread != simdjson::SUCCESS) { return parser.error = stage1_is_ok_thread; } std::swap(parser.structural_indexes, parser_thread.structural_indexes); parser.n_structural_indexes = parser_thread.n_structural_indexes; advance(last_json_buffer_loc); n_bytes_parsed += last_json_buffer_loc; } // let us decide whether we will start a new thread if (remaining() - _batch_size > 0) { last_json_buffer_loc = parser.structural_indexes[internal::find_last_json_buf_idx(buf(), _batch_size, parser)]; _batch_size = (std::min)(_batch_size, remaining() - last_json_buffer_loc); if (_batch_size > 0) { _batch_size = internal::trimmed_length_safe_utf8( (const char *)(buf() + last_json_buffer_loc), _batch_size); if (_batch_size == 0) { return parser.error = simdjson::UTF8_ERROR; } // let us capture read-only variables const uint8_t *const b = buf() + last_json_buffer_loc; const size_t bs = _batch_size; // we call the thread on a lambda that will update // this->stage1_is_ok_thread // there is only one thread that may write to this value stage_1_thread = std::thread([this, b, bs] { this->stage1_is_ok_thread = error_code(simdjson::active_implementation->stage1(b, bs, this->parser_thread, true)); }); } } next_json = 0; load_next_batch = false; } // load_next_batch int res = simdjson::active_implementation->stage2(buf(), remaining(), parser, next_json); if (res == simdjson::SUCCESS_AND_HAS_MORE) { n_parsed_docs++; current_buffer_loc = parser.structural_indexes[next_json]; load_next_batch = (current_buffer_loc == last_json_buffer_loc); } else if (res == simdjson::SUCCESS) { n_parsed_docs++; if (remaining() > _batch_size) { current_buffer_loc = parser.structural_indexes[next_json - 1]; load_next_batch = true; res = simdjson::SUCCESS_AND_HAS_MORE; } } return res; } #else // SIMDJSON_THREADS_ENABLED // single-threaded version of json_parse template int JsonStream::json_parse(document::parser &parser) { if (unlikely(parser.capacity() == 0)) { const bool allocok = parser.allocate_capacity(_batch_size); if (!allocok) { parser.valid = false; return parser.error = MEMALLOC; } } else if (unlikely(parser.capacity() < _batch_size)) { parser.valid = false; return parser.error = CAPACITY; } if (unlikely(load_next_batch)) { advance(current_buffer_loc); n_bytes_parsed += current_buffer_loc; _batch_size = (std::min)(_batch_size, remaining()); _batch_size = internal::trimmed_length_safe_utf8((const char *)buf(), _batch_size); auto stage1_is_ok = (error_code)simdjson::active_implementation->stage1(buf(), _batch_size, parser, true); if (stage1_is_ok != simdjson::SUCCESS) { parser.valid = false; return parser.error = stage1_is_ok; } size_t last_index = internal::find_last_json_buf_idx(buf(), _batch_size, parser); if (last_index == 0) { if (parser.n_structural_indexes == 0) { parser.valid = false; return parser.error = EMPTY; } } else { parser.n_structural_indexes = last_index + 1; } load_next_batch = false; } // load_next_batch int res = simdjson::active_implementation->stage2(buf(), remaining(), parser, next_json); if (likely(res == simdjson::SUCCESS_AND_HAS_MORE)) { n_parsed_docs++; current_buffer_loc = parser.structural_indexes[next_json]; } else if (res == simdjson::SUCCESS) { n_parsed_docs++; if (remaining() > _batch_size) { current_buffer_loc = parser.structural_indexes[next_json - 1]; next_json = 1; load_next_batch = true; res = simdjson::SUCCESS_AND_HAS_MORE; } } else { printf("E\n"); } return res; } #endif // SIMDJSON_THREADS_ENABLED } // end of namespace simdjson #endif // SIMDJSON_INLINE_JSONSTREAM_H /* end file include/simdjson/inline/jsonstream.h */ #endif // SIMDJSON_H /* end file include/simdjson/inline/jsonstream.h */