From 31e8a12e888c76f509c0994149153865dd2cc4ae Mon Sep 17 00:00:00 2001 From: John Keiser Date: Fri, 6 Mar 2020 11:55:10 -0800 Subject: [PATCH] Make error_message(error_code) return C string - Also move all error message logic to include inline --- Makefile | 4 +- benchmark/benchmarker.h | 2 +- benchmark/parse_stream.cpp | 4 +- include/CMakeLists.txt | 1 + include/simdjson.h | 1 + include/simdjson/document.h | 10 - include/simdjson/document_parser.h | 521 +++++++++++++++++++++++++++++ include/simdjson/error.h | 72 ++-- include/simdjson/inline/document.h | 8 +- include/simdjson/inline/error.h | 55 +++ singleheader/simdjson.h | 2 +- src/CMakeLists.txt | 1 - src/error.cpp | 45 --- src/simdjson.cpp | 1 - tests/basictests.cpp | 27 +- 15 files changed, 653 insertions(+), 101 deletions(-) create mode 100644 include/simdjson/document_parser.h create mode 100644 include/simdjson/inline/error.h delete mode 100644 src/error.cpp diff --git a/Makefile b/Makefile index c8c5ca6a6..eba1e2927 100644 --- a/Makefile +++ b/Makefile @@ -62,10 +62,10 @@ SRCHEADERS_GENERIC=src/generic/numberparsing.h src/generic/stage1_find_marks.h s SRCHEADERS_ARM64= src/arm64/bitmanipulation.h src/arm64/bitmask.h src/arm64/intrinsics.h src/arm64/numberparsing.h src/arm64/simd.h src/arm64/stage1_find_marks.h src/arm64/stage2_build_tape.h src/arm64/stringparsing.h SRCHEADERS_HASWELL= src/haswell/bitmanipulation.h src/haswell/bitmask.h src/haswell/intrinsics.h src/haswell/numberparsing.h src/haswell/simd.h src/haswell/stage1_find_marks.h src/haswell/stage2_build_tape.h src/haswell/stringparsing.h SRCHEADERS_WESTMERE=src/westmere/bitmanipulation.h src/westmere/bitmask.h src/westmere/intrinsics.h src/westmere/numberparsing.h src/westmere/simd.h src/westmere/stage1_find_marks.h src/westmere/stage2_build_tape.h src/westmere/stringparsing.h -SRCHEADERS_SRC=src/isadetection.h src/jsoncharutils.h src/simdprune_tables.h src/error.cpp src/jsonioutil.cpp src/implementation.cpp src/stage1_find_marks.cpp src/stage2_build_tape.cpp src/document_parser_callbacks.h +SRCHEADERS_SRC=src/isadetection.h src/jsoncharutils.h src/simdprune_tables.h src/jsonioutil.cpp src/implementation.cpp src/stage1_find_marks.cpp src/stage2_build_tape.cpp src/document_parser_callbacks.h SRCHEADERS=$(SRCHEADERS_SRC) $(SRCHEADERS_GENERIC) $(SRCHEADERS_ARM64) $(SRCHEADERS_HASWELL) $(SRCHEADERS_WESTMERE) -INCLUDEHEADERS=include/simdjson.h include/simdjson/common_defs.h include/simdjson/internal/jsonformatutils.h include/simdjson/jsonioutil.h include/simdjson/jsonminifier.h include/simdjson/jsonparser.h include/simdjson/padded_string.h include/simdjson/document.h include/simdjson/inline/document.h include/simdjson/document_iterator.h include/simdjson/inline/document_iterator.h include/simdjson/document_stream.h include/simdjson/inline/document_stream.h include/simdjson/implementation.h include/simdjson/parsedjson.h include/simdjson/jsonstream.h include/simdjson/inline/jsonstream.h include/simdjson/portability.h include/simdjson/error.h include/simdjson/simdjson.h include/simdjson/simdjson_version.h +INCLUDEHEADERS=include/simdjson.h include/simdjson/common_defs.h include/simdjson/internal/jsonformatutils.h include/simdjson/jsonioutil.h include/simdjson/jsonminifier.h include/simdjson/jsonparser.h include/simdjson/padded_string.h include/simdjson/document.h include/simdjson/inline/document.h include/simdjson/document_iterator.h include/simdjson/inline/document_iterator.h include/simdjson/document_stream.h include/simdjson/inline/document_stream.h include/simdjson/implementation.h include/simdjson/parsedjson.h include/simdjson/jsonstream.h include/simdjson/inline/jsonstream.h include/simdjson/portability.h include/simdjson/error.h include/simdjson/inline/error.h include/simdjson/simdjson.h include/simdjson/simdjson_version.h ifeq ($(SIMDJSON_TEST_AMALGAMATED_HEADERS),1) HEADERS=singleheader/simdjson.h diff --git a/benchmark/benchmarker.h b/benchmark/benchmarker.h index c4645c9f6..188a027f9 100644 --- a/benchmark/benchmarker.h +++ b/benchmark/benchmarker.h @@ -305,7 +305,7 @@ struct benchmarker { if(hotbuffers) { auto result = parser.parse((const uint8_t *)json.data(), json.size()); if (result.error) { - exit_error(string("Failed to parse ") + filename + string(":") + result.get_error_message()); + exit_error(string("Failed to parse ") + filename + string(":") + error_message(result.error)); } } diff --git a/benchmark/parse_stream.cpp b/benchmark/parse_stream.cpp index 8bc2ea6eb..3aafd731c 100755 --- a/benchmark/parse_stream.cpp +++ b/benchmark/parse_stream.cpp @@ -96,7 +96,7 @@ int main (int argc, char *argv[]){ batch_size_res[i] = speedinGBs; if (error != simdjson::SUCCESS) { - std::wcerr << "Parsing failed with: " << simdjson::error_message(error).c_str() << std::endl; + std::wcerr << "Parsing failed with: " << simdjson::error_message(error) << std::endl; exit(1); } } @@ -132,7 +132,7 @@ int main (int argc, char *argv[]){ res.push_back(secs.count()); if (error != simdjson::SUCCESS) { - std::wcerr << "Parsing failed with: " << simdjson::error_message(error).c_str() << std::endl; + std::wcerr << "Parsing failed with: " << simdjson::error_message(error) << std::endl; exit(1); } diff --git a/include/CMakeLists.txt b/include/CMakeLists.txt index cd5236e0c..49735a390 100644 --- a/include/CMakeLists.txt +++ b/include/CMakeLists.txt @@ -11,6 +11,7 @@ set(SIMDJSON_INCLUDE ${SIMDJSON_INCLUDE_DIR}/simdjson/inline/document_stream.h ${SIMDJSON_INCLUDE_DIR}/simdjson/inline/document_iterator.h ${SIMDJSON_INCLUDE_DIR}/simdjson/inline/document.h + ${SIMDJSON_INCLUDE_DIR}/simdjson/inline/error.h ${SIMDJSON_INCLUDE_DIR}/simdjson/inline/jsonstream.h ${SIMDJSON_INCLUDE_DIR}/simdjson/internal/jsonformatutils.h ${SIMDJSON_INCLUDE_DIR}/simdjson/jsonioutil.h diff --git a/include/simdjson.h b/include/simdjson.h index f05d61eae..9bb2a0e1f 100644 --- a/include/simdjson.h +++ b/include/simdjson.h @@ -23,6 +23,7 @@ #include "simdjson/inline/document.h" #include "simdjson/inline/document_iterator.h" #include "simdjson/inline/document_stream.h" +#include "simdjson/inline/error.h" #include "simdjson/inline/jsonstream.h" #endif // SIMDJSON_H diff --git a/include/simdjson/document.h b/include/simdjson/document.h index d50ad9f61..3c8c86a5a 100644 --- a/include/simdjson/document.h +++ b/include/simdjson/document.h @@ -243,11 +243,6 @@ public: */ operator document() noexcept(false); - /** - * Get the error message for the error. - */ - const std::string &get_error_message() const noexcept; - ~doc_result() noexcept=default; private: @@ -307,11 +302,6 @@ public: */ operator document&() noexcept(false); - /** - * Get the error message for the error. - */ - const std::string &get_error_message() const noexcept; - ~doc_ref_result()=default; private: diff --git a/include/simdjson/document_parser.h b/include/simdjson/document_parser.h new file mode 100644 index 000000000..417ed3188 --- /dev/null +++ b/include/simdjson/document_parser.h @@ -0,0 +1,521 @@ +#ifndef SIMDJSON_DOCUMENT_PARSER_H +#define SIMDJSON_DOCUMENT_PARSER_H + +#include "simdjson/document.h" +#include "simdjson/common_defs.h" +#include "simdjson/error.h" +#include "simdjson/padded_string.h" +#include + +namespace simdjson { + +/** + * A persistent document parser. + * + * Use this if you intend to parse more than one document. It holds the internal memory necessary + * to do parsing, as well as memory for a single document that is overwritten on each parse. + * + * This class cannot be copied, only moved, to avoid unintended allocations. + * + * @note This is not thread safe: one parser cannot produce two documents at the same time! + */ +class document::parser { +public: + /** + * Create a JSON parser with zero capacity. Call allocate_capacity() to initialize it. + */ + parser()=default; + ~parser()=default; + + /** + * Take another parser's buffers and state. + * + * @param other The parser to take. Its capacity is zeroed. + */ + parser(document::parser &&other) = default; + parser(const document::parser &) = delete; // Disallow copying + /** + * Take another parser's buffers and state. + * + * @param other The parser to take. Its capacity is zeroed. + */ + parser &operator=(document::parser &&other) = default; + parser &operator=(const document::parser &) = delete; // Disallow copying + + /** + * Parse a JSON document and return a reference to it. + * + * The JSON document still lives in the parser: this is the most efficient way to parse JSON + * documents because it reuses the same buffers, but you *must* use the document before you + * destroy the parser or call parse() again. + * + * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what + * those bytes are initialized to, as long as they are allocated. If realloc_if_needed is true, + * it is assumed that the buffer does *not* have enough padding, and it is reallocated, enlarged + * and copied before parsing. + * + * @param buf The JSON to parse. Must have at least len + SIMDJSON_PADDING allocated bytes, unless + * realloc_if_needed is true. + * @param len The length of the JSON. + * @param realloc_if_needed Whether to reallocate and enlarge the JSON buffer to add padding. + * @return the document, or an error if the JSON is invalid. + */ + inline doc_ref_result parse(const uint8_t *buf, size_t len, bool realloc_if_needed = true) noexcept; + + /** + * Parse a JSON document and return a reference to it. + * + * The JSON document still lives in the parser: this is the most efficient way to parse JSON + * documents because it reuses the same buffers, but you *must* use the document before you + * destroy the parser or call parse() again. + * + * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what + * those bytes are initialized to, as long as they are allocated. If realloc_if_needed is true, + * it is assumed that the buffer does *not* have enough padding, and it is reallocated, enlarged + * and copied before parsing. + * + * @param buf The JSON to parse. Must have at least len + SIMDJSON_PADDING allocated bytes, unless + * realloc_if_needed is true. + * @param len The length of the JSON. + * @param realloc_if_needed Whether to reallocate and enlarge the JSON buffer to add padding. + * @return the document, or an error if the JSON is invalid. + */ + really_inline doc_ref_result parse(const char *buf, size_t len, bool realloc_if_needed = true) noexcept; + + /** + * Parse a JSON document and return a reference to it. + * + * The JSON document still lives in the parser: this is the most efficient way to parse JSON + * documents because it reuses the same buffers, but you *must* use the document before you + * destroy the parser or call parse() again. + * + * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what + * those bytes are initialized to, as long as they are allocated. If `str.capacity() - str.size() + * < SIMDJSON_PADDING`, the string will be copied to a string with larger capacity before parsing. + * + * @param s The JSON to parse. Must have at least len + SIMDJSON_PADDING allocated bytes, or + * a new string will be created with the extra padding. + * @return the document, or an error if the JSON is invalid. + */ + really_inline doc_ref_result parse(const std::string &s) noexcept; + + /** + * Parse a JSON document and return a reference to it. + * + * The JSON document still lives in the parser: this is the most efficient way to parse JSON + * documents because it reuses the same buffers, but you *must* use the document before you + * destroy the parser or call parse() again. + * + * @param s The JSON to parse. + * @return the document, or an error if the JSON is invalid. + */ + really_inline doc_ref_result parse(const padded_string &s) noexcept; + + // We do not want to allow implicit conversion from C string to std::string. + really_inline doc_ref_result parse(const char *buf) noexcept = delete; + + /** + * Parse a buffer containing many JSON documents. + * + * document::parser parser; + * for (const document &doc : parser.parse_many(buf, len)) { + * cout << std::string(doc["title"]) << endl; + * } + * + * ### Format + * + * The buffer must contain a series of one or more JSON documents, concatenated into a single + * buffer, separated by whitespace. It effectively parses until it has a fully valid document, + * then starts parsing the next document at that point. (It does this with more parallelism and + * lookahead than you might think, though.) + * + * documents that consist of an object or array may omit the whitespace between them, concatenating + * with no separator. documents that consist of a single primitive (i.e. documents that are not + * arrays or objects) MUST be separated with whitespace. + * + * ### Error Handling + * + * All errors are returned during iteration: if there is a global error such as memory allocation, + * it will be yielded as the first result. Iteration always stops after the first error. + * + * As with all other simdjson methods, non-exception error handling is readily available through + * the same interface, requiring you to check the error before using the document: + * + * document::parser parser; + * for (auto [doc, error] : parser.parse_many(buf, len)) { + * if (error) { cerr << error << endl; exit(1); } + * cout << std::string(doc["title"]) << endl; + * } + * + * ### REQUIRED: Buffer Padding + * + * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what + * those bytes are initialized to, as long as they are allocated. + * + * ### Threads + * + * When compiled with SIMDJSON_THREADS_ENABLED, this method will use a single thread under the + * hood to do some lookahead. + * + * ### Parser Capacity + * + * If the parser is unallocated, it will be auto-allocated to batch_size. If it is already + * allocated, it must have a capacity at least as large as batch_size. + * + * @param buf The concatenated JSON to parse. Must have at least len + SIMDJSON_PADDING allocated bytes. + * @param len The length of the concatenated JSON. + * @param batch_size The batch size to use. MUST be larger than the largest document. The sweet + * spot is cache-related: small enough to fit in cache, yet big enough to + * parse as many documents as possible in one tight loop. + * Defaults to 10MB, which has been a reasonable sweet spot in our tests. + * @return The stream. If there is an error, it will be returned during iteration. An empty input + * will yield 0 documents rather than an EMPTY error. Errors: + * - MEMALLOC if the parser is unallocated and memory allocation fails + * - CAPACITY if the parser already has a capacity, and it is less than batch_size + * - other json errors if parsing fails. + */ + inline stream parse_many(const uint8_t *buf, size_t len, size_t batch_size = 1000000) noexcept; + + /** + * Parse a buffer containing many JSON documents. + * + * document::parser parser; + * for (const document &doc : parser.parse_many(buf, len)) { + * cout << std::string(doc["title"]) << endl; + * } + * + * ### Format + * + * The buffer must contain a series of one or more JSON documents, concatenated into a single + * buffer, separated by whitespace. It effectively parses until it has a fully valid document, + * then starts parsing the next document at that point. (It does this with more parallelism and + * lookahead than you might think, though.) + * + * documents that consist of an object or array may omit the whitespace between them, concatenating + * with no separator. documents that consist of a single primitive (i.e. documents that are not + * arrays or objects) MUST be separated with whitespace. + * + * ### Error Handling + * + * All errors are returned during iteration: if there is a global error such as memory allocation, + * it will be yielded as the first result. Iteration always stops after the first error. + * + * As with all other simdjson methods, non-exception error handling is readily available through + * the same interface, requiring you to check the error before using the document: + * + * document::parser parser; + * for (auto [doc, error] : parser.parse_many(buf, len)) { + * if (error) { cerr << error << endl; exit(1); } + * cout << std::string(doc["title"]) << endl; + * } + * + * ### REQUIRED: Buffer Padding + * + * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what + * those bytes are initialized to, as long as they are allocated. + * + * ### Threads + * + * When compiled with SIMDJSON_THREADS_ENABLED, this method will use a single thread under the + * hood to do some lookahead. + * + * ### Parser Capacity + * + * If the parser is unallocated, it will be auto-allocated to batch_size. If it is already + * allocated, it must have a capacity at least as large as batch_size. + * + * @param buf The concatenated JSON to parse. Must have at least len + SIMDJSON_PADDING allocated bytes. + * @param len The length of the concatenated JSON. + * @param batch_size The batch size to use. MUST be larger than the largest document. The sweet + * spot is cache-related: small enough to fit in cache, yet big enough to + * parse as many documents as possible in one tight loop. + * Defaults to 10MB, which has been a reasonable sweet spot in our tests. + * @return The stream. If there is an error, it will be returned during iteration. An empty input + * will yield 0 documents rather than an EMPTY error. Errors: + * - MEMALLOC if the parser is unallocated and memory allocation fails + * - CAPACITY if the parser already has a capacity, and it is less than batch_size + * - other json errors if parsing fails + */ + inline stream parse_many(const char *buf, size_t len, size_t batch_size = 1000000) noexcept; + + /** + * Parse a buffer containing many JSON documents. + * + * document::parser parser; + * for (const document &doc : parser.parse_many(buf, len)) { + * cout << std::string(doc["title"]) << endl; + * } + * + * ### Format + * + * The buffer must contain a series of one or more JSON documents, concatenated into a single + * buffer, separated by whitespace. It effectively parses until it has a fully valid document, + * then starts parsing the next document at that point. (It does this with more parallelism and + * lookahead than you might think, though.) + * + * documents that consist of an object or array may omit the whitespace between them, concatenating + * with no separator. documents that consist of a single primitive (i.e. documents that are not + * arrays or objects) MUST be separated with whitespace. + * + * ### Error Handling + * + * All errors are returned during iteration: if there is a global error such as memory allocation, + * it will be yielded as the first result. Iteration always stops after the first error. + * + * As with all other simdjson methods, non-exception error handling is readily available through + * the same interface, requiring you to check the error before using the document: + * + * document::parser parser; + * for (auto [doc, error] : parser.parse_many(buf, len)) { + * if (error) { cerr << error << endl; exit(1); } + * cout << std::string(doc["title"]) << endl; + * } + * + * ### REQUIRED: Buffer Padding + * + * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what + * those bytes are initialized to, as long as they are allocated. + * + * ### Threads + * + * When compiled with SIMDJSON_THREADS_ENABLED, this method will use a single thread under the + * hood to do some lookahead. + * + * ### Parser Capacity + * + * If the parser is unallocated, it will be auto-allocated to batch_size. If it is already + * allocated, it must have a capacity at least as large as batch_size. + * + * @param s The concatenated JSON to parse. Must have at least len + SIMDJSON_PADDING allocated bytes. + * @param batch_size The batch size to use. MUST be larger than the largest document. The sweet + * spot is cache-related: small enough to fit in cache, yet big enough to + * parse as many documents as possible in one tight loop. + * Defaults to 10MB, which has been a reasonable sweet spot in our tests. + * @return he stream. If there is an error, it will be returned during iteration. An empty input + * will yield 0 documents rather than an EMPTY error. Errors: + * - MEMALLOC if the parser is unallocated and memory allocation fails + * - CAPACITY if the parser already has a capacity, and it is less than batch_size + * - other json errors if parsing fails + */ + inline stream parse_many(const std::string &s, size_t batch_size = 1000000) noexcept; + + /** + * Parse a buffer containing many JSON documents. + * + * document::parser parser; + * for (const document &doc : parser.parse_many(buf, len)) { + * cout << std::string(doc["title"]) << endl; + * } + * + * ### Format + * + * The buffer must contain a series of one or more JSON documents, concatenated into a single + * buffer, separated by whitespace. It effectively parses until it has a fully valid document, + * then starts parsing the next document at that point. (It does this with more parallelism and + * lookahead than you might think, though.) + * + * documents that consist of an object or array may omit the whitespace between them, concatenating + * with no separator. documents that consist of a single primitive (i.e. documents that are not + * arrays or objects) MUST be separated with whitespace. + * + * ### Error Handling + * + * All errors are returned during iteration: if there is a global error such as memory allocation, + * it will be yielded as the first result. Iteration always stops after the first error. + * + * As with all other simdjson methods, non-exception error handling is readily available through + * the same interface, requiring you to check the error before using the document: + * + * document::parser parser; + * for (auto [doc, error] : parser.parse_many(buf, len)) { + * if (error) { cerr << error << endl; exit(1); } + * cout << std::string(doc["title"]) << endl; + * } + * + * ### REQUIRED: Buffer Padding + * + * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what + * those bytes are initialized to, as long as they are allocated. + * + * ### Threads + * + * When compiled with SIMDJSON_THREADS_ENABLED, this method will use a single thread under the + * hood to do some lookahead. + * + * ### Parser Capacity + * + * If the parser is unallocated, it will be auto-allocated to batch_size. If it is already + * allocated, it must have a capacity at least as large as batch_size. + * + * @param s The concatenated JSON to parse. + * @param batch_size The batch size to use. MUST be larger than the largest document. The sweet + * spot is cache-related: small enough to fit in cache, yet big enough to + * parse as many documents as possible in one tight loop. + * Defaults to 10MB, which has been a reasonable sweet spot in our tests. + * @return he stream. If there is an error, it will be returned during iteration. An empty input + * will yield 0 documents rather than an EMPTY error. Errors: + * - MEMALLOC if the parser is unallocated and memory allocation fails + * - CAPACITY if the parser already has a capacity, and it is less than batch_size + * - other json errors if parsing fails + */ + inline stream parse_many(const padded_string &s, size_t batch_size = 1000000) noexcept; + + // We do not want to allow implicit conversion from C string to std::string. + really_inline doc_ref_result parse_many(const char *buf, size_t batch_size = 1000000) noexcept = delete; + + /** + * Current capacity: the largest document this parser can support without reallocating. + */ + really_inline size_t capacity() const noexcept; + + /** + * The maximum level of nested object and arrays supported by this parser. + */ + really_inline size_t max_depth() const noexcept; + + /** + * Ensure this parser has enough memory to process JSON documents up to `capacity` bytes in length + * and `max_depth` depth. + */ + WARN_UNUSED inline bool allocate_capacity(size_t capacity, size_t max_depth = DEFAULT_MAX_DEPTH); + + // type aliases for backcompat + using Iterator = document::iterator; + using InvalidJSON = simdjson_error; + + // Next location to write to in the tape + uint32_t current_loc{0}; + + // structural indices passed from stage 1 to stage 2 + uint32_t n_structural_indexes{0}; + std::unique_ptr structural_indexes; + + // location and return address of each open { or [ + std::unique_ptr containing_scope_offset; +#ifdef SIMDJSON_USE_COMPUTED_GOTO + std::unique_ptr ret_address; +#else + std::unique_ptr ret_address; +#endif + + // Next place to write a string + uint8_t *current_string_buf_loc; + + bool valid{false}; + error_code error{UNINITIALIZED}; + + // Document we're writing to + document doc; + + // + // TODO these are deprecated; use the results of parse instead. + // + + // returns true if the document parsed was valid + inline bool is_valid() const noexcept; + + // return an error code corresponding to the last parsing attempt, see + // simdjson.h will return UNITIALIZED if no parsing was attempted + inline int get_error_code() const noexcept; + + // return the string equivalent of "get_error_code" + inline std::string get_error_message() const noexcept; + + // print the json to std::ostream (should be valid) + // return false if the tape is likely wrong (e.g., you did not parse a valid + // JSON). + inline bool print_json(std::ostream &os) const noexcept; + inline bool dump_raw_tape(std::ostream &os) const noexcept; + + // + // Parser callbacks: these are internal! + // + // TODO find a way to do this without exposing the interface or crippling performance + // + + // this should be called when parsing (right before writing the tapes) + inline void init_stage2() noexcept; + really_inline error_code on_error(error_code new_error_code) noexcept; + really_inline error_code on_success(error_code success_code) noexcept; + really_inline bool on_start_document(uint32_t depth) noexcept; + really_inline bool on_start_object(uint32_t depth) noexcept; + really_inline bool on_start_array(uint32_t depth) noexcept; + // TODO we're not checking this bool + really_inline bool on_end_document(uint32_t depth) noexcept; + really_inline bool on_end_object(uint32_t depth) noexcept; + really_inline bool on_end_array(uint32_t depth) noexcept; + really_inline bool on_true_atom() noexcept; + really_inline bool on_false_atom() noexcept; + really_inline bool on_null_atom() noexcept; + really_inline uint8_t *on_start_string() noexcept; + really_inline bool on_end_string(uint8_t *dst) noexcept; + really_inline bool on_number_s64(int64_t value) noexcept; + really_inline bool on_number_u64(uint64_t value) noexcept; + really_inline bool on_number_double(double value) noexcept; + // + // Called before a parse is initiated. + // + // - Returns CAPACITY if the document is too large + // - Returns MEMALLOC if we needed to allocate memory and could not + // + WARN_UNUSED inline error_code init_parse(size_t len) noexcept; + +private: + // + // The maximum document length this parser supports. + // + // Buffers are large enough to handle any document up to this length. + // + size_t _capacity{0}; + + // + // The maximum depth (number of nested objects and arrays) supported by this parser. + // + // Defaults to DEFAULT_MAX_DEPTH. + // + size_t _max_depth{0}; + + // all nodes are stored on the doc.tape using a 64-bit word. + // + // strings, double and ints are stored as + // a 64-bit word with a pointer to the actual value + // + // + // + // for objects or arrays, store [ or { at the beginning and } and ] at the + // end. For the openings ([ or {), we annotate them with a reference to the + // location on the doc.tape of the end, and for then closings (} and ]), we + // annotate them with a reference to the location of the opening + // + // + + inline void write_tape(uint64_t val, tape_type t) noexcept; + inline void annotate_previous_loc(uint32_t saved_loc, uint64_t val) noexcept; + + // + // Set the current capacity: the largest document this parser can support without reallocating. + // + // This will allocate *or deallocate* as necessary. + // + // Returns false if allocation fails. + // + inline WARN_UNUSED bool set_capacity(size_t capacity); + + // + // Set the maximum level of nested object and arrays supported by this parser. + // + // This will allocate *or deallocate* as necessary. + // + // Returns false if allocation fails. + // + inline WARN_UNUSED bool set_max_depth(size_t max_depth); + + // Used internally to get the document + inline const document &get_document() const noexcept(false); + + template friend class document_iterator; +}; // class parser + +} // namespace simdjson + +#endif // SIMDJSON_DOCUMENT_PARSER_H \ No newline at end of file diff --git a/include/simdjson/error.h b/include/simdjson/error.h index b48f9eabd..ceebe9201 100644 --- a/include/simdjson/error.h +++ b/include/simdjson/error.h @@ -5,42 +5,60 @@ namespace simdjson { +/** + * All possible errors returned by simdjson. + */ enum error_code { - SUCCESS = 0, - SUCCESS_AND_HAS_MORE, //No errors and buffer still has more data - CAPACITY, // This parser can't support a document that big - MEMALLOC, // Error allocating memory, most likely out of memory - TAPE_ERROR, // Something went wrong while writing to the tape (stage 2), this - // is a generic error - DEPTH_ERROR, // Your document exceeds the user-specified depth limitation - STRING_ERROR, // Problem while parsing a string - T_ATOM_ERROR, // Problem while parsing an atom starting with the letter 't' - F_ATOM_ERROR, // Problem while parsing an atom starting with the letter 'f' - N_ATOM_ERROR, // Problem while parsing an atom starting with the letter 'n' - NUMBER_ERROR, // Problem while parsing a number - UTF8_ERROR, // the input is not valid UTF-8 - UNINITIALIZED, // unknown error, or uninitialized document - EMPTY, // no structural element found - UNESCAPED_CHARS, // found unescaped characters in a string. - UNCLOSED_STRING, // missing quote at the end - UNSUPPORTED_ARCHITECTURE, // unsupported architecture - INCORRECT_TYPE, // JSON element has a different type than user expected - NUMBER_OUT_OF_RANGE, // JSON number does not fit in 64 bits - NO_SUCH_FIELD, // JSON field not found in object - UNEXPECTED_ERROR // indicative of a bug in simdjson + SUCCESS = 0, ///< No error + SUCCESS_AND_HAS_MORE, ///< No error and buffer still has more data + CAPACITY, ///< This parser can't support a document that big + MEMALLOC, ///< Error allocating memory, most likely out of memory + TAPE_ERROR, ///< Something went wrong while writing to the tape (stage 2), this is a generic error + DEPTH_ERROR, ///< Your document exceeds the user-specified depth limitation + STRING_ERROR, ///< Problem while parsing a string + T_ATOM_ERROR, ///< Problem while parsing an atom starting with the letter 't' + F_ATOM_ERROR, ///< Problem while parsing an atom starting with the letter 'f' + N_ATOM_ERROR, ///< Problem while parsing an atom starting with the letter 'n' + NUMBER_ERROR, ///< Problem while parsing a number + UTF8_ERROR, ///< the input is not valid UTF-8 + UNINITIALIZED, ///< unknown error, or uninitialized document + EMPTY, ///< no structural element found + UNESCAPED_CHARS, ///< found unescaped characters in a string. + UNCLOSED_STRING, ///< missing quote at the end + UNSUPPORTED_ARCHITECTURE, ///< unsupported architecture + INCORRECT_TYPE, ///< JSON element has a different type than user expected + NUMBER_OUT_OF_RANGE, ///< JSON number does not fit in 64 bits + NO_SUCH_FIELD, ///< JSON field not found in object + UNEXPECTED_ERROR, ///< indicative of a bug in simdjson + /** @private Number of error codes */ + NUM_ERROR_CODES }; -const std::string &error_message(error_code error) noexcept; +/** + * Get the error message for the given error code. + * + * auto [doc, error] = document::parse("foo"); + * if (error) { printf("Error: %s\n", error_message(error)); } + * + * @return The error message. + */ +inline const char *error_message(error_code error) noexcept; struct invalid_json : public std::exception { - invalid_json(error_code _error) : error{_error} { } - const char *what() const noexcept { return error_message(error).c_str(); } + invalid_json(error_code _error) noexcept : error{_error} { } + const char *what() const noexcept { return error_message(error); } error_code error; }; -// TODO these are deprecated, remove +/** + * @deprecated This is an alias and will be removed, use error_code instead + */ using ErrorValues = error_code; -inline const std::string &error_message(int error) noexcept { return error_message(error_code(error)); } + +/** + * @deprecated Error codes should be stored and returned as `error_code`, use `error_message()` instead. + */ +inline const std::string &error_message(int error) noexcept; } // namespace simdjson diff --git a/include/simdjson/inline/document.h b/include/simdjson/inline/document.h index 777d3a765..2005913fc 100644 --- a/include/simdjson/inline/document.h +++ b/include/simdjson/inline/document.h @@ -445,9 +445,6 @@ inline document::doc_ref_result::operator document&() noexcept(false) { } return doc; } -inline const std::string &document::doc_ref_result::get_error_message() const noexcept { - return error_message(error); -} // // document::doc_result inline implementation @@ -461,16 +458,13 @@ inline document::doc_result::operator document() noexcept(false) { } return std::move(doc); } -inline const std::string &document::doc_result::get_error_message() const noexcept { - return error_message(error); -} // // document::parser inline implementation // inline bool document::parser::is_valid() const noexcept { return valid; } inline int document::parser::get_error_code() const noexcept { return error; } -inline std::string document::parser::get_error_message() const noexcept { return error_message(error); } +inline std::string document::parser::get_error_message() const noexcept { return error_message(int(error)); } inline bool document::parser::print_json(std::ostream &os) const noexcept { return is_valid() ? doc.print_json(os) : false; } diff --git a/include/simdjson/inline/error.h b/include/simdjson/inline/error.h new file mode 100644 index 000000000..3a488bbf3 --- /dev/null +++ b/include/simdjson/inline/error.h @@ -0,0 +1,55 @@ +#ifndef SIMDJSON_INLINE_ERROR_H +#define SIMDJSON_INLINE_ERROR_H + +#include "simdjson/error.h" +#include + +namespace simdjson::internal { + // We store the error code so we can validate the error message is associated with the right code + struct error_code_info { + error_code code; + std::string message; + }; + // These MUST match the codes in error_code. We check this constraint in basictests. + inline const error_code_info error_codes[] { + { SUCCESS, "No error" }, + { SUCCESS_AND_HAS_MORE, "No error and buffer still has more data" }, + { CAPACITY, "This parser can't support a document that big" }, + { MEMALLOC, "Error allocating memory, we're most likely out of memory" }, + { TAPE_ERROR, "Something went wrong while writing to the tape" }, + { DEPTH_ERROR, "The JSON document was too deep (too many nested objects and arrays)" }, + { STRING_ERROR, "Problem while parsing a string" }, + { T_ATOM_ERROR, "Problem while parsing an atom starting with the letter 't'" }, + { F_ATOM_ERROR, "Problem while parsing an atom starting with the letter 'f'" }, + { N_ATOM_ERROR, "Problem while parsing an atom starting with the letter 'n'" }, + { NUMBER_ERROR, "Problem while parsing a number" }, + { UTF8_ERROR, "The input is not valid UTF-8" }, + { UNINITIALIZED, "Uninitialized" }, + { EMPTY, "Empty: no JSON found" }, + { UNESCAPED_CHARS, "Within strings, some characters must be escaped, we found unescaped characters" }, + { UNCLOSED_STRING, "A string is opened, but never closed." }, + { UNSUPPORTED_ARCHITECTURE, "simdjson does not have an implementation supported by this CPU architecture (perhaps it's a non-SIMD CPU?)." }, + { INCORRECT_TYPE, "The JSON element does not have the requested type." }, + { NUMBER_OUT_OF_RANGE, "The JSON number is too large or too small to fit within the requested type." }, + { NO_SUCH_FIELD, "The JSON field referenced does not exist in this object." }, + { UNEXPECTED_ERROR, "Unexpected error, consider reporting this problem as you may have found a bug in simdjson" } + }; // error_messages[] +} // namespace simdjson::internal + +namespace simdjson { + +inline const char *error_message(error_code error) noexcept { + // If you're using error_code, we're trusting you got it from the enum. + return internal::error_codes[int(error)].message.c_str(); +} + +inline const std::string &error_message(int error) noexcept { + if (error < 0 || error >= error_code::NUM_ERROR_CODES) { + return internal::error_codes[UNEXPECTED_ERROR].message; + } + return internal::error_codes[error].message; +} + +} // namespace simdjson + +#endif // SIMDJSON_INLINE_ERROR_H diff --git a/singleheader/simdjson.h b/singleheader/simdjson.h index 37294bc6e..9d0454ff9 100644 --- a/singleheader/simdjson.h +++ b/singleheader/simdjson.h @@ -79,7 +79,7 @@ const std::string &error_message(error_code error) noexcept; struct invalid_json : public std::exception { invalid_json(error_code _error) : error{_error} { } - const char *what() const noexcept { return error_message(error).c_str(); } + const char *what() const noexcept { return error_message(error); } error_code error; }; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index bfe7bf7fd..796e52769 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -27,7 +27,6 @@ set(SIMDJSON_SRC # Load headers and sources set(SIMDJSON_SRC_HEADERS - error.cpp implementation.cpp isadetection.h jsonioutil.cpp diff --git a/src/error.cpp b/src/error.cpp deleted file mode 100644 index 1208ff615..000000000 --- a/src/error.cpp +++ /dev/null @@ -1,45 +0,0 @@ -#include "simdjson.h" -#include - -namespace simdjson { - -const std::map error_strings = { - {SUCCESS, "No error"}, - {SUCCESS_AND_HAS_MORE, "No error and buffer still has more data"}, - {CAPACITY, "This parser can't support a document that big"}, - {MEMALLOC, "Error allocating memory, we're most likely out of memory"}, - {TAPE_ERROR, "Something went wrong while writing to the tape"}, - {STRING_ERROR, "Problem while parsing a string"}, - {T_ATOM_ERROR, "Problem while parsing an atom starting with the letter 't'"}, - {F_ATOM_ERROR, "Problem while parsing an atom starting with the letter 'f'"}, - {N_ATOM_ERROR, "Problem while parsing an atom starting with the letter 'n'"}, - {NUMBER_ERROR, "Problem while parsing a number"}, - {UTF8_ERROR, "The input is not valid UTF-8"}, - {UNINITIALIZED, "Uninitialized"}, - {EMPTY, "Empty: no JSON found"}, - {UNESCAPED_CHARS, "Within strings, some characters must be escaped, we" - " found unescaped characters"}, - {UNCLOSED_STRING, "A string is opened, but never closed."}, - {UNSUPPORTED_ARCHITECTURE, "simdjson does not have an implementation" - " supported by this CPU architecture (perhaps" - " it's a non-SIMD CPU?)."}, - {INCORRECT_TYPE, "The JSON element does not have the requested type."}, - {NUMBER_OUT_OF_RANGE, "The JSON number is too large or too small to fit within the requested type."}, - {NO_SUCH_FIELD, "The JSON field referenced does not exist in this object."}, - {UNEXPECTED_ERROR, "Unexpected error, consider reporting this problem as" - " you may have found a bug in simdjson"}, -}; - -// string returned when the error code is not recognized -const std::string unexpected_error_msg {"Unexpected error"}; - -// returns a string matching the error code -const std::string &error_message(error_code code) noexcept { - auto keyvalue = error_strings.find(code); - if(keyvalue == error_strings.end()) { - return unexpected_error_msg; - } - return keyvalue->second; -} - -} // namespace simdjson diff --git a/src/simdjson.cpp b/src/simdjson.cpp index 4d8cb4bd8..a57aed701 100644 --- a/src/simdjson.cpp +++ b/src/simdjson.cpp @@ -1,5 +1,4 @@ #include "simdjson.h" -#include "error.cpp" #include "implementation.cpp" #include "jsonioutil.cpp" #include "jsonminifier.cpp" diff --git a/tests/basictests.cpp b/tests/basictests.cpp index 5caef7254..394186a56 100644 --- a/tests/basictests.cpp +++ b/tests/basictests.cpp @@ -534,13 +534,13 @@ bool document_stream_test() { size_t count = 0; for (auto [doc, error] : parser.parse_many(str, batch_size)) { if (error) { - printf("Error at on document %zd at batch size %zu: %s\n", count, batch_size, simdjson::error_message(error).c_str()); + printf("Error at on document %zd at batch size %zu: %s\n", count, batch_size, simdjson::error_message(error)); return false; } auto [keyid, error2] = doc["id"].as_int64_t(); if (error2) { - printf("Error getting id as int64 on document %zd at batch size %zu: %s\n", count, batch_size, simdjson::error_message(error2).c_str()); + printf("Error getting id as int64 on document %zd at batch size %zu: %s\n", count, batch_size, simdjson::error_message(error2)); return false; } @@ -582,13 +582,13 @@ bool document_stream_utf8_test() { size_t count = 0; for (auto [doc, error] : parser.parse_many(str, batch_size)) { if (error) { - printf("Error at on document %zd at batch size %zu: %s\n", count, batch_size, simdjson::error_message(error).c_str()); + printf("Error at on document %zd at batch size %zu: %s\n", count, batch_size, simdjson::error_message(error)); return false; } auto [keyid, error2] = doc["id"].as_int64_t(); if (error2) { - printf("Error getting id as int64 on document %zd at batch size %zu: %s\n", count, batch_size, simdjson::error_message(error2).c_str()); + printf("Error getting id as int64 on document %zd at batch size %zu: %s\n", count, batch_size, simdjson::error_message(error2)); return false; } @@ -864,6 +864,23 @@ namespace dom_api { } } +bool error_messages_in_correct_order() { + using namespace simdjson; + using namespace simdjson::internal; + using namespace std; + if ((sizeof(error_codes)/sizeof(error_code_info)) != NUM_ERROR_CODES) { + cerr << "error_codes does not have all codes in error_code enum (or too many)" << endl; + return false; + } + for (int i=0; i