diff --git a/doc/dom.md b/doc/dom.md index a19856190..1f5d985eb 100644 --- a/doc/dom.md +++ b/doc/dom.md @@ -168,6 +168,23 @@ Once you have an element, you can navigate it with idiomatic C++ iterators, oper `SIMDJSON_MINUS_ZERO_AS_FLOAT` to `1` when building simdjson, you can get that `-0` is mapped to `-0.0` as in JavaScript. You can get the desired effect by building simdjson with cmake setting the `SIMDJSON_MINUS_ZERO_AS_FLOAT` to on: `cmake -B build -D SIMDJSON_MINUS_ZERO_AS_FLOAT=ON`. +* **Big Integer Support (opt-in):** By default, integers that exceed the 64-bit range cause parsing to fail with `BIGINT_ERROR`. You can opt in to big integer support so that these numbers are stored as raw digit strings on the tape instead: + ```cpp + simdjson::dom::parser parser; + parser.number_as_string(true); // opt-in, default false + simdjson::dom::element doc; + auto error = parser.parse("[1, 123456789012345678901]"_padded).get(doc); + if (error) { std::cerr << error << std::endl; return EXIT_FAILURE; } + for (simdjson::dom::element elem : doc) { + if (elem.is_bigint()) { + std::string_view digits; + error = elem.get_bigint().get(digits); + if (error) { std::cerr << error << std::endl; return EXIT_FAILURE; } + std::cout << "big integer: " << digits << std::endl; + } + } + ``` + When enabled, big integers have type `element_type::BIGINT`. Calling `get_int64()`, `get_uint64()`, or `get_double()` on a big integer returns `INCORRECT_TYPE`. Normal numbers (int64, uint64, double) are unaffected. * **Field Access:** To get the value of the "foo" field in an object, use `object["foo"]`. * **Array Iteration:** To iterate through an array, use `for (auto value : array) { ... }`. If you know the type of the value, you can cast it right there, too! `for (double value : array) { ... }` @@ -180,7 +197,7 @@ Once you have an element, you can navigate it with idiomatic C++ iterators, oper * **Array and Object size** Given an array or an object, you can get its size (number of elements or keys) with the `size()` method. * **Checking an Element Type:** You can check an element's type with `element.type()`. It - returns an `element_type` with values such as `simdjson::dom::element_type::ARRAY`, `simdjson::dom::element_type::OBJECT`, `simdjson::dom::element_type::INT64`, `simdjson::dom::element_type::UINT64`,`simdjson::dom::element_type::DOUBLE`, `simdjson::dom::element_type::STRING`, `simdjson::dom::element_type::BOOL` or, `simdjson::dom::element_type::NULL_VALUE`. + returns an `element_type` with values such as `simdjson::dom::element_type::ARRAY`, `simdjson::dom::element_type::OBJECT`, `simdjson::dom::element_type::INT64`, `simdjson::dom::element_type::UINT64`,`simdjson::dom::element_type::DOUBLE`, `simdjson::dom::element_type::STRING`, `simdjson::dom::element_type::BOOL`, `simdjson::dom::element_type::NULL_VALUE` or, `simdjson::dom::element_type::BIGINT` (when big integer support is enabled). * **Output to streams and strings:** Given a document or an element (or node) out of a JSON document, you can output a minified string version using the C++ stream idiom (`out << element`). You can also request the construction of a minified string version (`simdjson::minify(element)`) or a prettified string version (`simdjson::prettify(element)`). Numbers are serialized as 64-bit floating-point numbers (`double`). ### Examples diff --git a/doc/tape.md b/doc/tape.md index 934a871b1..919635944 100644 --- a/doc/tape.md +++ b/doc/tape.md @@ -93,6 +93,14 @@ Float values are represented as two 64-bit tape elements: Performance consideration: We store numbers of the main tape because we believe that locality of reference is helpful for performance. +## Big Integers + +When a JSON integer exceeds the 64-bit range (both signed and unsigned), it is classified as a big integer. By default, parsing returns `BIGINT_ERROR`. When `parser.number_as_string(true)` is set, the raw digits are stored on the string tape using the same format as strings (4-byte little-endian length prefix, followed by the UTF-8 digit bytes, followed by a null terminator). + +A big integer is represented on the main tape as the 64-bit tape element `('Z' << 56) + x` where the payload `x` is the location on the string tape of the null-terminated digit string. + +For example, the JSON value `99999999999999999999` would be stored as the string `"99999999999999999999"` on the string tape, with a `'Z'` tag on the main tape. Negative big integers include the leading minus sign. + ## Root node Each JSON document will have two special 64-bit tape elements representing a root node, one at the beginning and one at the end. diff --git a/fuzz/fuzz_dump.cpp b/fuzz/fuzz_dump.cpp index 2c5b1517e..50343ee6b 100644 --- a/fuzz/fuzz_dump.cpp +++ b/fuzz/fuzz_dump.cpp @@ -54,6 +54,9 @@ static void print_json(std::ostream& os, simdjson::dom::element element) noexcep case simdjson::dom::element_type::NULL_VALUE: os << "null" << endl; break; + case simdjson::dom::element_type::BIGINT: + os << element.get_bigint().value_unsafe() << endl; + break; } } extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { diff --git a/include/simdjson/dom/document-inl.h b/include/simdjson/dom/document-inl.h index 40d74b999..9caa7a3c0 100644 --- a/include/simdjson/dom/document-inl.h +++ b/include/simdjson/dom/document-inl.h @@ -141,6 +141,15 @@ inline bool document::dump_raw_tape(std::ostream &os) const noexcept { case 'r': // we start and end with the root node // should we be hitting the root node? return false; + case 'Z': // we have a big integer + os << "bigint "; + std::memcpy(&string_length, string_buf.get() + payload, sizeof(uint32_t)); + os << std::string_view( + reinterpret_cast(string_buf.get() + payload + sizeof(uint32_t)), + string_length + ); + os << '\n'; + break; default: return false; } diff --git a/include/simdjson/dom/element-inl.h b/include/simdjson/dom/element-inl.h index f5f9ed5b1..ff4d3d3a8 100644 --- a/include/simdjson/dom/element-inl.h +++ b/include/simdjson/dom/element-inl.h @@ -81,6 +81,10 @@ simdjson_inline simdjson_result simdjson_result::get_bool() if (error()) { return error(); } return first.get_bool(); } +simdjson_inline simdjson_result simdjson_result::get_bigint() const noexcept { + if (error()) { return error(); } + return first.get_bigint(); +} simdjson_inline bool simdjson_result::is_array() const noexcept { return !error() && first.is_array(); @@ -110,6 +114,9 @@ simdjson_inline bool simdjson_result::is_bool() const noexcept { simdjson_inline bool simdjson_result::is_null() const noexcept { return !error() && first.is_null(); } +simdjson_inline bool simdjson_result::is_bigint() const noexcept { + return !error() && first.is_bigint(); +} simdjson_inline simdjson_result simdjson_result::operator[](std::string_view key) const noexcept { if (error()) { return error(); } @@ -218,6 +225,15 @@ inline simdjson_result element::get_bool() const noexcept { } return INCORRECT_TYPE; } +inline simdjson_result element::get_bigint() const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); + switch (tape.tape_ref_type()) { + case internal::tape_type::BIGINT: + return tape.get_string_view(); + default: + return INCORRECT_TYPE; + } +} inline simdjson_result element::get_c_str() const noexcept { SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 switch (tape.tape_ref_type()) { @@ -360,6 +376,10 @@ inline bool element::is_null() const noexcept { return tape.is_null_on_tape(); } +inline bool element::is_bigint() const noexcept { + return tape.tape_ref_type() == internal::tape_type::BIGINT; +} + #if SIMDJSON_EXCEPTIONS inline element::operator bool() const noexcept(false) { return get(); } @@ -492,6 +512,8 @@ inline std::ostream& operator<<(std::ostream& out, element_type type) { return out << "bool"; case element_type::NULL_VALUE: return out << "null"; + case element_type::BIGINT: + return out << "bigint"; default: return out << "unexpected content!!!"; // abort() usage is forbidden in the library } diff --git a/include/simdjson/dom/element.h b/include/simdjson/dom/element.h index a661718f4..07877fed3 100644 --- a/include/simdjson/dom/element.h +++ b/include/simdjson/dom/element.h @@ -21,7 +21,8 @@ enum class element_type { DOUBLE = 'd', ///< double: Any number with a "." or "e" that fits in double. STRING = '"', ///< std::string_view BOOL = 't', ///< bool - NULL_VALUE = 'n' ///< null + NULL_VALUE = 'n', ///< null + BIGINT = 'Z' ///< std::string_view: big integer stored as raw digit string }; /** @@ -120,6 +121,14 @@ public: */ inline simdjson_result get_bool() const noexcept; + /** + * Read this element as a big integer (raw digit string). + * + * @returns A string_view of the raw digits, or: + * INCORRECT_TYPE if the JSON element is not a big integer. + */ + inline simdjson_result get_bigint() const noexcept; + /** * Whether this element is a json array. * @@ -175,6 +184,11 @@ public: */ inline bool is_null() const noexcept; + /** + * Whether this element is a big integer (number exceeding 64-bit range). + */ + inline bool is_bigint() const noexcept; + /** * Tell whether the value can be cast to provided type (T). * @@ -533,6 +547,7 @@ public: simdjson_inline simdjson_result get_uint64() const noexcept; simdjson_inline simdjson_result get_double() const noexcept; simdjson_inline simdjson_result get_bool() const noexcept; + simdjson_inline simdjson_result get_bigint() const noexcept; simdjson_inline bool is_array() const noexcept; simdjson_inline bool is_object() const noexcept; @@ -543,6 +558,7 @@ public: simdjson_inline bool is_number() const noexcept; simdjson_inline bool is_bool() const noexcept; simdjson_inline bool is_null() const noexcept; + simdjson_inline bool is_bigint() const noexcept; simdjson_inline simdjson_result operator[](std::string_view key) const noexcept; simdjson_inline simdjson_result operator[](const char *key) const noexcept; diff --git a/include/simdjson/dom/parser-inl.h b/include/simdjson/dom/parser-inl.h index bf26b298e..b116d63b3 100644 --- a/include/simdjson/dom/parser-inl.h +++ b/include/simdjson/dom/parser-inl.h @@ -132,6 +132,7 @@ inline simdjson_result parser::parse_into_document(document& provided_d buf += 3; len -= 3; } + implementation->_number_as_string = _number_as_string; _error = implementation->parse(buf, len, provided_doc); if (_error) { return _error; } diff --git a/include/simdjson/dom/parser.h b/include/simdjson/dom/parser.h index 6fa461b81..eee411bf7 100644 --- a/include/simdjson/dom/parser.h +++ b/include/simdjson/dom/parser.h @@ -610,6 +610,13 @@ public: inline bool dump_raw_tape(std::ostream &os) const noexcept; + /** + * When enabled, big integers (exceeding uint64 range) are stored as strings + * in the tape instead of returning BIGINT_ERROR. Default: false. + */ + inline void number_as_string(bool enabled) noexcept { _number_as_string = enabled; } + inline bool number_as_string() const noexcept { return _number_as_string; } + private: /** * The maximum document length this parser will automatically support. @@ -618,6 +625,9 @@ private: */ size_t _max_capacity; + /** Whether to store big integers as strings instead of returning BIGINT_ERROR */ + bool _number_as_string{false}; + /** * The loaded buffer (reused each time load() is called) */ diff --git a/include/simdjson/dom/serialization-inl.h b/include/simdjson/dom/serialization-inl.h index 6baf7f848..cfb963cad 100644 --- a/include/simdjson/dom/serialization-inl.h +++ b/include/simdjson/dom/serialization-inl.h @@ -460,6 +460,12 @@ inline void string_builder::append(simdjson::dom::element value) { case tape_type::STRING: format.string(iter.get_string_view()); break; + case tape_type::BIGINT: { + // Big integer stored as string — output raw digits (no quotes) + auto sv = iter.get_string_view(); + format.chars(sv.data(), sv.data() + sv.size()); + break; + } case tape_type::INT64: format.number(iter.next_tape_value()); iter.json_index++; // numbers take up 2 spots, so we need to increment diff --git a/include/simdjson/internal/dom_parser_implementation.h b/include/simdjson/internal/dom_parser_implementation.h index a93fe38ff..8d8401a2d 100644 --- a/include/simdjson/internal/dom_parser_implementation.h +++ b/include/simdjson/internal/dom_parser_implementation.h @@ -212,6 +212,12 @@ protected: */ size_t _max_depth{0}; +public: + /** Whether to store big integers as strings instead of returning BIGINT_ERROR */ + bool _number_as_string{false}; + +protected: + // Declaring these so that subclasses can use them to implement their constructors. simdjson_inline dom_parser_implementation() noexcept; simdjson_inline dom_parser_implementation(dom_parser_implementation &&other) noexcept; diff --git a/include/simdjson/internal/tape_type.h b/include/simdjson/internal/tape_type.h index d43c57c7a..938c5cf0f 100644 --- a/include/simdjson/internal/tape_type.h +++ b/include/simdjson/internal/tape_type.h @@ -19,7 +19,8 @@ enum class tape_type { DOUBLE = 'd', TRUE_VALUE = 't', FALSE_VALUE = 'f', - NULL_VALUE = 'n' + NULL_VALUE = 'n', + BIGINT = 'Z' // Big integer stored as string in string buffer }; // enum class tape_type } // namespace internal diff --git a/src/generic/stage2/tape_builder.h b/src/generic/stage2/tape_builder.h index 52931010f..58041aa99 100644 --- a/src/generic/stage2/tape_builder.h +++ b/src/generic/stage2/tape_builder.h @@ -172,7 +172,23 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_string( simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("number"); - return numberparsing::parse_number(value, tape); + error_code err = numberparsing::parse_number(value, tape); + if (simdjson_unlikely(err == BIGINT_ERROR && + iter.dom_parser._number_as_string)) { + // Write big integer to string buffer using the same format as strings. + // Scan digits the same way parse_number does (skip optional '-', then digits). + const uint8_t *p = value; + if (*p == '-') p++; + while (numberparsing::is_digit(*p)) p++; + size_t len = size_t(p - value); + tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::BIGINT); + uint8_t *dst = current_string_buf_loc + sizeof(uint32_t); + memcpy(dst, value, len); + dst += len; + on_end_string(dst); + return SUCCESS; + } + return err; } simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { diff --git a/src/generic/stage2/tape_writer.h b/src/generic/stage2/tape_writer.h index 947aa6d09..1f1f57777 100644 --- a/src/generic/stage2/tape_writer.h +++ b/src/generic/stage2/tape_writer.h @@ -26,6 +26,9 @@ struct tape_writer { /** Write a double value to tape. */ simdjson_inline void append_double(double value) noexcept; + /** Write a big integer (as string) to tape. src points to first digit, len is byte count. */ + simdjson_inline void append_bigint(const uint8_t *src, size_t len, uint8_t *&string_buf) noexcept; + /** * Append a tape entry (an 8-bit type,and 56 bits worth of value). */ @@ -109,6 +112,18 @@ simdjson_inline void tape_writer::write(uint64_t &tape_loc, uint64_t val, intern tape_loc = val | ((uint64_t(char(t))) << 56); } +simdjson_inline void tape_writer::append_bigint(const uint8_t *src, size_t len, uint8_t *&string_buf) noexcept { + // Write to string buffer: [4-byte LE length][digits][null] + uint32_t str_len = uint32_t(len); + memcpy(string_buf, &str_len, sizeof(uint32_t)); + memcpy(string_buf + sizeof(uint32_t), src, len); + string_buf[sizeof(uint32_t) + len] = 0; + // Tape entry: offset into string buffer + // The caller must set the offset relative to doc.string_buf base + append(0, internal::tape_type::BIGINT); // placeholder offset, caller patches + string_buf += sizeof(uint32_t) + len + 1; +} + } // namespace stage2 } // unnamed namespace } // namespace SIMDJSON_IMPLEMENTATION diff --git a/tests/dom/CMakeLists.txt b/tests/dom/CMakeLists.txt index 5534658b7..db20bdb0c 100644 --- a/tests/dom/CMakeLists.txt +++ b/tests/dom/CMakeLists.txt @@ -11,6 +11,7 @@ add_cpp_test(document_tests LABELS dom acceptance per_implementation add_cpp_test(errortests LABELS dom acceptance per_implementation) add_cpp_test(extracting_values_example LABELS dom acceptance per_implementation) add_cpp_test(integer_tests LABELS dom acceptance per_implementation) +add_cpp_test(big_integer_tests LABELS dom acceptance per_implementation) add_cpp_test(jsoncheck LABELS dom acceptance per_implementation) add_cpp_test(json_path_tests LABELS dom acceptance per_implementation) add_cpp_test(minefieldcheck LABELS dom acceptance per_implementation) diff --git a/tests/dom/big_integer_tests.cpp b/tests/dom/big_integer_tests.cpp new file mode 100644 index 000000000..98249a685 --- /dev/null +++ b/tests/dom/big_integer_tests.cpp @@ -0,0 +1,151 @@ +#include "simdjson.h" +#include "test_macros.h" +#include + +using namespace simdjson; + +namespace big_integer_tests { + + bool option_off_by_default() { + TEST_START(); + dom::parser parser; + ASSERT_EQUAL(parser.number_as_string(), false); + TEST_SUCCEED(); + } + + bool default_returns_bigint_error() { + TEST_START(); + dom::parser parser; + auto result = parser.parse(R"({"val":123456789012345678901})"_padded); + ASSERT_EQUAL(result.error(), BIGINT_ERROR); + TEST_SUCCEED(); + } + + bool opt_in_parses_big_integer() { + TEST_START(); + dom::parser parser; + parser.number_as_string(true); + dom::element doc; + ASSERT_SUCCESS(parser.parse(R"({"val":123456789012345678901})"_padded).get(doc)); + dom::element val; + ASSERT_SUCCESS(doc["val"].get(val)); + ASSERT_EQUAL(val.type(), dom::element_type::BIGINT); + ASSERT_TRUE(val.is_bigint()); + std::string_view digits; + ASSERT_SUCCESS(val.get_bigint().get(digits)); + ASSERT_EQUAL(digits, "123456789012345678901"); + // Verify streaming works (same pattern as readme_examples) + std::ostringstream os; + os << val.get_bigint().value_unsafe(); + ASSERT_EQUAL(os.str(), "123456789012345678901"); + TEST_SUCCEED(); + } + + bool negative_big_integer() { + TEST_START(); + dom::parser parser; + parser.number_as_string(true); + dom::element doc; + ASSERT_SUCCESS(parser.parse(R"({"val":-12345678901234567890})"_padded).get(doc)); + dom::element val; + ASSERT_SUCCESS(doc["val"].get(val)); + ASSERT_EQUAL(val.type(), dom::element_type::BIGINT); + std::string_view digits; + ASSERT_SUCCESS(val.get_bigint().get(digits)); + ASSERT_EQUAL(digits, "-12345678901234567890"); + TEST_SUCCEED(); + } + + bool big_integer_in_array() { + TEST_START(); + dom::parser parser; + parser.number_as_string(true); + dom::element doc; + ASSERT_SUCCESS(parser.parse(R"([1, 123456789012345678901, 3])"_padded).get(doc)); + dom::array arr; + ASSERT_SUCCESS(doc.get_array().get(arr)); + size_t count = 0; + for (auto elem : arr) { (void)elem; count++; } + ASSERT_EQUAL(count, 3); + TEST_SUCCEED(); + } + + bool serialization_preserves_digits() { + TEST_START(); + dom::parser parser; + parser.number_as_string(true); + dom::element doc; + ASSERT_SUCCESS(parser.parse(R"({"val":123456789012345678901})"_padded).get(doc)); + std::string output = simdjson::to_string(doc); + // Raw digits should appear + ASSERT_TRUE(output.find("123456789012345678901") != std::string::npos); + // Must not be quoted — it's a number, not a string + ASSERT_EQUAL(output.find("\"123456789012345678901\""), std::string::npos); + TEST_SUCCEED(); + } + + bool normal_numbers_unaffected() { + TEST_START(); + dom::parser parser; + parser.number_as_string(true); + dom::element doc; + ASSERT_SUCCESS(parser.parse(R"({"a":42,"b":-1,"c":3.14,"d":18446744073709551615})"_padded).get(doc)); + int64_t a; + ASSERT_SUCCESS(doc["a"].get_int64().get(a)); + ASSERT_EQUAL(a, 42); + uint64_t d; + ASSERT_SUCCESS(doc["d"].get_uint64().get(d)); + ASSERT_EQUAL(d, 18446744073709551615ULL); + TEST_SUCCEED(); + } + + bool type_checks() { + TEST_START(); + dom::parser parser; + parser.number_as_string(true); + dom::element doc; + ASSERT_SUCCESS(parser.parse(R"({"val":123456789012345678901})"_padded).get(doc)); + dom::element val; + ASSERT_SUCCESS(doc["val"].get(val)); + // is_bigint should be true + ASSERT_TRUE(val.is_bigint()); + // other type checks should be false + ASSERT_FALSE(val.is_int64()); + ASSERT_FALSE(val.is_uint64()); + ASSERT_FALSE(val.is_double()); + ASSERT_FALSE(val.is_string()); + ASSERT_FALSE(val.is_bool()); + ASSERT_FALSE(val.is_null()); + // get_int64/uint64/double should return INCORRECT_TYPE + ASSERT_ERROR(val.get_int64(), INCORRECT_TYPE); + ASSERT_ERROR(val.get_uint64(), INCORRECT_TYPE); + ASSERT_ERROR(val.get_double(), INCORRECT_TYPE); + ASSERT_ERROR(val.get_string(), INCORRECT_TYPE); + TEST_SUCCEED(); + } + + bool element_type_output() { + TEST_START(); + std::ostringstream os; + os << dom::element_type::BIGINT; + ASSERT_EQUAL(os.str(), "bigint"); + TEST_SUCCEED(); + } + + bool run() { + return option_off_by_default() && + default_returns_bigint_error() && + opt_in_parses_big_integer() && + negative_big_integer() && + big_integer_in_array() && + serialization_preserves_digits() && + normal_numbers_unaffected() && + type_checks() && + element_type_output(); + } + +} // namespace big_integer_tests + +int main() { + return big_integer_tests::run() ? 0 : 1; +} diff --git a/tests/dom/readme_examples.cpp b/tests/dom/readme_examples.cpp index e92812a57..da5c6dbb5 100644 --- a/tests/dom/readme_examples.cpp +++ b/tests/dom/readme_examples.cpp @@ -273,6 +273,9 @@ namespace treewalk_1 { case dom::element_type::NULL_VALUE: cout << "null" << endl; break; + case dom::element_type::BIGINT: + cout << element.get_bigint().value_unsafe() << endl; + break; } }